drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_bus_pci.h>
  19 #include <rte_ip.h>
  20 #include <rte_gre.h>
  21 #include <rte_vxlan.h>
  22 #include <rte_gtp.h>
  23 #include <rte_eal_paging.h>
  24 #include <rte_mpls.h>
  25 #include <rte_mtr.h>
  26 #include <rte_mtr_driver.h>
  27 #include <rte_tailq.h>
  28
  29 #include <mlx5_glue.h>
  30 #include <mlx5_devx_cmds.h>
  31 #include <mlx5_prm.h>
  32 #include <mlx5_malloc.h>
  33
  34 #include "mlx5_defs.h"
  35 #include "mlx5.h"
  36 #include "mlx5_common_os.h"
  37 #include "mlx5_flow.h"
  38 #include "mlx5_flow_os.h"
  39 #include "mlx5_rx.h"
  40 #include "mlx5_tx.h"
  41 #include "rte_pmd_mlx5.h"
  42
  43 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  44
  45 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  46 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR_ESWITCH
  50 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  51 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  52 #endif
  53 #endif
  54
  55 #ifndef HAVE_MLX5DV_DR
  56 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  57 #endif
  58
  59 /* VLAN header definitions */
  60 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  61 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  62 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  63 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  64 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  65
  66 union flow_dv_attr {
  67         struct {
  68                 uint32_t valid:1;
  69                 uint32_t ipv4:1;
  70                 uint32_t ipv6:1;
  71                 uint32_t tcp:1;
  72                 uint32_t udp:1;
  73                 uint32_t reserved:27;
  74         };
  75         uint32_t attr;
  76 };
  77
  78 static int
  79 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
  80                              struct mlx5_flow_tbl_resource *tbl);
  81
  82 static int
  83 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  84                                      uint32_t encap_decap_idx);
  85
  86 static int
  87 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  88                                         uint32_t port_id);
  89 static void
  90 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss);
  91
  92 static int
  93 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
  94                                   uint32_t rix_jump);
  95
  96 static int16_t
  97 flow_dv_get_esw_manager_vport_id(struct rte_eth_dev *dev)
  98 {
  99         struct mlx5_priv *priv = dev->data->dev_private;
 100         struct mlx5_common_device *cdev = priv->sh->cdev;
 101
 102         if (cdev->config.hca_attr.esw_mgr_vport_id_valid)
 103                 return (int16_t)cdev->config.hca_attr.esw_mgr_vport_id;
 104
 105         if (priv->pci_dev == NULL)
 106                 return 0;
 107         switch (priv->pci_dev->id.device_id) {
 108         case PCI_DEVICE_ID_MELLANOX_CONNECTX5BF:
 109         case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF:
 110         case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF:
 111                 return (int16_t)0xfffe;
 112         default:
 113                 return 0;
 114         }
 115 }
 116
 117 /**
 118  * Initialize flow attributes structure according to flow items' types.
 119  *
 120  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
 121  * mode. For tunnel mode, the items to be modified are the outermost ones.
 122  *
 123  * @param[in] item
 124  *   Pointer to item specification.
 125  * @param[out] attr
 126  *   Pointer to flow attributes structure.
 127  * @param[in] dev_flow
 128  *   Pointer to the sub flow.
 129  * @param[in] tunnel_decap
 130  *   Whether action is after tunnel decapsulation.
 131  */
 132 static void
 133 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 134                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 135 {
 136         uint64_t layers = dev_flow->handle->layers;
 137
 138         /*
 139          * If layers is already initialized, it means this dev_flow is the
 140          * suffix flow, the layers flags is set by the prefix flow. Need to
 141          * use the layer flags from prefix flow as the suffix flow may not
 142          * have the user defined items as the flow is split.
 143          */
 144         if (layers) {
 145                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 146                         attr->ipv4 = 1;
 147                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 148                         attr->ipv6 = 1;
 149                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 150                         attr->tcp = 1;
 151                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 152                         attr->udp = 1;
 153                 attr->valid = 1;
 154                 return;
 155         }
 156         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 157                 uint8_t next_protocol = 0xff;
 158                 switch (item->type) {
 159                 case RTE_FLOW_ITEM_TYPE_GRE:
 160                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 161                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 162                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 163                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 164                 case RTE_FLOW_ITEM_TYPE_MPLS:
 165                 case RTE_FLOW_ITEM_TYPE_GTP:
 166                         if (tunnel_decap)
 167                                 attr->attr = 0;
 168                         break;
 169                 case RTE_FLOW_ITEM_TYPE_IPV4:
 170                         if (!attr->ipv6)
 171                                 attr->ipv4 = 1;
 172                         if (item->mask != NULL &&
 173                             ((const struct rte_flow_item_ipv4 *)
 174                             item->mask)->hdr.next_proto_id)
 175                                 next_protocol =
 176                                     ((const struct rte_flow_item_ipv4 *)
 177                                       (item->spec))->hdr.next_proto_id &
 178                                     ((const struct rte_flow_item_ipv4 *)
 179                                       (item->mask))->hdr.next_proto_id;
 180                         if ((next_protocol == IPPROTO_IPIP ||
 181                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 182                                 attr->attr = 0;
 183                         break;
 184                 case RTE_FLOW_ITEM_TYPE_IPV6:
 185                         if (!attr->ipv4)
 186                                 attr->ipv6 = 1;
 187                         if (item->mask != NULL &&
 188                             ((const struct rte_flow_item_ipv6 *)
 189                             item->mask)->hdr.proto)
 190                                 next_protocol =
 191                                     ((const struct rte_flow_item_ipv6 *)
 192                                       (item->spec))->hdr.proto &
 193                                     ((const struct rte_flow_item_ipv6 *)
 194                                       (item->mask))->hdr.proto;
 195                         if ((next_protocol == IPPROTO_IPIP ||
 196                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 197                                 attr->attr = 0;
 198                         break;
 199                 case RTE_FLOW_ITEM_TYPE_UDP:
 200                         if (!attr->tcp)
 201                                 attr->udp = 1;
 202                         break;
 203                 case RTE_FLOW_ITEM_TYPE_TCP:
 204                         if (!attr->udp)
 205                                 attr->tcp = 1;
 206                         break;
 207                 default:
 208                         break;
 209                 }
 210         }
 211         attr->valid = 1;
 212 }
 213
 214 /*
 215  * Convert rte_mtr_color to mlx5 color.
 216  *
 217  * @param[in] rcol
 218  *   rte_mtr_color.
 219  *
 220  * @return
 221  *   mlx5 color.
 222  */
 223 static inline int
 224 rte_col_2_mlx5_col(enum rte_color rcol)
 225 {
 226         switch (rcol) {
 227         case RTE_COLOR_GREEN:
 228                 return MLX5_FLOW_COLOR_GREEN;
 229         case RTE_COLOR_YELLOW:
 230                 return MLX5_FLOW_COLOR_YELLOW;
 231         case RTE_COLOR_RED:
 232                 return MLX5_FLOW_COLOR_RED;
 233         default:
 234                 break;
 235         }
 236         return MLX5_FLOW_COLOR_UNDEFINED;
 237 }
 238
 239 struct field_modify_info {
 240         uint32_t size; /* Size of field in protocol header, in bytes. */
 241         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 242         enum mlx5_modification_field id;
 243 };
 244
 245 struct field_modify_info modify_eth[] = {
 246         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 247         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 248         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 249         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 250         {0, 0, 0},
 251 };
 252
 253 struct field_modify_info modify_vlan_out_first_vid[] = {
 254         /* Size in bits !!! */
 255         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 256         {0, 0, 0},
 257 };
 258
 259 struct field_modify_info modify_ipv4[] = {
 260         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 261         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 262         {4, 12, MLX5_MODI_OUT_SIPV4},
 263         {4, 16, MLX5_MODI_OUT_DIPV4},
 264         {0, 0, 0},
 265 };
 266
 267 struct field_modify_info modify_ipv6[] = {
 268         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 269         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 270         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 271         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 272         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 273         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 274         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 275         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 276         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 277         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 278         {0, 0, 0},
 279 };
 280
 281 struct field_modify_info modify_udp[] = {
 282         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 283         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 284         {0, 0, 0},
 285 };
 286
 287 struct field_modify_info modify_tcp[] = {
 288         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 289         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 290         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 291         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 292         {0, 0, 0},
 293 };
 294
 295 static void
 296 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 297                           uint8_t next_protocol, uint64_t *item_flags,
 298                           int *tunnel)
 299 {
 300         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 301                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 302         if (next_protocol == IPPROTO_IPIP) {
 303                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 304                 *tunnel = 1;
 305         }
 306         if (next_protocol == IPPROTO_IPV6) {
 307                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 308                 *tunnel = 1;
 309         }
 310 }
 311
 312 static inline struct mlx5_hlist *
 313 flow_dv_hlist_prepare(struct mlx5_dev_ctx_shared *sh, struct mlx5_hlist **phl,
 314                      const char *name, uint32_t size, bool direct_key,
 315                      bool lcores_share, void *ctx,
 316                      mlx5_list_create_cb cb_create,
 317                      mlx5_list_match_cb cb_match,
 318                      mlx5_list_remove_cb cb_remove,
 319                      mlx5_list_clone_cb cb_clone,
 320                      mlx5_list_clone_free_cb cb_clone_free,
 321                      struct rte_flow_error *error)
 322 {
 323         struct mlx5_hlist *hl;
 324         struct mlx5_hlist *expected = NULL;
 325         char s[MLX5_NAME_SIZE];
 326
 327         hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
 328         if (likely(hl))
 329                 return hl;
 330         snprintf(s, sizeof(s), "%s_%s", sh->ibdev_name, name);
 331         hl = mlx5_hlist_create(s, size, direct_key, lcores_share,
 332                         ctx, cb_create, cb_match, cb_remove, cb_clone,
 333                         cb_clone_free);
 334         if (!hl) {
 335                 DRV_LOG(ERR, "%s hash creation failed", name);
 336                 rte_flow_error_set(error, ENOMEM,
 337                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 338                                    "cannot allocate resource memory");
 339                 return NULL;
 340         }
 341         if (!__atomic_compare_exchange_n(phl, &expected, hl, false,
 342                                          __ATOMIC_SEQ_CST,
 343                                          __ATOMIC_SEQ_CST)) {
 344                 mlx5_hlist_destroy(hl);
 345                 hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
 346         }
 347         return hl;
 348 }
 349
 350 /* Update VLAN's VID/PCP based on input rte_flow_action.
 351  *
 352  * @param[in] action
 353  *   Pointer to struct rte_flow_action.
 354  * @param[out] vlan
 355  *   Pointer to struct rte_vlan_hdr.
 356  */
 357 static void
 358 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 359                          struct rte_vlan_hdr *vlan)
 360 {
 361         uint16_t vlan_tci;
 362         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 363                 vlan_tci =
 364                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 365                                                action->conf)->vlan_pcp;
 366                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 367                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 368                 vlan->vlan_tci |= vlan_tci;
 369         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 370                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 371                 vlan->vlan_tci |= rte_be_to_cpu_16
 372                     (((const struct rte_flow_action_of_set_vlan_vid *)
 373                                              action->conf)->vlan_vid);
 374         }
 375 }
 376
 377 /**
 378  * Fetch 1, 2, 3 or 4 byte field from the byte array
 379  * and return as unsigned integer in host-endian format.
 380  *
 381  * @param[in] data
 382  *   Pointer to data array.
 383  * @param[in] size
 384  *   Size of field to extract.
 385  *
 386  * @return
 387  *   converted field in host endian format.
 388  */
 389 static inline uint32_t
 390 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 391 {
 392         uint32_t ret;
 393
 394         switch (size) {
 395         case 1:
 396                 ret = *data;
 397                 break;
 398         case 2:
 399                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 400                 break;
 401         case 3:
 402                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 403                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 404                 break;
 405         case 4:
 406                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 407                 break;
 408         default:
 409                 MLX5_ASSERT(false);
 410                 ret = 0;
 411                 break;
 412         }
 413         return ret;
 414 }
 415
 416 /**
 417  * Convert modify-header action to DV specification.
 418  *
 419  * Data length of each action is determined by provided field description
 420  * and the item mask. Data bit offset and width of each action is determined
 421  * by provided item mask.
 422  *
 423  * @param[in] item
 424  *   Pointer to item specification.
 425  * @param[in] field
 426  *   Pointer to field modification information.
 427  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 428  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 429  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 430  * @param[in] dcopy
 431  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 432  *   Negative offset value sets the same offset as source offset.
 433  *   size field is ignored, value is taken from source field.
 434  * @param[in,out] resource
 435  *   Pointer to the modify-header resource.
 436  * @param[in] type
 437  *   Type of modification.
 438  * @param[out] error
 439  *   Pointer to the error structure.
 440  *
 441  * @return
 442  *   0 on success, a negative errno value otherwise and rte_errno is set.
 443  */
 444 static int
 445 flow_dv_convert_modify_action(struct rte_flow_item *item,
 446                               struct field_modify_info *field,
 447                               struct field_modify_info *dcopy,
 448                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 449                               uint32_t type, struct rte_flow_error *error)
 450 {
 451         uint32_t i = resource->actions_num;
 452         struct mlx5_modification_cmd *actions = resource->actions;
 453         uint32_t carry_b = 0;
 454
 455         /*
 456          * The item and mask are provided in big-endian format.
 457          * The fields should be presented as in big-endian format either.
 458          * Mask must be always present, it defines the actual field width.
 459          */
 460         MLX5_ASSERT(item->mask);
 461         MLX5_ASSERT(field->size);
 462         do {
 463                 uint32_t size_b;
 464                 uint32_t off_b;
 465                 uint32_t mask;
 466                 uint32_t data;
 467                 bool next_field = true;
 468                 bool next_dcopy = true;
 469
 470                 if (i >= MLX5_MAX_MODIFY_NUM)
 471                         return rte_flow_error_set(error, EINVAL,
 472                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 473                                  "too many items to modify");
 474                 /* Fetch variable byte size mask from the array. */
 475                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 476                                            field->offset, field->size);
 477                 if (!mask) {
 478                         ++field;
 479                         continue;
 480                 }
 481                 /* Deduce actual data width in bits from mask value. */
 482                 off_b = rte_bsf32(mask) + carry_b;
 483                 size_b = sizeof(uint32_t) * CHAR_BIT -
 484                          off_b - __builtin_clz(mask);
 485                 MLX5_ASSERT(size_b);
 486                 actions[i] = (struct mlx5_modification_cmd) {
 487                         .action_type = type,
 488                         .field = field->id,
 489                         .offset = off_b,
 490                         .length = (size_b == sizeof(uint32_t) * CHAR_BIT) ?
 491                                 0 : size_b,
 492                 };
 493                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 494                         MLX5_ASSERT(dcopy);
 495                         actions[i].dst_field = dcopy->id;
 496                         actions[i].dst_offset =
 497                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 498                         /* Convert entire record to big-endian format. */
 499                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 500                         /*
 501                          * Destination field overflow. Copy leftovers of
 502                          * a source field to the next destination field.
 503                          */
 504                         carry_b = 0;
 505                         if ((size_b > dcopy->size * CHAR_BIT - dcopy->offset) &&
 506                             dcopy->size != 0) {
 507                                 actions[i].length =
 508                                         dcopy->size * CHAR_BIT - dcopy->offset;
 509                                 carry_b = actions[i].length;
 510                                 next_field = false;
 511                         }
 512                         /*
 513                          * Not enough bits in a source filed to fill a
 514                          * destination field. Switch to the next source.
 515                          */
 516                         if ((size_b < dcopy->size * CHAR_BIT - dcopy->offset) &&
 517                             (size_b == field->size * CHAR_BIT - off_b)) {
 518                                 actions[i].length =
 519                                         field->size * CHAR_BIT - off_b;
 520                                 dcopy->offset += actions[i].length;
 521                                 next_dcopy = false;
 522                         }
 523                         if (next_dcopy)
 524                                 ++dcopy;
 525                 } else {
 526                         MLX5_ASSERT(item->spec);
 527                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 528                                                    field->offset, field->size);
 529                         /* Shift out the trailing masked bits from data. */
 530                         data = (data & mask) >> off_b;
 531                         actions[i].data1 = rte_cpu_to_be_32(data);
 532                 }
 533                 /* Convert entire record to expected big-endian format. */
 534                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 535                 if (next_field)
 536                         ++field;
 537                 ++i;
 538         } while (field->size);
 539         if (resource->actions_num == i)
 540                 return rte_flow_error_set(error, EINVAL,
 541                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 542                                           "invalid modification flow item");
 543         resource->actions_num = i;
 544         return 0;
 545 }
 546
 547 /**
 548  * Convert modify-header set IPv4 address action to DV specification.
 549  *
 550  * @param[in,out] resource
 551  *   Pointer to the modify-header resource.
 552  * @param[in] action
 553  *   Pointer to action specification.
 554  * @param[out] error
 555  *   Pointer to the error structure.
 556  *
 557  * @return
 558  *   0 on success, a negative errno value otherwise and rte_errno is set.
 559  */
 560 static int
 561 flow_dv_convert_action_modify_ipv4
 562                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 563                          const struct rte_flow_action *action,
 564                          struct rte_flow_error *error)
 565 {
 566         const struct rte_flow_action_set_ipv4 *conf =
 567                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 568         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 569         struct rte_flow_item_ipv4 ipv4;
 570         struct rte_flow_item_ipv4 ipv4_mask;
 571
 572         memset(&ipv4, 0, sizeof(ipv4));
 573         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 574         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 575                 ipv4.hdr.src_addr = conf->ipv4_addr;
 576                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 577         } else {
 578                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 579                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 580         }
 581         item.spec = &ipv4;
 582         item.mask = &ipv4_mask;
 583         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 584                                              MLX5_MODIFICATION_TYPE_SET, error);
 585 }
 586
 587 /**
 588  * Convert modify-header set IPv6 address action to DV specification.
 589  *
 590  * @param[in,out] resource
 591  *   Pointer to the modify-header resource.
 592  * @param[in] action
 593  *   Pointer to action specification.
 594  * @param[out] error
 595  *   Pointer to the error structure.
 596  *
 597  * @return
 598  *   0 on success, a negative errno value otherwise and rte_errno is set.
 599  */
 600 static int
 601 flow_dv_convert_action_modify_ipv6
 602                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 603                          const struct rte_flow_action *action,
 604                          struct rte_flow_error *error)
 605 {
 606         const struct rte_flow_action_set_ipv6 *conf =
 607                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 608         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 609         struct rte_flow_item_ipv6 ipv6;
 610         struct rte_flow_item_ipv6 ipv6_mask;
 611
 612         memset(&ipv6, 0, sizeof(ipv6));
 613         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 614         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 615                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 616                        sizeof(ipv6.hdr.src_addr));
 617                 memcpy(&ipv6_mask.hdr.src_addr,
 618                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 619                        sizeof(ipv6.hdr.src_addr));
 620         } else {
 621                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 622                        sizeof(ipv6.hdr.dst_addr));
 623                 memcpy(&ipv6_mask.hdr.dst_addr,
 624                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 625                        sizeof(ipv6.hdr.dst_addr));
 626         }
 627         item.spec = &ipv6;
 628         item.mask = &ipv6_mask;
 629         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 630                                              MLX5_MODIFICATION_TYPE_SET, error);
 631 }
 632
 633 /**
 634  * Convert modify-header set MAC address action to DV specification.
 635  *
 636  * @param[in,out] resource
 637  *   Pointer to the modify-header resource.
 638  * @param[in] action
 639  *   Pointer to action specification.
 640  * @param[out] error
 641  *   Pointer to the error structure.
 642  *
 643  * @return
 644  *   0 on success, a negative errno value otherwise and rte_errno is set.
 645  */
 646 static int
 647 flow_dv_convert_action_modify_mac
 648                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 649                          const struct rte_flow_action *action,
 650                          struct rte_flow_error *error)
 651 {
 652         const struct rte_flow_action_set_mac *conf =
 653                 (const struct rte_flow_action_set_mac *)(action->conf);
 654         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 655         struct rte_flow_item_eth eth;
 656         struct rte_flow_item_eth eth_mask;
 657
 658         memset(&eth, 0, sizeof(eth));
 659         memset(&eth_mask, 0, sizeof(eth_mask));
 660         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 661                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 662                        sizeof(eth.src.addr_bytes));
 663                 memcpy(&eth_mask.src.addr_bytes,
 664                        &rte_flow_item_eth_mask.src.addr_bytes,
 665                        sizeof(eth_mask.src.addr_bytes));
 666         } else {
 667                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 668                        sizeof(eth.dst.addr_bytes));
 669                 memcpy(&eth_mask.dst.addr_bytes,
 670                        &rte_flow_item_eth_mask.dst.addr_bytes,
 671                        sizeof(eth_mask.dst.addr_bytes));
 672         }
 673         item.spec = &eth;
 674         item.mask = &eth_mask;
 675         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 676                                              MLX5_MODIFICATION_TYPE_SET, error);
 677 }
 678
 679 /**
 680  * Convert modify-header set VLAN VID action to DV specification.
 681  *
 682  * @param[in,out] resource
 683  *   Pointer to the modify-header resource.
 684  * @param[in] action
 685  *   Pointer to action specification.
 686  * @param[out] error
 687  *   Pointer to the error structure.
 688  *
 689  * @return
 690  *   0 on success, a negative errno value otherwise and rte_errno is set.
 691  */
 692 static int
 693 flow_dv_convert_action_modify_vlan_vid
 694                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 695                          const struct rte_flow_action *action,
 696                          struct rte_flow_error *error)
 697 {
 698         const struct rte_flow_action_of_set_vlan_vid *conf =
 699                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 700         int i = resource->actions_num;
 701         struct mlx5_modification_cmd *actions = resource->actions;
 702         struct field_modify_info *field = modify_vlan_out_first_vid;
 703
 704         if (i >= MLX5_MAX_MODIFY_NUM)
 705                 return rte_flow_error_set(error, EINVAL,
 706                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 707                          "too many items to modify");
 708         actions[i] = (struct mlx5_modification_cmd) {
 709                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 710                 .field = field->id,
 711                 .length = field->size,
 712                 .offset = field->offset,
 713         };
 714         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 715         actions[i].data1 = conf->vlan_vid;
 716         actions[i].data1 = actions[i].data1 << 16;
 717         resource->actions_num = ++i;
 718         return 0;
 719 }
 720
 721 /**
 722  * Convert modify-header set TP action to DV specification.
 723  *
 724  * @param[in,out] resource
 725  *   Pointer to the modify-header resource.
 726  * @param[in] action
 727  *   Pointer to action specification.
 728  * @param[in] items
 729  *   Pointer to rte_flow_item objects list.
 730  * @param[in] attr
 731  *   Pointer to flow attributes structure.
 732  * @param[in] dev_flow
 733  *   Pointer to the sub flow.
 734  * @param[in] tunnel_decap
 735  *   Whether action is after tunnel decapsulation.
 736  * @param[out] error
 737  *   Pointer to the error structure.
 738  *
 739  * @return
 740  *   0 on success, a negative errno value otherwise and rte_errno is set.
 741  */
 742 static int
 743 flow_dv_convert_action_modify_tp
 744                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 745                          const struct rte_flow_action *action,
 746                          const struct rte_flow_item *items,
 747                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 748                          bool tunnel_decap, struct rte_flow_error *error)
 749 {
 750         const struct rte_flow_action_set_tp *conf =
 751                 (const struct rte_flow_action_set_tp *)(action->conf);
 752         struct rte_flow_item item;
 753         struct rte_flow_item_udp udp;
 754         struct rte_flow_item_udp udp_mask;
 755         struct rte_flow_item_tcp tcp;
 756         struct rte_flow_item_tcp tcp_mask;
 757         struct field_modify_info *field;
 758
 759         if (!attr->valid)
 760                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 761         if (attr->udp) {
 762                 memset(&udp, 0, sizeof(udp));
 763                 memset(&udp_mask, 0, sizeof(udp_mask));
 764                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 765                         udp.hdr.src_port = conf->port;
 766                         udp_mask.hdr.src_port =
 767                                         rte_flow_item_udp_mask.hdr.src_port;
 768                 } else {
 769                         udp.hdr.dst_port = conf->port;
 770                         udp_mask.hdr.dst_port =
 771                                         rte_flow_item_udp_mask.hdr.dst_port;
 772                 }
 773                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 774                 item.spec = &udp;
 775                 item.mask = &udp_mask;
 776                 field = modify_udp;
 777         } else {
 778                 MLX5_ASSERT(attr->tcp);
 779                 memset(&tcp, 0, sizeof(tcp));
 780                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 781                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 782                         tcp.hdr.src_port = conf->port;
 783                         tcp_mask.hdr.src_port =
 784                                         rte_flow_item_tcp_mask.hdr.src_port;
 785                 } else {
 786                         tcp.hdr.dst_port = conf->port;
 787                         tcp_mask.hdr.dst_port =
 788                                         rte_flow_item_tcp_mask.hdr.dst_port;
 789                 }
 790                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 791                 item.spec = &tcp;
 792                 item.mask = &tcp_mask;
 793                 field = modify_tcp;
 794         }
 795         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 796                                              MLX5_MODIFICATION_TYPE_SET, error);
 797 }
 798
 799 /**
 800  * Convert modify-header set TTL action to DV specification.
 801  *
 802  * @param[in,out] resource
 803  *   Pointer to the modify-header resource.
 804  * @param[in] action
 805  *   Pointer to action specification.
 806  * @param[in] items
 807  *   Pointer to rte_flow_item objects list.
 808  * @param[in] attr
 809  *   Pointer to flow attributes structure.
 810  * @param[in] dev_flow
 811  *   Pointer to the sub flow.
 812  * @param[in] tunnel_decap
 813  *   Whether action is after tunnel decapsulation.
 814  * @param[out] error
 815  *   Pointer to the error structure.
 816  *
 817  * @return
 818  *   0 on success, a negative errno value otherwise and rte_errno is set.
 819  */
 820 static int
 821 flow_dv_convert_action_modify_ttl
 822                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 823                          const struct rte_flow_action *action,
 824                          const struct rte_flow_item *items,
 825                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 826                          bool tunnel_decap, struct rte_flow_error *error)
 827 {
 828         const struct rte_flow_action_set_ttl *conf =
 829                 (const struct rte_flow_action_set_ttl *)(action->conf);
 830         struct rte_flow_item item;
 831         struct rte_flow_item_ipv4 ipv4;
 832         struct rte_flow_item_ipv4 ipv4_mask;
 833         struct rte_flow_item_ipv6 ipv6;
 834         struct rte_flow_item_ipv6 ipv6_mask;
 835         struct field_modify_info *field;
 836
 837         if (!attr->valid)
 838                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 839         if (attr->ipv4) {
 840                 memset(&ipv4, 0, sizeof(ipv4));
 841                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 842                 ipv4.hdr.time_to_live = conf->ttl_value;
 843                 ipv4_mask.hdr.time_to_live = 0xFF;
 844                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 845                 item.spec = &ipv4;
 846                 item.mask = &ipv4_mask;
 847                 field = modify_ipv4;
 848         } else {
 849                 MLX5_ASSERT(attr->ipv6);
 850                 memset(&ipv6, 0, sizeof(ipv6));
 851                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 852                 ipv6.hdr.hop_limits = conf->ttl_value;
 853                 ipv6_mask.hdr.hop_limits = 0xFF;
 854                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 855                 item.spec = &ipv6;
 856                 item.mask = &ipv6_mask;
 857                 field = modify_ipv6;
 858         }
 859         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 860                                              MLX5_MODIFICATION_TYPE_SET, error);
 861 }
 862
 863 /**
 864  * Convert modify-header decrement TTL action to DV specification.
 865  *
 866  * @param[in,out] resource
 867  *   Pointer to the modify-header resource.
 868  * @param[in] action
 869  *   Pointer to action specification.
 870  * @param[in] items
 871  *   Pointer to rte_flow_item objects list.
 872  * @param[in] attr
 873  *   Pointer to flow attributes structure.
 874  * @param[in] dev_flow
 875  *   Pointer to the sub flow.
 876  * @param[in] tunnel_decap
 877  *   Whether action is after tunnel decapsulation.
 878  * @param[out] error
 879  *   Pointer to the error structure.
 880  *
 881  * @return
 882  *   0 on success, a negative errno value otherwise and rte_errno is set.
 883  */
 884 static int
 885 flow_dv_convert_action_modify_dec_ttl
 886                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 887                          const struct rte_flow_item *items,
 888                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 889                          bool tunnel_decap, struct rte_flow_error *error)
 890 {
 891         struct rte_flow_item item;
 892         struct rte_flow_item_ipv4 ipv4;
 893         struct rte_flow_item_ipv4 ipv4_mask;
 894         struct rte_flow_item_ipv6 ipv6;
 895         struct rte_flow_item_ipv6 ipv6_mask;
 896         struct field_modify_info *field;
 897
 898         if (!attr->valid)
 899                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 900         if (attr->ipv4) {
 901                 memset(&ipv4, 0, sizeof(ipv4));
 902                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 903                 ipv4.hdr.time_to_live = 0xFF;
 904                 ipv4_mask.hdr.time_to_live = 0xFF;
 905                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 906                 item.spec = &ipv4;
 907                 item.mask = &ipv4_mask;
 908                 field = modify_ipv4;
 909         } else {
 910                 MLX5_ASSERT(attr->ipv6);
 911                 memset(&ipv6, 0, sizeof(ipv6));
 912                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 913                 ipv6.hdr.hop_limits = 0xFF;
 914                 ipv6_mask.hdr.hop_limits = 0xFF;
 915                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 916                 item.spec = &ipv6;
 917                 item.mask = &ipv6_mask;
 918                 field = modify_ipv6;
 919         }
 920         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 921                                              MLX5_MODIFICATION_TYPE_ADD, error);
 922 }
 923
 924 /**
 925  * Convert modify-header increment/decrement TCP Sequence number
 926  * to DV specification.
 927  *
 928  * @param[in,out] resource
 929  *   Pointer to the modify-header resource.
 930  * @param[in] action
 931  *   Pointer to action specification.
 932  * @param[out] error
 933  *   Pointer to the error structure.
 934  *
 935  * @return
 936  *   0 on success, a negative errno value otherwise and rte_errno is set.
 937  */
 938 static int
 939 flow_dv_convert_action_modify_tcp_seq
 940                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 941                          const struct rte_flow_action *action,
 942                          struct rte_flow_error *error)
 943 {
 944         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 945         uint64_t value = rte_be_to_cpu_32(*conf);
 946         struct rte_flow_item item;
 947         struct rte_flow_item_tcp tcp;
 948         struct rte_flow_item_tcp tcp_mask;
 949
 950         memset(&tcp, 0, sizeof(tcp));
 951         memset(&tcp_mask, 0, sizeof(tcp_mask));
 952         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 953                 /*
 954                  * The HW has no decrement operation, only increment operation.
 955                  * To simulate decrement X from Y using increment operation
 956                  * we need to add UINT32_MAX X times to Y.
 957                  * Each adding of UINT32_MAX decrements Y by 1.
 958                  */
 959                 value *= UINT32_MAX;
 960         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 961         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 962         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 963         item.spec = &tcp;
 964         item.mask = &tcp_mask;
 965         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 966                                              MLX5_MODIFICATION_TYPE_ADD, error);
 967 }
 968
 969 /**
 970  * Convert modify-header increment/decrement TCP Acknowledgment number
 971  * to DV specification.
 972  *
 973  * @param[in,out] resource
 974  *   Pointer to the modify-header resource.
 975  * @param[in] action
 976  *   Pointer to action specification.
 977  * @param[out] error
 978  *   Pointer to the error structure.
 979  *
 980  * @return
 981  *   0 on success, a negative errno value otherwise and rte_errno is set.
 982  */
 983 static int
 984 flow_dv_convert_action_modify_tcp_ack
 985                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 986                          const struct rte_flow_action *action,
 987                          struct rte_flow_error *error)
 988 {
 989         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 990         uint64_t value = rte_be_to_cpu_32(*conf);
 991         struct rte_flow_item item;
 992         struct rte_flow_item_tcp tcp;
 993         struct rte_flow_item_tcp tcp_mask;
 994
 995         memset(&tcp, 0, sizeof(tcp));
 996         memset(&tcp_mask, 0, sizeof(tcp_mask));
 997         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 998                 /*
 999                  * The HW has no decrement operation, only increment operation.
1000                  * To simulate decrement X from Y using increment operation
1001                  * we need to add UINT32_MAX X times to Y.
1002                  * Each adding of UINT32_MAX decrements Y by 1.
1003                  */
1004                 value *= UINT32_MAX;
1005         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
1006         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
1007         item.type = RTE_FLOW_ITEM_TYPE_TCP;
1008         item.spec = &tcp;
1009         item.mask = &tcp_mask;
1010         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
1011                                              MLX5_MODIFICATION_TYPE_ADD, error);
1012 }
1013
1014 static enum mlx5_modification_field reg_to_field[] = {
1015         [REG_NON] = MLX5_MODI_OUT_NONE,
1016         [REG_A] = MLX5_MODI_META_DATA_REG_A,
1017         [REG_B] = MLX5_MODI_META_DATA_REG_B,
1018         [REG_C_0] = MLX5_MODI_META_REG_C_0,
1019         [REG_C_1] = MLX5_MODI_META_REG_C_1,
1020         [REG_C_2] = MLX5_MODI_META_REG_C_2,
1021         [REG_C_3] = MLX5_MODI_META_REG_C_3,
1022         [REG_C_4] = MLX5_MODI_META_REG_C_4,
1023         [REG_C_5] = MLX5_MODI_META_REG_C_5,
1024         [REG_C_6] = MLX5_MODI_META_REG_C_6,
1025         [REG_C_7] = MLX5_MODI_META_REG_C_7,
1026 };
1027
1028 /**
1029  * Convert register set to DV specification.
1030  *
1031  * @param[in,out] resource
1032  *   Pointer to the modify-header resource.
1033  * @param[in] action
1034  *   Pointer to action specification.
1035  * @param[out] error
1036  *   Pointer to the error structure.
1037  *
1038  * @return
1039  *   0 on success, a negative errno value otherwise and rte_errno is set.
1040  */
1041 static int
1042 flow_dv_convert_action_set_reg
1043                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1044                          const struct rte_flow_action *action,
1045                          struct rte_flow_error *error)
1046 {
1047         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
1048         struct mlx5_modification_cmd *actions = resource->actions;
1049         uint32_t i = resource->actions_num;
1050
1051         if (i >= MLX5_MAX_MODIFY_NUM)
1052                 return rte_flow_error_set(error, EINVAL,
1053                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1054                                           "too many items to modify");
1055         MLX5_ASSERT(conf->id != REG_NON);
1056         MLX5_ASSERT(conf->id < (enum modify_reg)RTE_DIM(reg_to_field));
1057         actions[i] = (struct mlx5_modification_cmd) {
1058                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1059                 .field = reg_to_field[conf->id],
1060                 .offset = conf->offset,
1061                 .length = conf->length,
1062         };
1063         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1064         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1065         ++i;
1066         resource->actions_num = i;
1067         return 0;
1068 }
1069
1070 /**
1071  * Convert SET_TAG action to DV specification.
1072  *
1073  * @param[in] dev
1074  *   Pointer to the rte_eth_dev structure.
1075  * @param[in,out] resource
1076  *   Pointer to the modify-header resource.
1077  * @param[in] conf
1078  *   Pointer to action specification.
1079  * @param[out] error
1080  *   Pointer to the error structure.
1081  *
1082  * @return
1083  *   0 on success, a negative errno value otherwise and rte_errno is set.
1084  */
1085 static int
1086 flow_dv_convert_action_set_tag
1087                         (struct rte_eth_dev *dev,
1088                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1089                          const struct rte_flow_action_set_tag *conf,
1090                          struct rte_flow_error *error)
1091 {
1092         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1093         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1094         struct rte_flow_item item = {
1095                 .spec = &data,
1096                 .mask = &mask,
1097         };
1098         struct field_modify_info reg_c_x[] = {
1099                 [1] = {0, 0, 0},
1100         };
1101         enum mlx5_modification_field reg_type;
1102         int ret;
1103
1104         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1105         if (ret < 0)
1106                 return ret;
1107         MLX5_ASSERT(ret != REG_NON);
1108         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1109         reg_type = reg_to_field[ret];
1110         MLX5_ASSERT(reg_type > 0);
1111         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1112         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1113                                              MLX5_MODIFICATION_TYPE_SET, error);
1114 }
1115
1116 /**
1117  * Convert internal COPY_REG action to DV specification.
1118  *
1119  * @param[in] dev
1120  *   Pointer to the rte_eth_dev structure.
1121  * @param[in,out] res
1122  *   Pointer to the modify-header resource.
1123  * @param[in] action
1124  *   Pointer to action specification.
1125  * @param[out] error
1126  *   Pointer to the error structure.
1127  *
1128  * @return
1129  *   0 on success, a negative errno value otherwise and rte_errno is set.
1130  */
1131 static int
1132 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1133                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1134                                  const struct rte_flow_action *action,
1135                                  struct rte_flow_error *error)
1136 {
1137         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1138         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1139         struct rte_flow_item item = {
1140                 .spec = NULL,
1141                 .mask = &mask,
1142         };
1143         struct field_modify_info reg_src[] = {
1144                 {4, 0, reg_to_field[conf->src]},
1145                 {0, 0, 0},
1146         };
1147         struct field_modify_info reg_dst = {
1148                 .offset = 0,
1149                 .id = reg_to_field[conf->dst],
1150         };
1151         /* Adjust reg_c[0] usage according to reported mask. */
1152         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1153                 struct mlx5_priv *priv = dev->data->dev_private;
1154                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1155
1156                 MLX5_ASSERT(reg_c0);
1157                 MLX5_ASSERT(priv->sh->config.dv_xmeta_en !=
1158                             MLX5_XMETA_MODE_LEGACY);
1159                 if (conf->dst == REG_C_0) {
1160                         /* Copy to reg_c[0], within mask only. */
1161                         reg_dst.offset = rte_bsf32(reg_c0);
1162                         mask = rte_cpu_to_be_32(reg_c0 >> reg_dst.offset);
1163                 } else {
1164                         reg_dst.offset = 0;
1165                         mask = rte_cpu_to_be_32(reg_c0);
1166                 }
1167         }
1168         return flow_dv_convert_modify_action(&item,
1169                                              reg_src, &reg_dst, res,
1170                                              MLX5_MODIFICATION_TYPE_COPY,
1171                                              error);
1172 }
1173
1174 /**
1175  * Convert MARK action to DV specification. This routine is used
1176  * in extensive metadata only and requires metadata register to be
1177  * handled. In legacy mode hardware tag resource is engaged.
1178  *
1179  * @param[in] dev
1180  *   Pointer to the rte_eth_dev structure.
1181  * @param[in] conf
1182  *   Pointer to MARK action specification.
1183  * @param[in,out] resource
1184  *   Pointer to the modify-header resource.
1185  * @param[out] error
1186  *   Pointer to the error structure.
1187  *
1188  * @return
1189  *   0 on success, a negative errno value otherwise and rte_errno is set.
1190  */
1191 static int
1192 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1193                             const struct rte_flow_action_mark *conf,
1194                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1195                             struct rte_flow_error *error)
1196 {
1197         struct mlx5_priv *priv = dev->data->dev_private;
1198         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1199                                            priv->sh->dv_mark_mask);
1200         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1201         struct rte_flow_item item = {
1202                 .spec = &data,
1203                 .mask = &mask,
1204         };
1205         struct field_modify_info reg_c_x[] = {
1206                 [1] = {0, 0, 0},
1207         };
1208         int reg;
1209
1210         if (!mask)
1211                 return rte_flow_error_set(error, EINVAL,
1212                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1213                                           NULL, "zero mark action mask");
1214         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1215         if (reg < 0)
1216                 return reg;
1217         MLX5_ASSERT(reg > 0);
1218         if (reg == REG_C_0) {
1219                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1220                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1221
1222                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1223                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1224                 mask = rte_cpu_to_be_32(mask << shl_c0);
1225         }
1226         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1227         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1228                                              MLX5_MODIFICATION_TYPE_SET, error);
1229 }
1230
1231 /**
1232  * Get metadata register index for specified steering domain.
1233  *
1234  * @param[in] dev
1235  *   Pointer to the rte_eth_dev structure.
1236  * @param[in] attr
1237  *   Attributes of flow to determine steering domain.
1238  * @param[out] error
1239  *   Pointer to the error structure.
1240  *
1241  * @return
1242  *   positive index on success, a negative errno value otherwise
1243  *   and rte_errno is set.
1244  */
1245 static enum modify_reg
1246 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1247                          const struct rte_flow_attr *attr,
1248                          struct rte_flow_error *error)
1249 {
1250         int reg =
1251                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1252                                           MLX5_METADATA_FDB :
1253                                             attr->egress ?
1254                                             MLX5_METADATA_TX :
1255                                             MLX5_METADATA_RX, 0, error);
1256         if (reg < 0)
1257                 return rte_flow_error_set(error,
1258                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1259                                           NULL, "unavailable "
1260                                           "metadata register");
1261         return reg;
1262 }
1263
1264 /**
1265  * Convert SET_META action to DV specification.
1266  *
1267  * @param[in] dev
1268  *   Pointer to the rte_eth_dev structure.
1269  * @param[in,out] resource
1270  *   Pointer to the modify-header resource.
1271  * @param[in] attr
1272  *   Attributes of flow that includes this item.
1273  * @param[in] conf
1274  *   Pointer to action specification.
1275  * @param[out] error
1276  *   Pointer to the error structure.
1277  *
1278  * @return
1279  *   0 on success, a negative errno value otherwise and rte_errno is set.
1280  */
1281 static int
1282 flow_dv_convert_action_set_meta
1283                         (struct rte_eth_dev *dev,
1284                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1285                          const struct rte_flow_attr *attr,
1286                          const struct rte_flow_action_set_meta *conf,
1287                          struct rte_flow_error *error)
1288 {
1289         uint32_t mask = rte_cpu_to_be_32(conf->mask);
1290         uint32_t data = rte_cpu_to_be_32(conf->data) & mask;
1291         struct rte_flow_item item = {
1292                 .spec = &data,
1293                 .mask = &mask,
1294         };
1295         struct field_modify_info reg_c_x[] = {
1296                 [1] = {0, 0, 0},
1297         };
1298         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1299
1300         if (reg < 0)
1301                 return reg;
1302         MLX5_ASSERT(reg != REG_NON);
1303         if (reg == REG_C_0) {
1304                 struct mlx5_priv *priv = dev->data->dev_private;
1305                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1306                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1307
1308                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1309                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1310                 mask = rte_cpu_to_be_32(mask << shl_c0);
1311         }
1312         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1313         /* The routine expects parameters in memory as big-endian ones. */
1314         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1315                                              MLX5_MODIFICATION_TYPE_SET, error);
1316 }
1317
1318 /**
1319  * Convert modify-header set IPv4 DSCP action to DV specification.
1320  *
1321  * @param[in,out] resource
1322  *   Pointer to the modify-header resource.
1323  * @param[in] action
1324  *   Pointer to action specification.
1325  * @param[out] error
1326  *   Pointer to the error structure.
1327  *
1328  * @return
1329  *   0 on success, a negative errno value otherwise and rte_errno is set.
1330  */
1331 static int
1332 flow_dv_convert_action_modify_ipv4_dscp
1333                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1334                          const struct rte_flow_action *action,
1335                          struct rte_flow_error *error)
1336 {
1337         const struct rte_flow_action_set_dscp *conf =
1338                 (const struct rte_flow_action_set_dscp *)(action->conf);
1339         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1340         struct rte_flow_item_ipv4 ipv4;
1341         struct rte_flow_item_ipv4 ipv4_mask;
1342
1343         memset(&ipv4, 0, sizeof(ipv4));
1344         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1345         ipv4.hdr.type_of_service = conf->dscp;
1346         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1347         item.spec = &ipv4;
1348         item.mask = &ipv4_mask;
1349         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1350                                              MLX5_MODIFICATION_TYPE_SET, error);
1351 }
1352
1353 /**
1354  * Convert modify-header set IPv6 DSCP action to DV specification.
1355  *
1356  * @param[in,out] resource
1357  *   Pointer to the modify-header resource.
1358  * @param[in] action
1359  *   Pointer to action specification.
1360  * @param[out] error
1361  *   Pointer to the error structure.
1362  *
1363  * @return
1364  *   0 on success, a negative errno value otherwise and rte_errno is set.
1365  */
1366 static int
1367 flow_dv_convert_action_modify_ipv6_dscp
1368                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1369                          const struct rte_flow_action *action,
1370                          struct rte_flow_error *error)
1371 {
1372         const struct rte_flow_action_set_dscp *conf =
1373                 (const struct rte_flow_action_set_dscp *)(action->conf);
1374         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1375         struct rte_flow_item_ipv6 ipv6;
1376         struct rte_flow_item_ipv6 ipv6_mask;
1377
1378         memset(&ipv6, 0, sizeof(ipv6));
1379         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1380         /*
1381          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1382          * rdma-core only accept the DSCP bits byte aligned start from
1383          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1384          * bits in IPv6 case as rdma-core requires byte aligned value.
1385          */
1386         ipv6.hdr.vtc_flow = conf->dscp;
1387         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1388         item.spec = &ipv6;
1389         item.mask = &ipv6_mask;
1390         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1391                                              MLX5_MODIFICATION_TYPE_SET, error);
1392 }
1393
1394 static int
1395 mlx5_flow_item_field_width(struct rte_eth_dev *dev,
1396                            enum rte_flow_field_id field, int inherit,
1397                            const struct rte_flow_attr *attr,
1398                            struct rte_flow_error *error)
1399 {
1400         struct mlx5_priv *priv = dev->data->dev_private;
1401
1402         switch (field) {
1403         case RTE_FLOW_FIELD_START:
1404                 return 32;
1405         case RTE_FLOW_FIELD_MAC_DST:
1406         case RTE_FLOW_FIELD_MAC_SRC:
1407                 return 48;
1408         case RTE_FLOW_FIELD_VLAN_TYPE:
1409                 return 16;
1410         case RTE_FLOW_FIELD_VLAN_ID:
1411                 return 12;
1412         case RTE_FLOW_FIELD_MAC_TYPE:
1413                 return 16;
1414         case RTE_FLOW_FIELD_IPV4_DSCP:
1415                 return 6;
1416         case RTE_FLOW_FIELD_IPV4_TTL:
1417                 return 8;
1418         case RTE_FLOW_FIELD_IPV4_SRC:
1419         case RTE_FLOW_FIELD_IPV4_DST:
1420                 return 32;
1421         case RTE_FLOW_FIELD_IPV6_DSCP:
1422                 return 6;
1423         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1424                 return 8;
1425         case RTE_FLOW_FIELD_IPV6_SRC:
1426         case RTE_FLOW_FIELD_IPV6_DST:
1427                 return 128;
1428         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1429         case RTE_FLOW_FIELD_TCP_PORT_DST:
1430                 return 16;
1431         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1432         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1433                 return 32;
1434         case RTE_FLOW_FIELD_TCP_FLAGS:
1435                 return 9;
1436         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1437         case RTE_FLOW_FIELD_UDP_PORT_DST:
1438                 return 16;
1439         case RTE_FLOW_FIELD_VXLAN_VNI:
1440         case RTE_FLOW_FIELD_GENEVE_VNI:
1441                 return 24;
1442         case RTE_FLOW_FIELD_GTP_TEID:
1443         case RTE_FLOW_FIELD_TAG:
1444                 return 32;
1445         case RTE_FLOW_FIELD_MARK:
1446                 return __builtin_popcount(priv->sh->dv_mark_mask);
1447         case RTE_FLOW_FIELD_META:
1448                 return (flow_dv_get_metadata_reg(dev, attr, error) == REG_C_0) ?
1449                         __builtin_popcount(priv->sh->dv_meta_mask) : 32;
1450         case RTE_FLOW_FIELD_POINTER:
1451         case RTE_FLOW_FIELD_VALUE:
1452                 return inherit < 0 ? 0 : inherit;
1453         default:
1454                 MLX5_ASSERT(false);
1455         }
1456         return 0;
1457 }
1458
1459 static void
1460 mlx5_flow_field_id_to_modify_info
1461                 (const struct rte_flow_action_modify_data *data,
1462                  struct field_modify_info *info, uint32_t *mask,
1463                  uint32_t width, struct rte_eth_dev *dev,
1464                  const struct rte_flow_attr *attr, struct rte_flow_error *error)
1465 {
1466         struct mlx5_priv *priv = dev->data->dev_private;
1467         uint32_t idx = 0;
1468         uint32_t off = 0;
1469
1470         switch (data->field) {
1471         case RTE_FLOW_FIELD_START:
1472                 /* not supported yet */
1473                 MLX5_ASSERT(false);
1474                 break;
1475         case RTE_FLOW_FIELD_MAC_DST:
1476                 off = data->offset > 16 ? data->offset - 16 : 0;
1477                 if (mask) {
1478                         if (data->offset < 16) {
1479                                 info[idx] = (struct field_modify_info){2, 4,
1480                                                 MLX5_MODI_OUT_DMAC_15_0};
1481                                 if (width < 16) {
1482                                         mask[1] = rte_cpu_to_be_16(0xffff >>
1483                                                                  (16 - width));
1484                                         width = 0;
1485                                 } else {
1486                                         mask[1] = RTE_BE16(0xffff);
1487                                         width -= 16;
1488                                 }
1489                                 if (!width)
1490                                         break;
1491                                 ++idx;
1492                         }
1493                         info[idx] = (struct field_modify_info){4, 0,
1494                                                 MLX5_MODI_OUT_DMAC_47_16};
1495                         mask[0] = rte_cpu_to_be_32((0xffffffff >>
1496                                                     (32 - width)) << off);
1497                 } else {
1498                         if (data->offset < 16)
1499                                 info[idx++] = (struct field_modify_info){2, 0,
1500                                                 MLX5_MODI_OUT_DMAC_15_0};
1501                         info[idx] = (struct field_modify_info){4, off,
1502                                                 MLX5_MODI_OUT_DMAC_47_16};
1503                 }
1504                 break;
1505         case RTE_FLOW_FIELD_MAC_SRC:
1506                 off = data->offset > 16 ? data->offset - 16 : 0;
1507                 if (mask) {
1508                         if (data->offset < 16) {
1509                                 info[idx] = (struct field_modify_info){2, 4,
1510                                                 MLX5_MODI_OUT_SMAC_15_0};
1511                                 if (width < 16) {
1512                                         mask[1] = rte_cpu_to_be_16(0xffff >>
1513                                                                  (16 - width));
1514                                         width = 0;
1515                                 } else {
1516                                         mask[1] = RTE_BE16(0xffff);
1517                                         width -= 16;
1518                                 }
1519                                 if (!width)
1520                                         break;
1521                                 ++idx;
1522                         }
1523                         info[idx] = (struct field_modify_info){4, 0,
1524                                                 MLX5_MODI_OUT_SMAC_47_16};
1525                         mask[0] = rte_cpu_to_be_32((0xffffffff >>
1526                                                     (32 - width)) << off);
1527                 } else {
1528                         if (data->offset < 16)
1529                                 info[idx++] = (struct field_modify_info){2, 0,
1530                                                 MLX5_MODI_OUT_SMAC_15_0};
1531                         info[idx] = (struct field_modify_info){4, off,
1532                                                 MLX5_MODI_OUT_SMAC_47_16};
1533                 }
1534                 break;
1535         case RTE_FLOW_FIELD_VLAN_TYPE:
1536                 /* not supported yet */
1537                 break;
1538         case RTE_FLOW_FIELD_VLAN_ID:
1539                 info[idx] = (struct field_modify_info){2, 0,
1540                                         MLX5_MODI_OUT_FIRST_VID};
1541                 if (mask)
1542                         mask[idx] = rte_cpu_to_be_16(0x0fff >> (12 - width));
1543                 break;
1544         case RTE_FLOW_FIELD_MAC_TYPE:
1545                 info[idx] = (struct field_modify_info){2, 0,
1546                                         MLX5_MODI_OUT_ETHERTYPE};
1547                 if (mask)
1548                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1549                 break;
1550         case RTE_FLOW_FIELD_IPV4_DSCP:
1551                 info[idx] = (struct field_modify_info){1, 0,
1552                                         MLX5_MODI_OUT_IP_DSCP};
1553                 if (mask)
1554                         mask[idx] = 0x3f >> (6 - width);
1555                 break;
1556         case RTE_FLOW_FIELD_IPV4_TTL:
1557                 info[idx] = (struct field_modify_info){1, 0,
1558                                         MLX5_MODI_OUT_IPV4_TTL};
1559                 if (mask)
1560                         mask[idx] = 0xff >> (8 - width);
1561                 break;
1562         case RTE_FLOW_FIELD_IPV4_SRC:
1563                 info[idx] = (struct field_modify_info){4, 0,
1564                                         MLX5_MODI_OUT_SIPV4};
1565                 if (mask)
1566                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1567                                                      (32 - width));
1568                 break;
1569         case RTE_FLOW_FIELD_IPV4_DST:
1570                 info[idx] = (struct field_modify_info){4, 0,
1571                                         MLX5_MODI_OUT_DIPV4};
1572                 if (mask)
1573                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1574                                                      (32 - width));
1575                 break;
1576         case RTE_FLOW_FIELD_IPV6_DSCP:
1577                 info[idx] = (struct field_modify_info){1, 0,
1578                                         MLX5_MODI_OUT_IP_DSCP};
1579                 if (mask)
1580                         mask[idx] = 0x3f >> (6 - width);
1581                 break;
1582         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1583                 info[idx] = (struct field_modify_info){1, 0,
1584                                         MLX5_MODI_OUT_IPV6_HOPLIMIT};
1585                 if (mask)
1586                         mask[idx] = 0xff >> (8 - width);
1587                 break;
1588         case RTE_FLOW_FIELD_IPV6_SRC:
1589                 if (mask) {
1590                         if (data->offset < 32) {
1591                                 info[idx] = (struct field_modify_info){4, 12,
1592                                                 MLX5_MODI_OUT_SIPV6_31_0};
1593                                 if (width < 32) {
1594                                         mask[3] =
1595                                                 rte_cpu_to_be_32(0xffffffff >>
1596                                                                  (32 - width));
1597                                         width = 0;
1598                                 } else {
1599                                         mask[3] = RTE_BE32(0xffffffff);
1600                                         width -= 32;
1601                                 }
1602                                 if (!width)
1603                                         break;
1604                                 ++idx;
1605                         }
1606                         if (data->offset < 64) {
1607                                 info[idx] = (struct field_modify_info){4, 8,
1608                                                 MLX5_MODI_OUT_SIPV6_63_32};
1609                                 if (width < 32) {
1610                                         mask[2] =
1611                                                 rte_cpu_to_be_32(0xffffffff >>
1612                                                                  (32 - width));
1613                                         width = 0;
1614                                 } else {
1615                                         mask[2] = RTE_BE32(0xffffffff);
1616                                         width -= 32;
1617                                 }
1618                                 if (!width)
1619                                         break;
1620                                 ++idx;
1621                         }
1622                         if (data->offset < 96) {
1623                                 info[idx] = (struct field_modify_info){4, 4,
1624                                                 MLX5_MODI_OUT_SIPV6_95_64};
1625                                 if (width < 32) {
1626                                         mask[1] =
1627                                                 rte_cpu_to_be_32(0xffffffff >>
1628                                                                  (32 - width));
1629                                         width = 0;
1630                                 } else {
1631                                         mask[1] = RTE_BE32(0xffffffff);
1632                                         width -= 32;
1633                                 }
1634                                 if (!width)
1635                                         break;
1636                                 ++idx;
1637                         }
1638                         info[idx] = (struct field_modify_info){4, 0,
1639                                                 MLX5_MODI_OUT_SIPV6_127_96};
1640                         mask[0] = rte_cpu_to_be_32(0xffffffff >> (32 - width));
1641                 } else {
1642                         if (data->offset < 32)
1643                                 info[idx++] = (struct field_modify_info){4, 0,
1644                                                 MLX5_MODI_OUT_SIPV6_31_0};
1645                         if (data->offset < 64)
1646                                 info[idx++] = (struct field_modify_info){4, 0,
1647                                                 MLX5_MODI_OUT_SIPV6_63_32};
1648                         if (data->offset < 96)
1649                                 info[idx++] = (struct field_modify_info){4, 0,
1650                                                 MLX5_MODI_OUT_SIPV6_95_64};
1651                         if (data->offset < 128)
1652                                 info[idx++] = (struct field_modify_info){4, 0,
1653                                                 MLX5_MODI_OUT_SIPV6_127_96};
1654                 }
1655                 break;
1656         case RTE_FLOW_FIELD_IPV6_DST:
1657                 if (mask) {
1658                         if (data->offset < 32) {
1659                                 info[idx] = (struct field_modify_info){4, 12,
1660                                                 MLX5_MODI_OUT_DIPV6_31_0};
1661                                 if (width < 32) {
1662                                         mask[3] =
1663                                                 rte_cpu_to_be_32(0xffffffff >>
1664                                                                  (32 - width));
1665                                         width = 0;
1666                                 } else {
1667                                         mask[3] = RTE_BE32(0xffffffff);
1668                                         width -= 32;
1669                                 }
1670                                 if (!width)
1671                                         break;
1672                                 ++idx;
1673                         }
1674                         if (data->offset < 64) {
1675                                 info[idx] = (struct field_modify_info){4, 8,
1676                                                 MLX5_MODI_OUT_DIPV6_63_32};
1677                                 if (width < 32) {
1678                                         mask[2] =
1679                                                 rte_cpu_to_be_32(0xffffffff >>
1680                                                                  (32 - width));
1681                                         width = 0;
1682                                 } else {
1683                                         mask[2] = RTE_BE32(0xffffffff);
1684                                         width -= 32;
1685                                 }
1686                                 if (!width)
1687                                         break;
1688                                 ++idx;
1689                         }
1690                         if (data->offset < 96) {
1691                                 info[idx] = (struct field_modify_info){4, 4,
1692                                                 MLX5_MODI_OUT_DIPV6_95_64};
1693                                 if (width < 32) {
1694                                         mask[1] =
1695                                                 rte_cpu_to_be_32(0xffffffff >>
1696                                                                  (32 - width));
1697                                         width = 0;
1698                                 } else {
1699                                         mask[1] = RTE_BE32(0xffffffff);
1700                                         width -= 32;
1701                                 }
1702                                 if (!width)
1703                                         break;
1704                                 ++idx;
1705                         }
1706                         info[idx] = (struct field_modify_info){4, 0,
1707                                                 MLX5_MODI_OUT_DIPV6_127_96};
1708                         mask[0] = rte_cpu_to_be_32(0xffffffff >> (32 - width));
1709                 } else {
1710                         if (data->offset < 32)
1711                                 info[idx++] = (struct field_modify_info){4, 0,
1712                                                 MLX5_MODI_OUT_DIPV6_31_0};
1713                         if (data->offset < 64)
1714                                 info[idx++] = (struct field_modify_info){4, 0,
1715                                                 MLX5_MODI_OUT_DIPV6_63_32};
1716                         if (data->offset < 96)
1717                                 info[idx++] = (struct field_modify_info){4, 0,
1718                                                 MLX5_MODI_OUT_DIPV6_95_64};
1719                         if (data->offset < 128)
1720                                 info[idx++] = (struct field_modify_info){4, 0,
1721                                                 MLX5_MODI_OUT_DIPV6_127_96};
1722                 }
1723                 break;
1724         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1725                 info[idx] = (struct field_modify_info){2, 0,
1726                                         MLX5_MODI_OUT_TCP_SPORT};
1727                 if (mask)
1728                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1729                 break;
1730         case RTE_FLOW_FIELD_TCP_PORT_DST:
1731                 info[idx] = (struct field_modify_info){2, 0,
1732                                         MLX5_MODI_OUT_TCP_DPORT};
1733                 if (mask)
1734                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1735                 break;
1736         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1737                 info[idx] = (struct field_modify_info){4, 0,
1738                                         MLX5_MODI_OUT_TCP_SEQ_NUM};
1739                 if (mask)
1740                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1741                                                      (32 - width));
1742                 break;
1743         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1744                 info[idx] = (struct field_modify_info){4, 0,
1745                                         MLX5_MODI_OUT_TCP_ACK_NUM};
1746                 if (mask)
1747                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1748                                                      (32 - width));
1749                 break;
1750         case RTE_FLOW_FIELD_TCP_FLAGS:
1751                 info[idx] = (struct field_modify_info){2, 0,
1752                                         MLX5_MODI_OUT_TCP_FLAGS};
1753                 if (mask)
1754                         mask[idx] = rte_cpu_to_be_16(0x1ff >> (9 - width));
1755                 break;
1756         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1757                 info[idx] = (struct field_modify_info){2, 0,
1758                                         MLX5_MODI_OUT_UDP_SPORT};
1759                 if (mask)
1760                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1761                 break;
1762         case RTE_FLOW_FIELD_UDP_PORT_DST:
1763                 info[idx] = (struct field_modify_info){2, 0,
1764                                         MLX5_MODI_OUT_UDP_DPORT};
1765                 if (mask)
1766                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1767                 break;
1768         case RTE_FLOW_FIELD_VXLAN_VNI:
1769                 /* not supported yet */
1770                 break;
1771         case RTE_FLOW_FIELD_GENEVE_VNI:
1772                 /* not supported yet*/
1773                 break;
1774         case RTE_FLOW_FIELD_GTP_TEID:
1775                 info[idx] = (struct field_modify_info){4, 0,
1776                                         MLX5_MODI_GTP_TEID};
1777                 if (mask)
1778                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1779                                                      (32 - width));
1780                 break;
1781         case RTE_FLOW_FIELD_TAG:
1782                 {
1783                         int reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG,
1784                                                    data->level, error);
1785                         if (reg < 0)
1786                                 return;
1787                         MLX5_ASSERT(reg != REG_NON);
1788                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1789                         info[idx] = (struct field_modify_info){4, 0,
1790                                                 reg_to_field[reg]};
1791                         if (mask)
1792                                 mask[idx] =
1793                                         rte_cpu_to_be_32(0xffffffff >>
1794                                                          (32 - width));
1795                 }
1796                 break;
1797         case RTE_FLOW_FIELD_MARK:
1798                 {
1799                         uint32_t mark_mask = priv->sh->dv_mark_mask;
1800                         uint32_t mark_count = __builtin_popcount(mark_mask);
1801                         int reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK,
1802                                                        0, error);
1803                         if (reg < 0)
1804                                 return;
1805                         MLX5_ASSERT(reg != REG_NON);
1806                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1807                         info[idx] = (struct field_modify_info){4, 0,
1808                                                 reg_to_field[reg]};
1809                         if (mask)
1810                                 mask[idx] = rte_cpu_to_be_32((mark_mask >>
1811                                          (mark_count - width)) & mark_mask);
1812                 }
1813                 break;
1814         case RTE_FLOW_FIELD_META:
1815                 {
1816                         uint32_t meta_mask = priv->sh->dv_meta_mask;
1817                         uint32_t meta_count = __builtin_popcount(meta_mask);
1818                         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1819                         if (reg < 0)
1820                                 return;
1821                         MLX5_ASSERT(reg != REG_NON);
1822                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1823                         info[idx] = (struct field_modify_info){4, 0,
1824                                                 reg_to_field[reg]};
1825                         if (mask)
1826                                 mask[idx] = rte_cpu_to_be_32((meta_mask >>
1827                                         (meta_count - width)) & meta_mask);
1828                 }
1829                 break;
1830         case RTE_FLOW_FIELD_POINTER:
1831         case RTE_FLOW_FIELD_VALUE:
1832         default:
1833                 MLX5_ASSERT(false);
1834                 break;
1835         }
1836 }
1837
1838 /**
1839  * Convert modify_field action to DV specification.
1840  *
1841  * @param[in] dev
1842  *   Pointer to the rte_eth_dev structure.
1843  * @param[in,out] resource
1844  *   Pointer to the modify-header resource.
1845  * @param[in] action
1846  *   Pointer to action specification.
1847  * @param[in] attr
1848  *   Attributes of flow that includes this item.
1849  * @param[out] error
1850  *   Pointer to the error structure.
1851  *
1852  * @return
1853  *   0 on success, a negative errno value otherwise and rte_errno is set.
1854  */
1855 static int
1856 flow_dv_convert_action_modify_field
1857                         (struct rte_eth_dev *dev,
1858                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1859                          const struct rte_flow_action *action,
1860                          const struct rte_flow_attr *attr,
1861                          struct rte_flow_error *error)
1862 {
1863         const struct rte_flow_action_modify_field *conf =
1864                 (const struct rte_flow_action_modify_field *)(action->conf);
1865         struct rte_flow_item item = {
1866                 .spec = NULL,
1867                 .mask = NULL
1868         };
1869         struct field_modify_info field[MLX5_ACT_MAX_MOD_FIELDS] = {
1870                                                                 {0, 0, 0} };
1871         struct field_modify_info dcopy[MLX5_ACT_MAX_MOD_FIELDS] = {
1872                                                                 {0, 0, 0} };
1873         uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0};
1874         uint32_t type, meta = 0;
1875
1876         if (conf->src.field == RTE_FLOW_FIELD_POINTER ||
1877             conf->src.field == RTE_FLOW_FIELD_VALUE) {
1878                 type = MLX5_MODIFICATION_TYPE_SET;
1879                 /** For SET fill the destination field (field) first. */
1880                 mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask,
1881                                                   conf->width, dev,
1882                                                   attr, error);
1883                 item.spec = conf->src.field == RTE_FLOW_FIELD_POINTER ?
1884                                         (void *)(uintptr_t)conf->src.pvalue :
1885                                         (void *)(uintptr_t)&conf->src.value;
1886                 if (conf->dst.field == RTE_FLOW_FIELD_META) {
1887                         meta = *(const unaligned_uint32_t *)item.spec;
1888                         meta = rte_cpu_to_be_32(meta);
1889                         item.spec = &meta;
1890                 }
1891         } else {
1892                 type = MLX5_MODIFICATION_TYPE_COPY;
1893                 /** For COPY fill the destination field (dcopy) without mask. */
1894                 mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL,
1895                                                   conf->width, dev,
1896                                                   attr, error);
1897                 /** Then construct the source field (field) with mask. */
1898                 mlx5_flow_field_id_to_modify_info(&conf->src, field, mask,
1899                                                   conf->width, dev,
1900                                                   attr, error);
1901         }
1902         item.mask = &mask;
1903         return flow_dv_convert_modify_action(&item,
1904                         field, dcopy, resource, type, error);
1905 }
1906
1907 /**
1908  * Validate MARK item.
1909  *
1910  * @param[in] dev
1911  *   Pointer to the rte_eth_dev structure.
1912  * @param[in] item
1913  *   Item specification.
1914  * @param[in] attr
1915  *   Attributes of flow that includes this item.
1916  * @param[out] error
1917  *   Pointer to error structure.
1918  *
1919  * @return
1920  *   0 on success, a negative errno value otherwise and rte_errno is set.
1921  */
1922 static int
1923 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1924                            const struct rte_flow_item *item,
1925                            const struct rte_flow_attr *attr __rte_unused,
1926                            struct rte_flow_error *error)
1927 {
1928         struct mlx5_priv *priv = dev->data->dev_private;
1929         struct mlx5_sh_config *config = &priv->sh->config;
1930         const struct rte_flow_item_mark *spec = item->spec;
1931         const struct rte_flow_item_mark *mask = item->mask;
1932         const struct rte_flow_item_mark nic_mask = {
1933                 .id = priv->sh->dv_mark_mask,
1934         };
1935         int ret;
1936
1937         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1938                 return rte_flow_error_set(error, ENOTSUP,
1939                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1940                                           "extended metadata feature"
1941                                           " isn't enabled");
1942         if (!mlx5_flow_ext_mreg_supported(dev))
1943                 return rte_flow_error_set(error, ENOTSUP,
1944                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1945                                           "extended metadata register"
1946                                           " isn't supported");
1947         if (!nic_mask.id)
1948                 return rte_flow_error_set(error, ENOTSUP,
1949                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1950                                           "extended metadata register"
1951                                           " isn't available");
1952         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1953         if (ret < 0)
1954                 return ret;
1955         if (!spec)
1956                 return rte_flow_error_set(error, EINVAL,
1957                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1958                                           item->spec,
1959                                           "data cannot be empty");
1960         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1961                 return rte_flow_error_set(error, EINVAL,
1962                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1963                                           &spec->id,
1964                                           "mark id exceeds the limit");
1965         if (!mask)
1966                 mask = &nic_mask;
1967         if (!mask->id)
1968                 return rte_flow_error_set(error, EINVAL,
1969                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1970                                         "mask cannot be zero");
1971
1972         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1973                                         (const uint8_t *)&nic_mask,
1974                                         sizeof(struct rte_flow_item_mark),
1975                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1976         if (ret < 0)
1977                 return ret;
1978         return 0;
1979 }
1980
1981 /**
1982  * Validate META item.
1983  *
1984  * @param[in] dev
1985  *   Pointer to the rte_eth_dev structure.
1986  * @param[in] item
1987  *   Item specification.
1988  * @param[in] attr
1989  *   Attributes of flow that includes this item.
1990  * @param[out] error
1991  *   Pointer to error structure.
1992  *
1993  * @return
1994  *   0 on success, a negative errno value otherwise and rte_errno is set.
1995  */
1996 static int
1997 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1998                            const struct rte_flow_item *item,
1999                            const struct rte_flow_attr *attr,
2000                            struct rte_flow_error *error)
2001 {
2002         struct mlx5_priv *priv = dev->data->dev_private;
2003         struct mlx5_sh_config *config = &priv->sh->config;
2004         const struct rte_flow_item_meta *spec = item->spec;
2005         const struct rte_flow_item_meta *mask = item->mask;
2006         struct rte_flow_item_meta nic_mask = {
2007                 .data = UINT32_MAX
2008         };
2009         int reg;
2010         int ret;
2011
2012         if (!spec)
2013                 return rte_flow_error_set(error, EINVAL,
2014                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2015                                           item->spec,
2016                                           "data cannot be empty");
2017         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
2018                 if (!mlx5_flow_ext_mreg_supported(dev))
2019                         return rte_flow_error_set(error, ENOTSUP,
2020                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2021                                           "extended metadata register"
2022                                           " isn't supported");
2023                 reg = flow_dv_get_metadata_reg(dev, attr, error);
2024                 if (reg < 0)
2025                         return reg;
2026                 if (reg == REG_NON)
2027                         return rte_flow_error_set(error, ENOTSUP,
2028                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2029                                         "unavailable extended metadata register");
2030                 if (reg == REG_B)
2031                         return rte_flow_error_set(error, ENOTSUP,
2032                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2033                                           "match on reg_b "
2034                                           "isn't supported");
2035                 if (reg != REG_A)
2036                         nic_mask.data = priv->sh->dv_meta_mask;
2037         } else {
2038                 if (attr->transfer)
2039                         return rte_flow_error_set(error, ENOTSUP,
2040                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2041                                         "extended metadata feature "
2042                                         "should be enabled when "
2043                                         "meta item is requested "
2044                                         "with e-switch mode ");
2045                 if (attr->ingress)
2046                         return rte_flow_error_set(error, ENOTSUP,
2047                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2048                                         "match on metadata for ingress "
2049                                         "is not supported in legacy "
2050                                         "metadata mode");
2051         }
2052         if (!mask)
2053                 mask = &rte_flow_item_meta_mask;
2054         if (!mask->data)
2055                 return rte_flow_error_set(error, EINVAL,
2056                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2057                                         "mask cannot be zero");
2058
2059         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2060                                         (const uint8_t *)&nic_mask,
2061                                         sizeof(struct rte_flow_item_meta),
2062                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2063         return ret;
2064 }
2065
2066 /**
2067  * Validate TAG item.
2068  *
2069  * @param[in] dev
2070  *   Pointer to the rte_eth_dev structure.
2071  * @param[in] item
2072  *   Item specification.
2073  * @param[in] attr
2074  *   Attributes of flow that includes this item.
2075  * @param[out] error
2076  *   Pointer to error structure.
2077  *
2078  * @return
2079  *   0 on success, a negative errno value otherwise and rte_errno is set.
2080  */
2081 static int
2082 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
2083                           const struct rte_flow_item *item,
2084                           const struct rte_flow_attr *attr __rte_unused,
2085                           struct rte_flow_error *error)
2086 {
2087         const struct rte_flow_item_tag *spec = item->spec;
2088         const struct rte_flow_item_tag *mask = item->mask;
2089         const struct rte_flow_item_tag nic_mask = {
2090                 .data = RTE_BE32(UINT32_MAX),
2091                 .index = 0xff,
2092         };
2093         int ret;
2094
2095         if (!mlx5_flow_ext_mreg_supported(dev))
2096                 return rte_flow_error_set(error, ENOTSUP,
2097                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2098                                           "extensive metadata register"
2099                                           " isn't supported");
2100         if (!spec)
2101                 return rte_flow_error_set(error, EINVAL,
2102                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2103                                           item->spec,
2104                                           "data cannot be empty");
2105         if (!mask)
2106                 mask = &rte_flow_item_tag_mask;
2107         if (!mask->data)
2108                 return rte_flow_error_set(error, EINVAL,
2109                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2110                                         "mask cannot be zero");
2111
2112         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2113                                         (const uint8_t *)&nic_mask,
2114                                         sizeof(struct rte_flow_item_tag),
2115                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2116         if (ret < 0)
2117                 return ret;
2118         if (mask->index != 0xff)
2119                 return rte_flow_error_set(error, EINVAL,
2120                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2121                                           "partial mask for tag index"
2122                                           " is not supported");
2123         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
2124         if (ret < 0)
2125                 return ret;
2126         MLX5_ASSERT(ret != REG_NON);
2127         return 0;
2128 }
2129
2130 /**
2131  * Validate vport item.
2132  *
2133  * @param[in] dev
2134  *   Pointer to the rte_eth_dev structure.
2135  * @param[in] item
2136  *   Item specification.
2137  * @param[in] attr
2138  *   Attributes of flow that includes this item.
2139  * @param[in] item_flags
2140  *   Bit-fields that holds the items detected until now.
2141  * @param[out] error
2142  *   Pointer to error structure.
2143  *
2144  * @return
2145  *   0 on success, a negative errno value otherwise and rte_errno is set.
2146  */
2147 static int
2148 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
2149                               const struct rte_flow_item *item,
2150                               const struct rte_flow_attr *attr,
2151                               uint64_t item_flags,
2152                               struct rte_flow_error *error)
2153 {
2154         const struct rte_flow_item_port_id *spec = item->spec;
2155         const struct rte_flow_item_port_id *mask = item->mask;
2156         const struct rte_flow_item_port_id switch_mask = {
2157                         .id = 0xffffffff,
2158         };
2159         struct mlx5_priv *esw_priv;
2160         struct mlx5_priv *dev_priv;
2161         int ret;
2162
2163         if (!attr->transfer)
2164                 return rte_flow_error_set(error, EINVAL,
2165                                           RTE_FLOW_ERROR_TYPE_ITEM,
2166                                           NULL,
2167                                           "match on port id is valid only"
2168                                           " when transfer flag is enabled");
2169         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
2170                 return rte_flow_error_set(error, ENOTSUP,
2171                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2172                                           "multiple source ports are not"
2173                                           " supported");
2174         if (!mask)
2175                 mask = &switch_mask;
2176         if (mask->id != 0xffffffff)
2177                 return rte_flow_error_set(error, ENOTSUP,
2178                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2179                                            mask,
2180                                            "no support for partial mask on"
2181                                            " \"id\" field");
2182         ret = mlx5_flow_item_acceptable
2183                                 (item, (const uint8_t *)mask,
2184                                  (const uint8_t *)&rte_flow_item_port_id_mask,
2185                                  sizeof(struct rte_flow_item_port_id),
2186                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2187         if (ret)
2188                 return ret;
2189         if (!spec)
2190                 return 0;
2191         if (spec->id == MLX5_PORT_ESW_MGR)
2192                 return 0;
2193         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
2194         if (!esw_priv)
2195                 return rte_flow_error_set(error, rte_errno,
2196                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2197                                           "failed to obtain E-Switch info for"
2198                                           " port");
2199         dev_priv = mlx5_dev_to_eswitch_info(dev);
2200         if (!dev_priv)
2201                 return rte_flow_error_set(error, rte_errno,
2202                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2203                                           NULL,
2204                                           "failed to obtain E-Switch info");
2205         if (esw_priv->domain_id != dev_priv->domain_id)
2206                 return rte_flow_error_set(error, EINVAL,
2207                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2208                                           "cannot match on a port from a"
2209                                           " different E-Switch");
2210         return 0;
2211 }
2212
2213 /**
2214  * Validate VLAN item.
2215  *
2216  * @param[in] item
2217  *   Item specification.
2218  * @param[in] item_flags
2219  *   Bit-fields that holds the items detected until now.
2220  * @param[in] dev
2221  *   Ethernet device flow is being created on.
2222  * @param[out] error
2223  *   Pointer to error structure.
2224  *
2225  * @return
2226  *   0 on success, a negative errno value otherwise and rte_errno is set.
2227  */
2228 static int
2229 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
2230                            uint64_t item_flags,
2231                            struct rte_eth_dev *dev,
2232                            struct rte_flow_error *error)
2233 {
2234         const struct rte_flow_item_vlan *mask = item->mask;
2235         const struct rte_flow_item_vlan nic_mask = {
2236                 .tci = RTE_BE16(UINT16_MAX),
2237                 .inner_type = RTE_BE16(UINT16_MAX),
2238                 .has_more_vlan = 1,
2239         };
2240         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2241         int ret;
2242         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2243                                         MLX5_FLOW_LAYER_INNER_L4) :
2244                                        (MLX5_FLOW_LAYER_OUTER_L3 |
2245                                         MLX5_FLOW_LAYER_OUTER_L4);
2246         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2247                                         MLX5_FLOW_LAYER_OUTER_VLAN;
2248
2249         if (item_flags & vlanm)
2250                 return rte_flow_error_set(error, EINVAL,
2251                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2252                                           "multiple VLAN layers not supported");
2253         else if ((item_flags & l34m) != 0)
2254                 return rte_flow_error_set(error, EINVAL,
2255                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2256                                           "VLAN cannot follow L3/L4 layer");
2257         if (!mask)
2258                 mask = &rte_flow_item_vlan_mask;
2259         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2260                                         (const uint8_t *)&nic_mask,
2261                                         sizeof(struct rte_flow_item_vlan),
2262                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2263         if (ret)
2264                 return ret;
2265         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2266                 struct mlx5_priv *priv = dev->data->dev_private;
2267
2268                 if (priv->vmwa_context) {
2269                         /*
2270                          * Non-NULL context means we have a virtual machine
2271                          * and SR-IOV enabled, we have to create VLAN interface
2272                          * to make hypervisor to setup E-Switch vport
2273                          * context correctly. We avoid creating the multiple
2274                          * VLAN interfaces, so we cannot support VLAN tag mask.
2275                          */
2276                         return rte_flow_error_set(error, EINVAL,
2277                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2278                                                   item,
2279                                                   "VLAN tag mask is not"
2280                                                   " supported in virtual"
2281                                                   " environment");
2282                 }
2283         }
2284         return 0;
2285 }
2286
2287 /*
2288  * GTP flags are contained in 1 byte of the format:
2289  * -------------------------------------------
2290  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
2291  * |-----------------------------------------|
2292  * | value | Version | PT | Res | E | S | PN |
2293  * -------------------------------------------
2294  *
2295  * Matching is supported only for GTP flags E, S, PN.
2296  */
2297 #define MLX5_GTP_FLAGS_MASK     0x07
2298
2299 /**
2300  * Validate GTP item.
2301  *
2302  * @param[in] dev
2303  *   Pointer to the rte_eth_dev structure.
2304  * @param[in] item
2305  *   Item specification.
2306  * @param[in] item_flags
2307  *   Bit-fields that holds the items detected until now.
2308  * @param[out] error
2309  *   Pointer to error structure.
2310  *
2311  * @return
2312  *   0 on success, a negative errno value otherwise and rte_errno is set.
2313  */
2314 static int
2315 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
2316                           const struct rte_flow_item *item,
2317                           uint64_t item_flags,
2318                           struct rte_flow_error *error)
2319 {
2320         struct mlx5_priv *priv = dev->data->dev_private;
2321         const struct rte_flow_item_gtp *spec = item->spec;
2322         const struct rte_flow_item_gtp *mask = item->mask;
2323         const struct rte_flow_item_gtp nic_mask = {
2324                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
2325                 .msg_type = 0xff,
2326                 .teid = RTE_BE32(0xffffffff),
2327         };
2328
2329         if (!priv->sh->cdev->config.hca_attr.tunnel_stateless_gtp)
2330                 return rte_flow_error_set(error, ENOTSUP,
2331                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2332                                           "GTP support is not enabled");
2333         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2334                 return rte_flow_error_set(error, ENOTSUP,
2335                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2336                                           "multiple tunnel layers not"
2337                                           " supported");
2338         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2339                 return rte_flow_error_set(error, EINVAL,
2340                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2341                                           "no outer UDP layer found");
2342         if (!mask)
2343                 mask = &rte_flow_item_gtp_mask;
2344         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
2345                 return rte_flow_error_set(error, ENOTSUP,
2346                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2347                                           "Match is supported for GTP"
2348                                           " flags only");
2349         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2350                                          (const uint8_t *)&nic_mask,
2351                                          sizeof(struct rte_flow_item_gtp),
2352                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2353 }
2354
2355 /**
2356  * Validate GTP PSC item.
2357  *
2358  * @param[in] item
2359  *   Item specification.
2360  * @param[in] last_item
2361  *   Previous validated item in the pattern items.
2362  * @param[in] gtp_item
2363  *   Previous GTP item specification.
2364  * @param[in] attr
2365  *   Pointer to flow attributes.
2366  * @param[out] error
2367  *   Pointer to error structure.
2368  *
2369  * @return
2370  *   0 on success, a negative errno value otherwise and rte_errno is set.
2371  */
2372 static int
2373 flow_dv_validate_item_gtp_psc(const struct rte_flow_item *item,
2374                               uint64_t last_item,
2375                               const struct rte_flow_item *gtp_item,
2376                               const struct rte_flow_attr *attr,
2377                               struct rte_flow_error *error)
2378 {
2379         const struct rte_flow_item_gtp *gtp_spec;
2380         const struct rte_flow_item_gtp *gtp_mask;
2381         const struct rte_flow_item_gtp_psc *mask;
2382         const struct rte_flow_item_gtp_psc nic_mask = {
2383                 .hdr.type = 0xF,
2384                 .hdr.qfi = 0x3F,
2385         };
2386
2387         if (!gtp_item || !(last_item & MLX5_FLOW_LAYER_GTP))
2388                 return rte_flow_error_set
2389                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2390                          "GTP PSC item must be preceded with GTP item");
2391         gtp_spec = gtp_item->spec;
2392         gtp_mask = gtp_item->mask ? gtp_item->mask : &rte_flow_item_gtp_mask;
2393         /* GTP spec and E flag is requested to match zero. */
2394         if (gtp_spec &&
2395                 (gtp_mask->v_pt_rsv_flags &
2396                 ~gtp_spec->v_pt_rsv_flags & MLX5_GTP_EXT_HEADER_FLAG))
2397                 return rte_flow_error_set
2398                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2399                          "GTP E flag must be 1 to match GTP PSC");
2400         /* Check the flow is not created in group zero. */
2401         if (!attr->transfer && !attr->group)
2402                 return rte_flow_error_set
2403                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2404                          "GTP PSC is not supported for group 0");
2405         /* GTP spec is here and E flag is requested to match zero. */
2406         if (!item->spec)
2407                 return 0;
2408         mask = item->mask ? item->mask : &rte_flow_item_gtp_psc_mask;
2409         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2410                                          (const uint8_t *)&nic_mask,
2411                                          sizeof(struct rte_flow_item_gtp_psc),
2412                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2413 }
2414
2415 /**
2416  * Validate IPV4 item.
2417  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
2418  * add specific validation of fragment_offset field,
2419  *
2420  * @param[in] item
2421  *   Item specification.
2422  * @param[in] item_flags
2423  *   Bit-fields that holds the items detected until now.
2424  * @param[out] error
2425  *   Pointer to error structure.
2426  *
2427  * @return
2428  *   0 on success, a negative errno value otherwise and rte_errno is set.
2429  */
2430 static int
2431 flow_dv_validate_item_ipv4(struct rte_eth_dev *dev,
2432                            const struct rte_flow_item *item,
2433                            uint64_t item_flags, uint64_t last_item,
2434                            uint16_t ether_type, struct rte_flow_error *error)
2435 {
2436         int ret;
2437         struct mlx5_priv *priv = dev->data->dev_private;
2438         struct mlx5_hca_attr *attr = &priv->sh->cdev->config.hca_attr;
2439         const struct rte_flow_item_ipv4 *spec = item->spec;
2440         const struct rte_flow_item_ipv4 *last = item->last;
2441         const struct rte_flow_item_ipv4 *mask = item->mask;
2442         rte_be16_t fragment_offset_spec = 0;
2443         rte_be16_t fragment_offset_last = 0;
2444         struct rte_flow_item_ipv4 nic_ipv4_mask = {
2445                 .hdr = {
2446                         .src_addr = RTE_BE32(0xffffffff),
2447                         .dst_addr = RTE_BE32(0xffffffff),
2448                         .type_of_service = 0xff,
2449                         .fragment_offset = RTE_BE16(0xffff),
2450                         .next_proto_id = 0xff,
2451                         .time_to_live = 0xff,
2452                 },
2453         };
2454
2455         if (mask && (mask->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK)) {
2456                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2457                 bool ihl_cap = !tunnel ?
2458                                attr->outer_ipv4_ihl : attr->inner_ipv4_ihl;
2459                 if (!ihl_cap)
2460                         return rte_flow_error_set(error, ENOTSUP,
2461                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2462                                                   item,
2463                                                   "IPV4 ihl offload not supported");
2464                 nic_ipv4_mask.hdr.version_ihl = mask->hdr.version_ihl;
2465         }
2466         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
2467                                            ether_type, &nic_ipv4_mask,
2468                                            MLX5_ITEM_RANGE_ACCEPTED, error);
2469         if (ret < 0)
2470                 return ret;
2471         if (spec && mask)
2472                 fragment_offset_spec = spec->hdr.fragment_offset &
2473                                        mask->hdr.fragment_offset;
2474         if (!fragment_offset_spec)
2475                 return 0;
2476         /*
2477          * spec and mask are valid, enforce using full mask to make sure the
2478          * complete value is used correctly.
2479          */
2480         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2481                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2482                 return rte_flow_error_set(error, EINVAL,
2483                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2484                                           item, "must use full mask for"
2485                                           " fragment_offset");
2486         /*
2487          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
2488          * indicating this is 1st fragment of fragmented packet.
2489          * This is not yet supported in MLX5, return appropriate error message.
2490          */
2491         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
2492                 return rte_flow_error_set(error, ENOTSUP,
2493                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2494                                           "match on first fragment not "
2495                                           "supported");
2496         if (fragment_offset_spec && !last)
2497                 return rte_flow_error_set(error, ENOTSUP,
2498                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2499                                           "specified value not supported");
2500         /* spec and last are valid, validate the specified range. */
2501         fragment_offset_last = last->hdr.fragment_offset &
2502                                mask->hdr.fragment_offset;
2503         /*
2504          * Match on fragment_offset spec 0x2001 and last 0x3fff
2505          * means MF is 1 and frag-offset is > 0.
2506          * This packet is fragment 2nd and onward, excluding last.
2507          * This is not yet supported in MLX5, return appropriate
2508          * error message.
2509          */
2510         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
2511             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2512                 return rte_flow_error_set(error, ENOTSUP,
2513                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2514                                           last, "match on following "
2515                                           "fragments not supported");
2516         /*
2517          * Match on fragment_offset spec 0x0001 and last 0x1fff
2518          * means MF is 0 and frag-offset is > 0.
2519          * This packet is last fragment of fragmented packet.
2520          * This is not yet supported in MLX5, return appropriate
2521          * error message.
2522          */
2523         if (fragment_offset_spec == RTE_BE16(1) &&
2524             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
2525                 return rte_flow_error_set(error, ENOTSUP,
2526                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2527                                           last, "match on last "
2528                                           "fragment not supported");
2529         /*
2530          * Match on fragment_offset spec 0x0001 and last 0x3fff
2531          * means MF and/or frag-offset is not 0.
2532          * This is a fragmented packet.
2533          * Other range values are invalid and rejected.
2534          */
2535         if (!(fragment_offset_spec == RTE_BE16(1) &&
2536               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
2537                 return rte_flow_error_set(error, ENOTSUP,
2538                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2539                                           "specified range not supported");
2540         return 0;
2541 }
2542
2543 /**
2544  * Validate IPV6 fragment extension item.
2545  *
2546  * @param[in] item
2547  *   Item specification.
2548  * @param[in] item_flags
2549  *   Bit-fields that holds the items detected until now.
2550  * @param[out] error
2551  *   Pointer to error structure.
2552  *
2553  * @return
2554  *   0 on success, a negative errno value otherwise and rte_errno is set.
2555  */
2556 static int
2557 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
2558                                     uint64_t item_flags,
2559                                     struct rte_flow_error *error)
2560 {
2561         const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
2562         const struct rte_flow_item_ipv6_frag_ext *last = item->last;
2563         const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
2564         rte_be16_t frag_data_spec = 0;
2565         rte_be16_t frag_data_last = 0;
2566         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2567         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2568                                       MLX5_FLOW_LAYER_OUTER_L4;
2569         int ret = 0;
2570         struct rte_flow_item_ipv6_frag_ext nic_mask = {
2571                 .hdr = {
2572                         .next_header = 0xff,
2573                         .frag_data = RTE_BE16(0xffff),
2574                 },
2575         };
2576
2577         if (item_flags & l4m)
2578                 return rte_flow_error_set(error, EINVAL,
2579                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2580                                           "ipv6 fragment extension item cannot "
2581                                           "follow L4 item.");
2582         if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
2583             (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2584                 return rte_flow_error_set(error, EINVAL,
2585                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2586                                           "ipv6 fragment extension item must "
2587                                           "follow ipv6 item");
2588         if (spec && mask)
2589                 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
2590         if (!frag_data_spec)
2591                 return 0;
2592         /*
2593          * spec and mask are valid, enforce using full mask to make sure the
2594          * complete value is used correctly.
2595          */
2596         if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
2597                                 RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2598                 return rte_flow_error_set(error, EINVAL,
2599                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2600                                           item, "must use full mask for"
2601                                           " frag_data");
2602         /*
2603          * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
2604          * This is 1st fragment of fragmented packet.
2605          */
2606         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
2607                 return rte_flow_error_set(error, ENOTSUP,
2608                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2609                                           "match on first fragment not "
2610                                           "supported");
2611         if (frag_data_spec && !last)
2612                 return rte_flow_error_set(error, EINVAL,
2613                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2614                                           "specified value not supported");
2615         ret = mlx5_flow_item_acceptable
2616                                 (item, (const uint8_t *)mask,
2617                                  (const uint8_t *)&nic_mask,
2618                                  sizeof(struct rte_flow_item_ipv6_frag_ext),
2619                                  MLX5_ITEM_RANGE_ACCEPTED, error);
2620         if (ret)
2621                 return ret;
2622         /* spec and last are valid, validate the specified range. */
2623         frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
2624         /*
2625          * Match on frag_data spec 0x0009 and last 0xfff9
2626          * means M is 1 and frag-offset is > 0.
2627          * This packet is fragment 2nd and onward, excluding last.
2628          * This is not yet supported in MLX5, return appropriate
2629          * error message.
2630          */
2631         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
2632                                        RTE_IPV6_EHDR_MF_MASK) &&
2633             frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2634                 return rte_flow_error_set(error, ENOTSUP,
2635                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2636                                           last, "match on following "
2637                                           "fragments not supported");
2638         /*
2639          * Match on frag_data spec 0x0008 and last 0xfff8
2640          * means M is 0 and frag-offset is > 0.
2641          * This packet is last fragment of fragmented packet.
2642          * This is not yet supported in MLX5, return appropriate
2643          * error message.
2644          */
2645         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
2646             frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
2647                 return rte_flow_error_set(error, ENOTSUP,
2648                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2649                                           last, "match on last "
2650                                           "fragment not supported");
2651         /* Other range values are invalid and rejected. */
2652         return rte_flow_error_set(error, EINVAL,
2653                                   RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2654                                   "specified range not supported");
2655 }
2656
2657 /*
2658  * Validate ASO CT item.
2659  *
2660  * @param[in] dev
2661  *   Pointer to the rte_eth_dev structure.
2662  * @param[in] item
2663  *   Item specification.
2664  * @param[in] item_flags
2665  *   Pointer to bit-fields that holds the items detected until now.
2666  * @param[out] error
2667  *   Pointer to error structure.
2668  *
2669  * @return
2670  *   0 on success, a negative errno value otherwise and rte_errno is set.
2671  */
2672 static int
2673 flow_dv_validate_item_aso_ct(struct rte_eth_dev *dev,
2674                              const struct rte_flow_item *item,
2675                              uint64_t *item_flags,
2676                              struct rte_flow_error *error)
2677 {
2678         const struct rte_flow_item_conntrack *spec = item->spec;
2679         const struct rte_flow_item_conntrack *mask = item->mask;
2680         RTE_SET_USED(dev);
2681         uint32_t flags;
2682
2683         if (*item_flags & MLX5_FLOW_LAYER_ASO_CT)
2684                 return rte_flow_error_set(error, EINVAL,
2685                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2686                                           "Only one CT is supported");
2687         if (!mask)
2688                 mask = &rte_flow_item_conntrack_mask;
2689         flags = spec->flags & mask->flags;
2690         if ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID) &&
2691             ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID) ||
2692              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD) ||
2693              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)))
2694                 return rte_flow_error_set(error, EINVAL,
2695                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2696                                           "Conflict status bits");
2697         /* State change also needs to be considered. */
2698         *item_flags |= MLX5_FLOW_LAYER_ASO_CT;
2699         return 0;
2700 }
2701
2702 /**
2703  * Validate the pop VLAN action.
2704  *
2705  * @param[in] dev
2706  *   Pointer to the rte_eth_dev structure.
2707  * @param[in] action_flags
2708  *   Holds the actions detected until now.
2709  * @param[in] action
2710  *   Pointer to the pop vlan action.
2711  * @param[in] item_flags
2712  *   The items found in this flow rule.
2713  * @param[in] attr
2714  *   Pointer to flow attributes.
2715  * @param[out] error
2716  *   Pointer to error structure.
2717  *
2718  * @return
2719  *   0 on success, a negative errno value otherwise and rte_errno is set.
2720  */
2721 static int
2722 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
2723                                  uint64_t action_flags,
2724                                  const struct rte_flow_action *action,
2725                                  uint64_t item_flags,
2726                                  const struct rte_flow_attr *attr,
2727                                  struct rte_flow_error *error)
2728 {
2729         const struct mlx5_priv *priv = dev->data->dev_private;
2730         struct mlx5_dev_ctx_shared *sh = priv->sh;
2731         bool direction_error = false;
2732
2733         if (!priv->sh->pop_vlan_action)
2734                 return rte_flow_error_set(error, ENOTSUP,
2735                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2736                                           NULL,
2737                                           "pop vlan action is not supported");
2738         /* Pop VLAN is not supported in egress except for CX6 FDB mode. */
2739         if (attr->transfer) {
2740                 bool fdb_tx = priv->representor_id != UINT16_MAX;
2741                 bool is_cx5 = sh->steering_format_version ==
2742                     MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5;
2743
2744                 if (fdb_tx && is_cx5)
2745                         direction_error = true;
2746         } else if (attr->egress) {
2747                 direction_error = true;
2748         }
2749         if (direction_error)
2750                 return rte_flow_error_set(error, ENOTSUP,
2751                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2752                                           NULL,
2753                                           "pop vlan action not supported for egress");
2754         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
2755                 return rte_flow_error_set(error, ENOTSUP,
2756                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2757                                           "no support for multiple VLAN "
2758                                           "actions");
2759         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
2760         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
2761             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
2762                 return rte_flow_error_set(error, ENOTSUP,
2763                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2764                                           NULL,
2765                                           "cannot pop vlan after decap without "
2766                                           "match on inner vlan in the flow");
2767         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
2768         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
2769             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2770                 return rte_flow_error_set(error, ENOTSUP,
2771                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2772                                           NULL,
2773                                           "cannot pop vlan without a "
2774                                           "match on (outer) vlan in the flow");
2775         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2776                 return rte_flow_error_set(error, EINVAL,
2777                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2778                                           "wrong action order, port_id should "
2779                                           "be after pop VLAN action");
2780         if (!attr->transfer && priv->representor)
2781                 return rte_flow_error_set(error, ENOTSUP,
2782                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2783                                           "pop vlan action for VF representor "
2784                                           "not supported on NIC table");
2785         return 0;
2786 }
2787
2788 /**
2789  * Get VLAN default info from vlan match info.
2790  *
2791  * @param[in] items
2792  *   the list of item specifications.
2793  * @param[out] vlan
2794  *   pointer VLAN info to fill to.
2795  *
2796  * @return
2797  *   0 on success, a negative errno value otherwise and rte_errno is set.
2798  */
2799 static void
2800 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2801                                   struct rte_vlan_hdr *vlan)
2802 {
2803         const struct rte_flow_item_vlan nic_mask = {
2804                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2805                                 MLX5DV_FLOW_VLAN_VID_MASK),
2806                 .inner_type = RTE_BE16(0xffff),
2807         };
2808
2809         if (items == NULL)
2810                 return;
2811         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2812                 int type = items->type;
2813
2814                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2815                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2816                         break;
2817         }
2818         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2819                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2820                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2821
2822                 /* If VLAN item in pattern doesn't contain data, return here. */
2823                 if (!vlan_v)
2824                         return;
2825                 if (!vlan_m)
2826                         vlan_m = &nic_mask;
2827                 /* Only full match values are accepted */
2828                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2829                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2830                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2831                         vlan->vlan_tci |=
2832                                 rte_be_to_cpu_16(vlan_v->tci &
2833                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2834                 }
2835                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2836                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2837                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2838                         vlan->vlan_tci |=
2839                                 rte_be_to_cpu_16(vlan_v->tci &
2840                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2841                 }
2842                 if (vlan_m->inner_type == nic_mask.inner_type)
2843                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2844                                                            vlan_m->inner_type);
2845         }
2846 }
2847
2848 /**
2849  * Validate the push VLAN action.
2850  *
2851  * @param[in] dev
2852  *   Pointer to the rte_eth_dev structure.
2853  * @param[in] action_flags
2854  *   Holds the actions detected until now.
2855  * @param[in] item_flags
2856  *   The items found in this flow rule.
2857  * @param[in] action
2858  *   Pointer to the action structure.
2859  * @param[in] attr
2860  *   Pointer to flow attributes
2861  * @param[out] error
2862  *   Pointer to error structure.
2863  *
2864  * @return
2865  *   0 on success, a negative errno value otherwise and rte_errno is set.
2866  */
2867 static int
2868 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2869                                   uint64_t action_flags,
2870                                   const struct rte_flow_item_vlan *vlan_m,
2871                                   const struct rte_flow_action *action,
2872                                   const struct rte_flow_attr *attr,
2873                                   struct rte_flow_error *error)
2874 {
2875         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2876         const struct mlx5_priv *priv = dev->data->dev_private;
2877
2878         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2879             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2880                 return rte_flow_error_set(error, EINVAL,
2881                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2882                                           "invalid vlan ethertype");
2883         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2884                 return rte_flow_error_set(error, EINVAL,
2885                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2886                                           "wrong action order, port_id should "
2887                                           "be after push VLAN");
2888         if (!attr->transfer && priv->representor)
2889                 return rte_flow_error_set(error, ENOTSUP,
2890                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2891                                           "push vlan action for VF representor "
2892                                           "not supported on NIC table");
2893         if (vlan_m &&
2894             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2895             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2896                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2897             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2898             !(mlx5_flow_find_action
2899                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2900                 return rte_flow_error_set(error, EINVAL,
2901                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2902                                           "not full match mask on VLAN PCP and "
2903                                           "there is no of_set_vlan_pcp action, "
2904                                           "push VLAN action cannot figure out "
2905                                           "PCP value");
2906         if (vlan_m &&
2907             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2908             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2909                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2910             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2911             !(mlx5_flow_find_action
2912                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2913                 return rte_flow_error_set(error, EINVAL,
2914                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2915                                           "not full match mask on VLAN VID and "
2916                                           "there is no of_set_vlan_vid action, "
2917                                           "push VLAN action cannot figure out "
2918                                           "VID value");
2919         (void)attr;
2920         return 0;
2921 }
2922
2923 /**
2924  * Validate the set VLAN PCP.
2925  *
2926  * @param[in] action_flags
2927  *   Holds the actions detected until now.
2928  * @param[in] actions
2929  *   Pointer to the list of actions remaining in the flow rule.
2930  * @param[out] error
2931  *   Pointer to error structure.
2932  *
2933  * @return
2934  *   0 on success, a negative errno value otherwise and rte_errno is set.
2935  */
2936 static int
2937 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2938                                      const struct rte_flow_action actions[],
2939                                      struct rte_flow_error *error)
2940 {
2941         const struct rte_flow_action *action = actions;
2942         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2943
2944         if (conf->vlan_pcp > 7)
2945                 return rte_flow_error_set(error, EINVAL,
2946                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2947                                           "VLAN PCP value is too big");
2948         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2949                 return rte_flow_error_set(error, ENOTSUP,
2950                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2951                                           "set VLAN PCP action must follow "
2952                                           "the push VLAN action");
2953         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2954                 return rte_flow_error_set(error, ENOTSUP,
2955                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2956                                           "Multiple VLAN PCP modification are "
2957                                           "not supported");
2958         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2959                 return rte_flow_error_set(error, EINVAL,
2960                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2961                                           "wrong action order, port_id should "
2962                                           "be after set VLAN PCP");
2963         return 0;
2964 }
2965
2966 /**
2967  * Validate the set VLAN VID.
2968  *
2969  * @param[in] item_flags
2970  *   Holds the items detected in this rule.
2971  * @param[in] action_flags
2972  *   Holds the actions detected until now.
2973  * @param[in] actions
2974  *   Pointer to the list of actions remaining in the flow rule.
2975  * @param[out] error
2976  *   Pointer to error structure.
2977  *
2978  * @return
2979  *   0 on success, a negative errno value otherwise and rte_errno is set.
2980  */
2981 static int
2982 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2983                                      uint64_t action_flags,
2984                                      const struct rte_flow_action actions[],
2985                                      struct rte_flow_error *error)
2986 {
2987         const struct rte_flow_action *action = actions;
2988         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2989
2990         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2991                 return rte_flow_error_set(error, EINVAL,
2992                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2993                                           "VLAN VID value is too big");
2994         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2995             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2996                 return rte_flow_error_set(error, ENOTSUP,
2997                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2998                                           "set VLAN VID action must follow push"
2999                                           " VLAN action or match on VLAN item");
3000         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
3001                 return rte_flow_error_set(error, ENOTSUP,
3002                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3003                                           "Multiple VLAN VID modifications are "
3004                                           "not supported");
3005         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
3006                 return rte_flow_error_set(error, EINVAL,
3007                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3008                                           "wrong action order, port_id should "
3009                                           "be after set VLAN VID");
3010         return 0;
3011 }
3012
3013 /*
3014  * Validate the FLAG action.
3015  *
3016  * @param[in] dev
3017  *   Pointer to the rte_eth_dev structure.
3018  * @param[in] action_flags
3019  *   Holds the actions detected until now.
3020  * @param[in] attr
3021  *   Pointer to flow attributes
3022  * @param[out] error
3023  *   Pointer to error structure.
3024  *
3025  * @return
3026  *   0 on success, a negative errno value otherwise and rte_errno is set.
3027  */
3028 static int
3029 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
3030                              uint64_t action_flags,
3031                              const struct rte_flow_attr *attr,
3032                              struct rte_flow_error *error)
3033 {
3034         struct mlx5_priv *priv = dev->data->dev_private;
3035         struct mlx5_sh_config *config = &priv->sh->config;
3036         int ret;
3037
3038         /* Fall back if no extended metadata register support. */
3039         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3040                 return mlx5_flow_validate_action_flag(action_flags, attr,
3041                                                       error);
3042         /* Extensive metadata mode requires registers. */
3043         if (!mlx5_flow_ext_mreg_supported(dev))
3044                 return rte_flow_error_set(error, ENOTSUP,
3045                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3046                                           "no metadata registers "
3047                                           "to support flag action");
3048         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
3049                 return rte_flow_error_set(error, ENOTSUP,
3050                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3051                                           "extended metadata register"
3052                                           " isn't available");
3053         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3054         if (ret < 0)
3055                 return ret;
3056         MLX5_ASSERT(ret > 0);
3057         if (action_flags & MLX5_FLOW_ACTION_MARK)
3058                 return rte_flow_error_set(error, EINVAL,
3059                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3060                                           "can't mark and flag in same flow");
3061         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3062                 return rte_flow_error_set(error, EINVAL,
3063                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3064                                           "can't have 2 flag"
3065                                           " actions in same flow");
3066         return 0;
3067 }
3068
3069 /**
3070  * Validate MARK action.
3071  *
3072  * @param[in] dev
3073  *   Pointer to the rte_eth_dev structure.
3074  * @param[in] action
3075  *   Pointer to action.
3076  * @param[in] action_flags
3077  *   Holds the actions detected until now.
3078  * @param[in] attr
3079  *   Pointer to flow attributes
3080  * @param[out] error
3081  *   Pointer to error structure.
3082  *
3083  * @return
3084  *   0 on success, a negative errno value otherwise and rte_errno is set.
3085  */
3086 static int
3087 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
3088                              const struct rte_flow_action *action,
3089                              uint64_t action_flags,
3090                              const struct rte_flow_attr *attr,
3091                              struct rte_flow_error *error)
3092 {
3093         struct mlx5_priv *priv = dev->data->dev_private;
3094         struct mlx5_sh_config *config = &priv->sh->config;
3095         const struct rte_flow_action_mark *mark = action->conf;
3096         int ret;
3097
3098         if (is_tunnel_offload_active(dev))
3099                 return rte_flow_error_set(error, ENOTSUP,
3100                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3101                                           "no mark action "
3102                                           "if tunnel offload active");
3103         /* Fall back if no extended metadata register support. */
3104         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3105                 return mlx5_flow_validate_action_mark(action, action_flags,
3106                                                       attr, error);
3107         /* Extensive metadata mode requires registers. */
3108         if (!mlx5_flow_ext_mreg_supported(dev))
3109                 return rte_flow_error_set(error, ENOTSUP,
3110                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3111                                           "no metadata registers "
3112                                           "to support mark action");
3113         if (!priv->sh->dv_mark_mask)
3114                 return rte_flow_error_set(error, ENOTSUP,
3115                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3116                                           "extended metadata register"
3117                                           " isn't available");
3118         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3119         if (ret < 0)
3120                 return ret;
3121         MLX5_ASSERT(ret > 0);
3122         if (!mark)
3123                 return rte_flow_error_set(error, EINVAL,
3124                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3125                                           "configuration cannot be null");
3126         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
3127                 return rte_flow_error_set(error, EINVAL,
3128                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3129                                           &mark->id,
3130                                           "mark id exceeds the limit");
3131         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3132                 return rte_flow_error_set(error, EINVAL,
3133                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3134                                           "can't flag and mark in same flow");
3135         if (action_flags & MLX5_FLOW_ACTION_MARK)
3136                 return rte_flow_error_set(error, EINVAL,
3137                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3138                                           "can't have 2 mark actions in same"
3139                                           " flow");
3140         return 0;
3141 }
3142
3143 /**
3144  * Validate SET_META action.
3145  *
3146  * @param[in] dev
3147  *   Pointer to the rte_eth_dev structure.
3148  * @param[in] action
3149  *   Pointer to the action structure.
3150  * @param[in] action_flags
3151  *   Holds the actions detected until now.
3152  * @param[in] attr
3153  *   Pointer to flow attributes
3154  * @param[out] error
3155  *   Pointer to error structure.
3156  *
3157  * @return
3158  *   0 on success, a negative errno value otherwise and rte_errno is set.
3159  */
3160 static int
3161 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
3162                                  const struct rte_flow_action *action,
3163                                  uint64_t action_flags __rte_unused,
3164                                  const struct rte_flow_attr *attr,
3165                                  struct rte_flow_error *error)
3166 {
3167         struct mlx5_priv *priv = dev->data->dev_private;
3168         struct mlx5_sh_config *config = &priv->sh->config;
3169         const struct rte_flow_action_set_meta *conf;
3170         uint32_t nic_mask = UINT32_MAX;
3171         int reg;
3172
3173         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
3174             !mlx5_flow_ext_mreg_supported(dev))
3175                 return rte_flow_error_set(error, ENOTSUP,
3176                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3177                                           "extended metadata register"
3178                                           " isn't supported");
3179         reg = flow_dv_get_metadata_reg(dev, attr, error);
3180         if (reg < 0)
3181                 return reg;
3182         if (reg == REG_NON)
3183                 return rte_flow_error_set(error, ENOTSUP,
3184                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3185                                           "unavailable extended metadata register");
3186         if (reg != REG_A && reg != REG_B) {
3187                 struct mlx5_priv *priv = dev->data->dev_private;
3188
3189                 nic_mask = priv->sh->dv_meta_mask;
3190         }
3191         if (!(action->conf))
3192                 return rte_flow_error_set(error, EINVAL,
3193                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3194                                           "configuration cannot be null");
3195         conf = (const struct rte_flow_action_set_meta *)action->conf;
3196         if (!conf->mask)
3197                 return rte_flow_error_set(error, EINVAL,
3198                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3199                                           "zero mask doesn't have any effect");
3200         if (conf->mask & ~nic_mask)
3201                 return rte_flow_error_set(error, EINVAL,
3202                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3203                                           "meta data must be within reg C0");
3204         return 0;
3205 }
3206
3207 /**
3208  * Validate SET_TAG action.
3209  *
3210  * @param[in] dev
3211  *   Pointer to the rte_eth_dev structure.
3212  * @param[in] action
3213  *   Pointer to the action structure.
3214  * @param[in] action_flags
3215  *   Holds the actions detected until now.
3216  * @param[in] attr
3217  *   Pointer to flow attributes
3218  * @param[out] error
3219  *   Pointer to error structure.
3220  *
3221  * @return
3222  *   0 on success, a negative errno value otherwise and rte_errno is set.
3223  */
3224 static int
3225 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
3226                                 const struct rte_flow_action *action,
3227                                 uint64_t action_flags,
3228                                 const struct rte_flow_attr *attr,
3229                                 struct rte_flow_error *error)
3230 {
3231         const struct rte_flow_action_set_tag *conf;
3232         const uint64_t terminal_action_flags =
3233                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
3234                 MLX5_FLOW_ACTION_RSS;
3235         int ret;
3236
3237         if (!mlx5_flow_ext_mreg_supported(dev))
3238                 return rte_flow_error_set(error, ENOTSUP,
3239                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3240                                           "extensive metadata register"
3241                                           " isn't supported");
3242         if (!(action->conf))
3243                 return rte_flow_error_set(error, EINVAL,
3244                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3245                                           "configuration cannot be null");
3246         conf = (const struct rte_flow_action_set_tag *)action->conf;
3247         if (!conf->mask)
3248                 return rte_flow_error_set(error, EINVAL,
3249                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3250                                           "zero mask doesn't have any effect");
3251         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
3252         if (ret < 0)
3253                 return ret;
3254         if (!attr->transfer && attr->ingress &&
3255             (action_flags & terminal_action_flags))
3256                 return rte_flow_error_set(error, EINVAL,
3257                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3258                                           "set_tag has no effect"
3259                                           " with terminal actions");
3260         return 0;
3261 }
3262
3263 /**
3264  * Indicates whether ASO aging is supported.
3265  *
3266  * @param[in] sh
3267  *   Pointer to shared device context structure.
3268  * @param[in] attr
3269  *   Attributes of flow that includes AGE action.
3270  *
3271  * @return
3272  *   True when ASO aging is supported, false otherwise.
3273  */
3274 static inline bool
3275 flow_hit_aso_supported(const struct mlx5_dev_ctx_shared *sh,
3276                 const struct rte_flow_attr *attr)
3277 {
3278         MLX5_ASSERT(sh && attr);
3279         return (sh->flow_hit_aso_en && (attr->transfer || attr->group));
3280 }
3281
3282 /**
3283  * Validate count action.
3284  *
3285  * @param[in] dev
3286  *   Pointer to rte_eth_dev structure.
3287  * @param[in] shared
3288  *   Indicator if action is shared.
3289  * @param[in] action_flags
3290  *   Holds the actions detected until now.
3291  * @param[in] attr
3292  *   Attributes of flow that includes this action.
3293  * @param[out] error
3294  *   Pointer to error structure.
3295  *
3296  * @return
3297  *   0 on success, a negative errno value otherwise and rte_errno is set.
3298  */
3299 static int
3300 flow_dv_validate_action_count(struct rte_eth_dev *dev, bool shared,
3301                               uint64_t action_flags,
3302                               const struct rte_flow_attr *attr,
3303                               struct rte_flow_error *error)
3304 {
3305         struct mlx5_priv *priv = dev->data->dev_private;
3306
3307         if (!priv->sh->cdev->config.devx)
3308                 goto notsup_err;
3309         if (action_flags & MLX5_FLOW_ACTION_COUNT)
3310                 return rte_flow_error_set(error, EINVAL,
3311                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3312                                           "duplicate count actions set");
3313         if (shared && (action_flags & MLX5_FLOW_ACTION_AGE) &&
3314             !flow_hit_aso_supported(priv->sh, attr))
3315                 return rte_flow_error_set(error, EINVAL,
3316                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3317                                           "old age and indirect count combination is not supported");
3318 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
3319         return 0;
3320 #endif
3321 notsup_err:
3322         return rte_flow_error_set
3323                       (error, ENOTSUP,
3324                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3325                        NULL,
3326                        "count action not supported");
3327 }
3328
3329 /**
3330  * Validate the L2 encap action.
3331  *
3332  * @param[in] dev
3333  *   Pointer to the rte_eth_dev structure.
3334  * @param[in] action_flags
3335  *   Holds the actions detected until now.
3336  * @param[in] action
3337  *   Pointer to the action structure.
3338  * @param[in] attr
3339  *   Pointer to flow attributes.
3340  * @param[out] error
3341  *   Pointer to error structure.
3342  *
3343  * @return
3344  *   0 on success, a negative errno value otherwise and rte_errno is set.
3345  */
3346 static int
3347 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
3348                                  uint64_t action_flags,
3349                                  const struct rte_flow_action *action,
3350                                  const struct rte_flow_attr *attr,
3351                                  struct rte_flow_error *error)
3352 {
3353         const struct mlx5_priv *priv = dev->data->dev_private;
3354
3355         if (!(action->conf))
3356                 return rte_flow_error_set(error, EINVAL,
3357                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3358                                           "configuration cannot be null");
3359         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3360                 return rte_flow_error_set(error, EINVAL,
3361                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3362                                           "can only have a single encap action "
3363                                           "in a flow");
3364         if (!attr->transfer && priv->representor)
3365                 return rte_flow_error_set(error, ENOTSUP,
3366                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3367                                           "encap action for VF representor "
3368                                           "not supported on NIC table");
3369         return 0;
3370 }
3371
3372 /**
3373  * Validate a decap action.
3374  *
3375  * @param[in] dev
3376  *   Pointer to the rte_eth_dev structure.
3377  * @param[in] action_flags
3378  *   Holds the actions detected until now.
3379  * @param[in] action
3380  *   Pointer to the action structure.
3381  * @param[in] item_flags
3382  *   Holds the items detected.
3383  * @param[in] attr
3384  *   Pointer to flow attributes
3385  * @param[out] error
3386  *   Pointer to error structure.
3387  *
3388  * @return
3389  *   0 on success, a negative errno value otherwise and rte_errno is set.
3390  */
3391 static int
3392 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
3393                               uint64_t action_flags,
3394                               const struct rte_flow_action *action,
3395                               const uint64_t item_flags,
3396                               const struct rte_flow_attr *attr,
3397                               struct rte_flow_error *error)
3398 {
3399         const struct mlx5_priv *priv = dev->data->dev_private;
3400
3401         if (priv->sh->cdev->config.hca_attr.scatter_fcs_w_decap_disable &&
3402             !priv->sh->config.decap_en)
3403                 return rte_flow_error_set(error, ENOTSUP,
3404                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3405                                           "decap is not enabled");
3406         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
3407                 return rte_flow_error_set(error, ENOTSUP,
3408                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3409                                           action_flags &
3410                                           MLX5_FLOW_ACTION_DECAP ? "can only "
3411                                           "have a single decap action" : "decap "
3412                                           "after encap is not supported");
3413         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
3414                 return rte_flow_error_set(error, EINVAL,
3415                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3416                                           "can't have decap action after"
3417                                           " modify action");
3418         if (attr->egress)
3419                 return rte_flow_error_set(error, ENOTSUP,
3420                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
3421                                           NULL,
3422                                           "decap action not supported for "
3423                                           "egress");
3424         if (!attr->transfer && priv->representor)
3425                 return rte_flow_error_set(error, ENOTSUP,
3426                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3427                                           "decap action for VF representor "
3428                                           "not supported on NIC table");
3429         if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_DECAP &&
3430             !(item_flags & MLX5_FLOW_LAYER_VXLAN))
3431                 return rte_flow_error_set(error, ENOTSUP,
3432                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3433                                 "VXLAN item should be present for VXLAN decap");
3434         return 0;
3435 }
3436
3437 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
3438
3439 /**
3440  * Validate the raw encap and decap actions.
3441  *
3442  * @param[in] dev
3443  *   Pointer to the rte_eth_dev structure.
3444  * @param[in] decap
3445  *   Pointer to the decap action.
3446  * @param[in] encap
3447  *   Pointer to the encap action.
3448  * @param[in] attr
3449  *   Pointer to flow attributes
3450  * @param[in/out] action_flags
3451  *   Holds the actions detected until now.
3452  * @param[out] actions_n
3453  *   pointer to the number of actions counter.
3454  * @param[in] action
3455  *   Pointer to the action structure.
3456  * @param[in] item_flags
3457  *   Holds the items detected.
3458  * @param[out] error
3459  *   Pointer to error structure.
3460  *
3461  * @return
3462  *   0 on success, a negative errno value otherwise and rte_errno is set.
3463  */
3464 static int
3465 flow_dv_validate_action_raw_encap_decap
3466         (struct rte_eth_dev *dev,
3467          const struct rte_flow_action_raw_decap *decap,
3468          const struct rte_flow_action_raw_encap *encap,
3469          const struct rte_flow_attr *attr, uint64_t *action_flags,
3470          int *actions_n, const struct rte_flow_action *action,
3471          uint64_t item_flags, struct rte_flow_error *error)
3472 {
3473         const struct mlx5_priv *priv = dev->data->dev_private;
3474         int ret;
3475
3476         if (encap && (!encap->size || !encap->data))
3477                 return rte_flow_error_set(error, EINVAL,
3478                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3479                                           "raw encap data cannot be empty");
3480         if (decap && encap) {
3481                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
3482                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3483                         /* L3 encap. */
3484                         decap = NULL;
3485                 else if (encap->size <=
3486                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3487                            decap->size >
3488                            MLX5_ENCAPSULATION_DECISION_SIZE)
3489                         /* L3 decap. */
3490                         encap = NULL;
3491                 else if (encap->size >
3492                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3493                            decap->size >
3494                            MLX5_ENCAPSULATION_DECISION_SIZE)
3495                         /* 2 L2 actions: encap and decap. */
3496                         ;
3497                 else
3498                         return rte_flow_error_set(error,
3499                                 ENOTSUP,
3500                                 RTE_FLOW_ERROR_TYPE_ACTION,
3501                                 NULL, "unsupported too small "
3502                                 "raw decap and too small raw "
3503                                 "encap combination");
3504         }
3505         if (decap) {
3506                 ret = flow_dv_validate_action_decap(dev, *action_flags, action,
3507                                                     item_flags, attr, error);
3508                 if (ret < 0)
3509                         return ret;
3510                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
3511                 ++(*actions_n);
3512         }
3513         if (encap) {
3514                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
3515                         return rte_flow_error_set(error, ENOTSUP,
3516                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3517                                                   NULL,
3518                                                   "small raw encap size");
3519                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
3520                         return rte_flow_error_set(error, EINVAL,
3521                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3522                                                   NULL,
3523                                                   "more than one encap action");
3524                 if (!attr->transfer && priv->representor)
3525                         return rte_flow_error_set
3526                                         (error, ENOTSUP,
3527                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3528                                          "encap action for VF representor "
3529                                          "not supported on NIC table");
3530                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
3531                 ++(*actions_n);
3532         }
3533         return 0;
3534 }
3535
3536 /*
3537  * Validate the ASO CT action.
3538  *
3539  * @param[in] dev
3540  *   Pointer to the rte_eth_dev structure.
3541  * @param[in] action_flags
3542  *   Holds the actions detected until now.
3543  * @param[in] item_flags
3544  *   The items found in this flow rule.
3545  * @param[in] attr
3546  *   Pointer to flow attributes.
3547  * @param[out] error
3548  *   Pointer to error structure.
3549  *
3550  * @return
3551  *   0 on success, a negative errno value otherwise and rte_errno is set.
3552  */
3553 static int
3554 flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev,
3555                                uint64_t action_flags,
3556                                uint64_t item_flags,
3557                                const struct rte_flow_attr *attr,
3558                                struct rte_flow_error *error)
3559 {
3560         RTE_SET_USED(dev);
3561
3562         if (attr->group == 0 && !attr->transfer)
3563                 return rte_flow_error_set(error, ENOTSUP,
3564                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3565                                           NULL,
3566                                           "Only support non-root table");
3567         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
3568                 return rte_flow_error_set(error, ENOTSUP,
3569                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3570                                           "CT cannot follow a fate action");
3571         if ((action_flags & MLX5_FLOW_ACTION_METER) ||
3572             (action_flags & MLX5_FLOW_ACTION_AGE))
3573                 return rte_flow_error_set(error, EINVAL,
3574                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3575                                           "Only one ASO action is supported");
3576         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3577                 return rte_flow_error_set(error, EINVAL,
3578                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3579                                           "Encap cannot exist before CT");
3580         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
3581                 return rte_flow_error_set(error, EINVAL,
3582                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3583                                           "Not a outer TCP packet");
3584         return 0;
3585 }
3586
3587 int
3588 flow_dv_encap_decap_match_cb(void *tool_ctx __rte_unused,
3589                              struct mlx5_list_entry *entry, void *cb_ctx)
3590 {
3591         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3592         struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
3593         struct mlx5_flow_dv_encap_decap_resource *resource;
3594
3595         resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource,
3596                                 entry);
3597         if (resource->reformat_type == ctx_resource->reformat_type &&
3598             resource->ft_type == ctx_resource->ft_type &&
3599             resource->flags == ctx_resource->flags &&
3600             resource->size == ctx_resource->size &&
3601             !memcmp((const void *)resource->buf,
3602                     (const void *)ctx_resource->buf,
3603                     resource->size))
3604                 return 0;
3605         return -1;
3606 }
3607
3608 struct mlx5_list_entry *
3609 flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
3610 {
3611         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3612         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3613         struct mlx5dv_dr_domain *domain;
3614         struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
3615         struct mlx5_flow_dv_encap_decap_resource *resource;
3616         uint32_t idx;
3617         int ret;
3618
3619         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3620                 domain = sh->fdb_domain;
3621         else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3622                 domain = sh->rx_domain;
3623         else
3624                 domain = sh->tx_domain;
3625         /* Register new encap/decap resource. */
3626         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &idx);
3627         if (!resource) {
3628                 rte_flow_error_set(ctx->error, ENOMEM,
3629                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3630                                    "cannot allocate resource memory");
3631                 return NULL;
3632         }
3633         *resource = *ctx_resource;
3634         resource->idx = idx;
3635         ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->cdev->ctx,
3636                                                               domain, resource,
3637                                                              &resource->action);
3638         if (ret) {
3639                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
3640                 rte_flow_error_set(ctx->error, ENOMEM,
3641                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3642                                    NULL, "cannot create action");
3643                 return NULL;
3644         }
3645
3646         return &resource->entry;
3647 }
3648
3649 struct mlx5_list_entry *
3650 flow_dv_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
3651                              void *cb_ctx)
3652 {
3653         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3654         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3655         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
3656         uint32_t idx;
3657
3658         cache_resource = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
3659                                            &idx);
3660         if (!cache_resource) {
3661                 rte_flow_error_set(ctx->error, ENOMEM,
3662                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3663                                    "cannot allocate resource memory");
3664                 return NULL;
3665         }
3666         memcpy(cache_resource, oentry, sizeof(*cache_resource));
3667         cache_resource->idx = idx;
3668         return &cache_resource->entry;
3669 }
3670
3671 void
3672 flow_dv_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3673 {
3674         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3675         struct mlx5_flow_dv_encap_decap_resource *res =
3676                                        container_of(entry, typeof(*res), entry);
3677
3678         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
3679 }
3680
3681 /**
3682  * Find existing encap/decap resource or create and register a new one.
3683  *
3684  * @param[in, out] dev
3685  *   Pointer to rte_eth_dev structure.
3686  * @param[in, out] resource
3687  *   Pointer to encap/decap resource.
3688  * @parm[in, out] dev_flow
3689  *   Pointer to the dev_flow.
3690  * @param[out] error
3691  *   pointer to error structure.
3692  *
3693  * @return
3694  *   0 on success otherwise -errno and errno is set.
3695  */
3696 static int
3697 flow_dv_encap_decap_resource_register
3698                         (struct rte_eth_dev *dev,
3699                          struct mlx5_flow_dv_encap_decap_resource *resource,
3700                          struct mlx5_flow *dev_flow,
3701                          struct rte_flow_error *error)
3702 {
3703         struct mlx5_priv *priv = dev->data->dev_private;
3704         struct mlx5_dev_ctx_shared *sh = priv->sh;
3705         struct mlx5_list_entry *entry;
3706         union {
3707                 struct {
3708                         uint32_t ft_type:8;
3709                         uint32_t refmt_type:8;
3710                         /*
3711                          * Header reformat actions can be shared between
3712                          * non-root tables. One bit to indicate non-root
3713                          * table or not.
3714                          */
3715                         uint32_t is_root:1;
3716                         uint32_t reserve:15;
3717                 };
3718                 uint32_t v32;
3719         } encap_decap_key = {
3720                 {
3721                         .ft_type = resource->ft_type,
3722                         .refmt_type = resource->reformat_type,
3723                         .is_root = !!dev_flow->dv.group,
3724                         .reserve = 0,
3725                 }
3726         };
3727         struct mlx5_flow_cb_ctx ctx = {
3728                 .error = error,
3729                 .data = resource,
3730         };
3731         struct mlx5_hlist *encaps_decaps;
3732         uint64_t key64;
3733
3734         encaps_decaps = flow_dv_hlist_prepare(sh, &sh->encaps_decaps,
3735                                 "encaps_decaps",
3736                                 MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
3737                                 true, true, sh,
3738                                 flow_dv_encap_decap_create_cb,
3739                                 flow_dv_encap_decap_match_cb,
3740                                 flow_dv_encap_decap_remove_cb,
3741                                 flow_dv_encap_decap_clone_cb,
3742                                 flow_dv_encap_decap_clone_free_cb,
3743                                 error);
3744         if (unlikely(!encaps_decaps))
3745                 return -rte_errno;
3746         resource->flags = dev_flow->dv.group ? 0 : 1;
3747         key64 =  __rte_raw_cksum(&encap_decap_key.v32,
3748                                  sizeof(encap_decap_key.v32), 0);
3749         if (resource->reformat_type !=
3750             MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 &&
3751             resource->size)
3752                 key64 = __rte_raw_cksum(resource->buf, resource->size, key64);
3753         entry = mlx5_hlist_register(encaps_decaps, key64, &ctx);
3754         if (!entry)
3755                 return -rte_errno;
3756         resource = container_of(entry, typeof(*resource), entry);
3757         dev_flow->dv.encap_decap = resource;
3758         dev_flow->handle->dvh.rix_encap_decap = resource->idx;
3759         return 0;
3760 }
3761
3762 /**
3763  * Find existing table jump resource or create and register a new one.
3764  *
3765  * @param[in, out] dev
3766  *   Pointer to rte_eth_dev structure.
3767  * @param[in, out] tbl
3768  *   Pointer to flow table resource.
3769  * @parm[in, out] dev_flow
3770  *   Pointer to the dev_flow.
3771  * @param[out] error
3772  *   pointer to error structure.
3773  *
3774  * @return
3775  *   0 on success otherwise -errno and errno is set.
3776  */
3777 static int
3778 flow_dv_jump_tbl_resource_register
3779                         (struct rte_eth_dev *dev __rte_unused,
3780                          struct mlx5_flow_tbl_resource *tbl,
3781                          struct mlx5_flow *dev_flow,
3782                          struct rte_flow_error *error __rte_unused)
3783 {
3784         struct mlx5_flow_tbl_data_entry *tbl_data =
3785                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
3786
3787         MLX5_ASSERT(tbl);
3788         MLX5_ASSERT(tbl_data->jump.action);
3789         dev_flow->handle->rix_jump = tbl_data->idx;
3790         dev_flow->dv.jump = &tbl_data->jump;
3791         return 0;
3792 }
3793
3794 int
3795 flow_dv_port_id_match_cb(void *tool_ctx __rte_unused,
3796                          struct mlx5_list_entry *entry, void *cb_ctx)
3797 {
3798         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3799         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3800         struct mlx5_flow_dv_port_id_action_resource *res =
3801                                        container_of(entry, typeof(*res), entry);
3802
3803         return ref->port_id != res->port_id;
3804 }
3805
3806 struct mlx5_list_entry *
3807 flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx)
3808 {
3809         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3810         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3811         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3812         struct mlx5_flow_dv_port_id_action_resource *resource;
3813         uint32_t idx;
3814         int ret;
3815
3816         /* Register new port id action resource. */
3817         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
3818         if (!resource) {
3819                 rte_flow_error_set(ctx->error, ENOMEM,
3820                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3821                                    "cannot allocate port_id action memory");
3822                 return NULL;
3823         }
3824         *resource = *ref;
3825         ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain,
3826                                                         ref->port_id,
3827                                                         &resource->action);
3828         if (ret) {
3829                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx);
3830                 rte_flow_error_set(ctx->error, ENOMEM,
3831                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3832                                    "cannot create action");
3833                 return NULL;
3834         }
3835         resource->idx = idx;
3836         return &resource->entry;
3837 }
3838
3839 struct mlx5_list_entry *
3840 flow_dv_port_id_clone_cb(void *tool_ctx,
3841                          struct mlx5_list_entry *entry __rte_unused,
3842                          void *cb_ctx)
3843 {
3844         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3845         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3846         struct mlx5_flow_dv_port_id_action_resource *resource;
3847         uint32_t idx;
3848
3849         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
3850         if (!resource) {
3851                 rte_flow_error_set(ctx->error, ENOMEM,
3852                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3853                                    "cannot allocate port_id action memory");
3854                 return NULL;
3855         }
3856         memcpy(resource, entry, sizeof(*resource));
3857         resource->idx = idx;
3858         return &resource->entry;
3859 }
3860
3861 void
3862 flow_dv_port_id_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3863 {
3864         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3865         struct mlx5_flow_dv_port_id_action_resource *resource =
3866                                   container_of(entry, typeof(*resource), entry);
3867
3868         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
3869 }
3870
3871 /**
3872  * Find existing table port ID resource or create and register a new one.
3873  *
3874  * @param[in, out] dev
3875  *   Pointer to rte_eth_dev structure.
3876  * @param[in, out] ref
3877  *   Pointer to port ID action resource reference.
3878  * @parm[in, out] dev_flow
3879  *   Pointer to the dev_flow.
3880  * @param[out] error
3881  *   pointer to error structure.
3882  *
3883  * @return
3884  *   0 on success otherwise -errno and errno is set.
3885  */
3886 static int
3887 flow_dv_port_id_action_resource_register
3888                         (struct rte_eth_dev *dev,
3889                          struct mlx5_flow_dv_port_id_action_resource *ref,
3890                          struct mlx5_flow *dev_flow,
3891                          struct rte_flow_error *error)
3892 {
3893         struct mlx5_priv *priv = dev->data->dev_private;
3894         struct mlx5_list_entry *entry;
3895         struct mlx5_flow_dv_port_id_action_resource *resource;
3896         struct mlx5_flow_cb_ctx ctx = {
3897                 .error = error,
3898                 .data = ref,
3899         };
3900
3901         entry = mlx5_list_register(priv->sh->port_id_action_list, &ctx);
3902         if (!entry)
3903                 return -rte_errno;
3904         resource = container_of(entry, typeof(*resource), entry);
3905         dev_flow->dv.port_id_action = resource;
3906         dev_flow->handle->rix_port_id_action = resource->idx;
3907         return 0;
3908 }
3909
3910 int
3911 flow_dv_push_vlan_match_cb(void *tool_ctx __rte_unused,
3912                            struct mlx5_list_entry *entry, void *cb_ctx)
3913 {
3914         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3915         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3916         struct mlx5_flow_dv_push_vlan_action_resource *res =
3917                                        container_of(entry, typeof(*res), entry);
3918
3919         return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
3920 }
3921
3922 struct mlx5_list_entry *
3923 flow_dv_push_vlan_create_cb(void *tool_ctx, void *cb_ctx)
3924 {
3925         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3926         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3927         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3928         struct mlx5_flow_dv_push_vlan_action_resource *resource;
3929         struct mlx5dv_dr_domain *domain;
3930         uint32_t idx;
3931         int ret;
3932
3933         /* Register new port id action resource. */
3934         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
3935         if (!resource) {
3936                 rte_flow_error_set(ctx->error, ENOMEM,
3937                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3938                                    "cannot allocate push_vlan action memory");
3939                 return NULL;
3940         }
3941         *resource = *ref;
3942         if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3943                 domain = sh->fdb_domain;
3944         else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3945                 domain = sh->rx_domain;
3946         else
3947                 domain = sh->tx_domain;
3948         ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag,
3949                                                         &resource->action);
3950         if (ret) {
3951                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
3952                 rte_flow_error_set(ctx->error, ENOMEM,
3953                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3954                                    "cannot create push vlan action");
3955                 return NULL;
3956         }
3957         resource->idx = idx;
3958         return &resource->entry;
3959 }
3960
3961 struct mlx5_list_entry *
3962 flow_dv_push_vlan_clone_cb(void *tool_ctx,
3963                            struct mlx5_list_entry *entry __rte_unused,
3964                            void *cb_ctx)
3965 {
3966         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3967         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3968         struct mlx5_flow_dv_push_vlan_action_resource *resource;
3969         uint32_t idx;
3970
3971         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
3972         if (!resource) {
3973                 rte_flow_error_set(ctx->error, ENOMEM,
3974                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3975                                    "cannot allocate push_vlan action memory");
3976                 return NULL;
3977         }
3978         memcpy(resource, entry, sizeof(*resource));
3979         resource->idx = idx;
3980         return &resource->entry;
3981 }
3982
3983 void
3984 flow_dv_push_vlan_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3985 {
3986         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3987         struct mlx5_flow_dv_push_vlan_action_resource *resource =
3988                                   container_of(entry, typeof(*resource), entry);
3989
3990         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
3991 }
3992
3993 /**
3994  * Find existing push vlan resource or create and register a new one.
3995  *
3996  * @param [in, out] dev
3997  *   Pointer to rte_eth_dev structure.
3998  * @param[in, out] ref
3999  *   Pointer to port ID action resource reference.
4000  * @parm[in, out] dev_flow
4001  *   Pointer to the dev_flow.
4002  * @param[out] error
4003  *   pointer to error structure.
4004  *
4005  * @return
4006  *   0 on success otherwise -errno and errno is set.
4007  */
4008 static int
4009 flow_dv_push_vlan_action_resource_register
4010                        (struct rte_eth_dev *dev,
4011                         struct mlx5_flow_dv_push_vlan_action_resource *ref,
4012                         struct mlx5_flow *dev_flow,
4013                         struct rte_flow_error *error)
4014 {
4015         struct mlx5_priv *priv = dev->data->dev_private;
4016         struct mlx5_flow_dv_push_vlan_action_resource *resource;
4017         struct mlx5_list_entry *entry;
4018         struct mlx5_flow_cb_ctx ctx = {
4019                 .error = error,
4020                 .data = ref,
4021         };
4022
4023         entry = mlx5_list_register(priv->sh->push_vlan_action_list, &ctx);
4024         if (!entry)
4025                 return -rte_errno;
4026         resource = container_of(entry, typeof(*resource), entry);
4027
4028         dev_flow->handle->dvh.rix_push_vlan = resource->idx;
4029         dev_flow->dv.push_vlan_res = resource;
4030         return 0;
4031 }
4032
4033 /**
4034  * Get the size of specific rte_flow_item_type hdr size
4035  *
4036  * @param[in] item_type
4037  *   Tested rte_flow_item_type.
4038  *
4039  * @return
4040  *   sizeof struct item_type, 0 if void or irrelevant.
4041  */
4042 size_t
4043 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
4044 {
4045         size_t retval;
4046
4047         switch (item_type) {
4048         case RTE_FLOW_ITEM_TYPE_ETH:
4049                 retval = sizeof(struct rte_ether_hdr);
4050                 break;
4051         case RTE_FLOW_ITEM_TYPE_VLAN:
4052                 retval = sizeof(struct rte_vlan_hdr);
4053                 break;
4054         case RTE_FLOW_ITEM_TYPE_IPV4:
4055                 retval = sizeof(struct rte_ipv4_hdr);
4056                 break;
4057         case RTE_FLOW_ITEM_TYPE_IPV6:
4058                 retval = sizeof(struct rte_ipv6_hdr);
4059                 break;
4060         case RTE_FLOW_ITEM_TYPE_UDP:
4061                 retval = sizeof(struct rte_udp_hdr);
4062                 break;
4063         case RTE_FLOW_ITEM_TYPE_TCP:
4064                 retval = sizeof(struct rte_tcp_hdr);
4065                 break;
4066         case RTE_FLOW_ITEM_TYPE_VXLAN:
4067         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4068                 retval = sizeof(struct rte_vxlan_hdr);
4069                 break;
4070         case RTE_FLOW_ITEM_TYPE_GRE:
4071         case RTE_FLOW_ITEM_TYPE_NVGRE:
4072                 retval = sizeof(struct rte_gre_hdr);
4073                 break;
4074         case RTE_FLOW_ITEM_TYPE_MPLS:
4075                 retval = sizeof(struct rte_mpls_hdr);
4076                 break;
4077         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
4078         default:
4079                 retval = 0;
4080                 break;
4081         }
4082         return retval;
4083 }
4084
4085 #define MLX5_ENCAP_IPV4_VERSION         0x40
4086 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
4087 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
4088 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
4089 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
4090 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
4091 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
4092
4093 /**
4094  * Convert the encap action data from list of rte_flow_item to raw buffer
4095  *
4096  * @param[in] items
4097  *   Pointer to rte_flow_item objects list.
4098  * @param[out] buf
4099  *   Pointer to the output buffer.
4100  * @param[out] size
4101  *   Pointer to the output buffer size.
4102  * @param[out] error
4103  *   Pointer to the error structure.
4104  *
4105  * @return
4106  *   0 on success, a negative errno value otherwise and rte_errno is set.
4107  */
4108 int
4109 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
4110                            size_t *size, struct rte_flow_error *error)
4111 {
4112         struct rte_ether_hdr *eth = NULL;
4113         struct rte_vlan_hdr *vlan = NULL;
4114         struct rte_ipv4_hdr *ipv4 = NULL;
4115         struct rte_ipv6_hdr *ipv6 = NULL;
4116         struct rte_udp_hdr *udp = NULL;
4117         struct rte_vxlan_hdr *vxlan = NULL;
4118         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
4119         struct rte_gre_hdr *gre = NULL;
4120         size_t len;
4121         size_t temp_size = 0;
4122
4123         if (!items)
4124                 return rte_flow_error_set(error, EINVAL,
4125                                           RTE_FLOW_ERROR_TYPE_ACTION,
4126                                           NULL, "invalid empty data");
4127         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4128                 len = flow_dv_get_item_hdr_len(items->type);
4129                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
4130                         return rte_flow_error_set(error, EINVAL,
4131                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4132                                                   (void *)items->type,
4133                                                   "items total size is too big"
4134                                                   " for encap action");
4135                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
4136                 switch (items->type) {
4137                 case RTE_FLOW_ITEM_TYPE_ETH:
4138                         eth = (struct rte_ether_hdr *)&buf[temp_size];
4139                         break;
4140                 case RTE_FLOW_ITEM_TYPE_VLAN:
4141                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
4142                         if (!eth)
4143                                 return rte_flow_error_set(error, EINVAL,
4144                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4145                                                 (void *)items->type,
4146                                                 "eth header not found");
4147                         if (!eth->ether_type)
4148                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
4149                         break;
4150                 case RTE_FLOW_ITEM_TYPE_IPV4:
4151                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
4152                         if (!vlan && !eth)
4153                                 return rte_flow_error_set(error, EINVAL,
4154                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4155                                                 (void *)items->type,
4156                                                 "neither eth nor vlan"
4157                                                 " header found");
4158                         if (vlan && !vlan->eth_proto)
4159                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4160                         else if (eth && !eth->ether_type)
4161                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4162                         if (!ipv4->version_ihl)
4163                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
4164                                                     MLX5_ENCAP_IPV4_IHL_MIN;
4165                         if (!ipv4->time_to_live)
4166                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
4167                         break;
4168                 case RTE_FLOW_ITEM_TYPE_IPV6:
4169                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
4170                         if (!vlan && !eth)
4171                                 return rte_flow_error_set(error, EINVAL,
4172                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4173                                                 (void *)items->type,
4174                                                 "neither eth nor vlan"
4175                                                 " header found");
4176                         if (vlan && !vlan->eth_proto)
4177                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4178                         else if (eth && !eth->ether_type)
4179                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4180                         if (!ipv6->vtc_flow)
4181                                 ipv6->vtc_flow =
4182                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
4183                         if (!ipv6->hop_limits)
4184                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
4185                         break;
4186                 case RTE_FLOW_ITEM_TYPE_UDP:
4187                         udp = (struct rte_udp_hdr *)&buf[temp_size];
4188                         if (!ipv4 && !ipv6)
4189                                 return rte_flow_error_set(error, EINVAL,
4190                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4191                                                 (void *)items->type,
4192                                                 "ip header not found");
4193                         if (ipv4 && !ipv4->next_proto_id)
4194                                 ipv4->next_proto_id = IPPROTO_UDP;
4195                         else if (ipv6 && !ipv6->proto)
4196                                 ipv6->proto = IPPROTO_UDP;
4197                         break;
4198                 case RTE_FLOW_ITEM_TYPE_VXLAN:
4199                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
4200                         if (!udp)
4201                                 return rte_flow_error_set(error, EINVAL,
4202                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4203                                                 (void *)items->type,
4204                                                 "udp header not found");
4205                         if (!udp->dst_port)
4206                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
4207                         if (!vxlan->vx_flags)
4208                                 vxlan->vx_flags =
4209                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
4210                         break;
4211                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4212                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
4213                         if (!udp)
4214                                 return rte_flow_error_set(error, EINVAL,
4215                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4216                                                 (void *)items->type,
4217                                                 "udp header not found");
4218                         if (!vxlan_gpe->proto)
4219                                 return rte_flow_error_set(error, EINVAL,
4220                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4221                                                 (void *)items->type,
4222                                                 "next protocol not found");
4223                         if (!udp->dst_port)
4224                                 udp->dst_port =
4225                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
4226                         if (!vxlan_gpe->vx_flags)
4227                                 vxlan_gpe->vx_flags =
4228                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
4229                         break;
4230                 case RTE_FLOW_ITEM_TYPE_GRE:
4231                 case RTE_FLOW_ITEM_TYPE_NVGRE:
4232                         gre = (struct rte_gre_hdr *)&buf[temp_size];
4233                         if (!gre->proto)
4234                                 return rte_flow_error_set(error, EINVAL,
4235                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4236                                                 (void *)items->type,
4237                                                 "next protocol not found");
4238                         if (!ipv4 && !ipv6)
4239                                 return rte_flow_error_set(error, EINVAL,
4240                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4241                                                 (void *)items->type,
4242                                                 "ip header not found");
4243                         if (ipv4 && !ipv4->next_proto_id)
4244                                 ipv4->next_proto_id = IPPROTO_GRE;
4245                         else if (ipv6 && !ipv6->proto)
4246                                 ipv6->proto = IPPROTO_GRE;
4247                         break;
4248                 case RTE_FLOW_ITEM_TYPE_VOID:
4249                         break;
4250                 default:
4251                         return rte_flow_error_set(error, EINVAL,
4252                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4253                                                   (void *)items->type,
4254                                                   "unsupported item type");
4255                         break;
4256                 }
4257                 temp_size += len;
4258         }
4259         *size = temp_size;
4260         return 0;
4261 }
4262
4263 static int
4264 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
4265 {
4266         struct rte_ether_hdr *eth = NULL;
4267         struct rte_vlan_hdr *vlan = NULL;
4268         struct rte_ipv6_hdr *ipv6 = NULL;
4269         struct rte_udp_hdr *udp = NULL;
4270         char *next_hdr;
4271         uint16_t proto;
4272
4273         eth = (struct rte_ether_hdr *)data;
4274         next_hdr = (char *)(eth + 1);
4275         proto = RTE_BE16(eth->ether_type);
4276
4277         /* VLAN skipping */
4278         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
4279                 vlan = (struct rte_vlan_hdr *)next_hdr;
4280                 proto = RTE_BE16(vlan->eth_proto);
4281                 next_hdr += sizeof(struct rte_vlan_hdr);
4282         }
4283
4284         /* HW calculates IPv4 csum. no need to proceed */
4285         if (proto == RTE_ETHER_TYPE_IPV4)
4286                 return 0;
4287
4288         /* non IPv4/IPv6 header. not supported */
4289         if (proto != RTE_ETHER_TYPE_IPV6) {
4290                 return rte_flow_error_set(error, ENOTSUP,
4291                                           RTE_FLOW_ERROR_TYPE_ACTION,
4292                                           NULL, "Cannot offload non IPv4/IPv6");
4293         }
4294
4295         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
4296
4297         /* ignore non UDP */
4298         if (ipv6->proto != IPPROTO_UDP)
4299                 return 0;
4300
4301         udp = (struct rte_udp_hdr *)(ipv6 + 1);
4302         udp->dgram_cksum = 0;
4303
4304         return 0;
4305 }
4306
4307 /**
4308  * Convert L2 encap action to DV specification.
4309  *
4310  * @param[in] dev
4311  *   Pointer to rte_eth_dev structure.
4312  * @param[in] action
4313  *   Pointer to action structure.
4314  * @param[in, out] dev_flow
4315  *   Pointer to the mlx5_flow.
4316  * @param[in] transfer
4317  *   Mark if the flow is E-Switch flow.
4318  * @param[out] error
4319  *   Pointer to the error structure.
4320  *
4321  * @return
4322  *   0 on success, a negative errno value otherwise and rte_errno is set.
4323  */
4324 static int
4325 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
4326                                const struct rte_flow_action *action,
4327                                struct mlx5_flow *dev_flow,
4328                                uint8_t transfer,
4329                                struct rte_flow_error *error)
4330 {
4331         const struct rte_flow_item *encap_data;
4332         const struct rte_flow_action_raw_encap *raw_encap_data;
4333         struct mlx5_flow_dv_encap_decap_resource res = {
4334                 .reformat_type =
4335                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
4336                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4337                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
4338         };
4339
4340         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
4341                 raw_encap_data =
4342                         (const struct rte_flow_action_raw_encap *)action->conf;
4343                 res.size = raw_encap_data->size;
4344                 memcpy(res.buf, raw_encap_data->data, res.size);
4345         } else {
4346                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
4347                         encap_data =
4348                                 ((const struct rte_flow_action_vxlan_encap *)
4349                                                 action->conf)->definition;
4350                 else
4351                         encap_data =
4352                                 ((const struct rte_flow_action_nvgre_encap *)
4353                                                 action->conf)->definition;
4354                 if (flow_dv_convert_encap_data(encap_data, res.buf,
4355                                                &res.size, error))
4356                         return -rte_errno;
4357         }
4358         if (flow_dv_zero_encap_udp_csum(res.buf, error))
4359                 return -rte_errno;
4360         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4361                 return rte_flow_error_set(error, EINVAL,
4362                                           RTE_FLOW_ERROR_TYPE_ACTION,
4363                                           NULL, "can't create L2 encap action");
4364         return 0;
4365 }
4366
4367 /**
4368  * Convert L2 decap action to DV specification.
4369  *
4370  * @param[in] dev
4371  *   Pointer to rte_eth_dev structure.
4372  * @param[in, out] dev_flow
4373  *   Pointer to the mlx5_flow.
4374  * @param[in] transfer
4375  *   Mark if the flow is E-Switch flow.
4376  * @param[out] error
4377  *   Pointer to the error structure.
4378  *
4379  * @return
4380  *   0 on success, a negative errno value otherwise and rte_errno is set.
4381  */
4382 static int
4383 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
4384                                struct mlx5_flow *dev_flow,
4385                                uint8_t transfer,
4386                                struct rte_flow_error *error)
4387 {
4388         struct mlx5_flow_dv_encap_decap_resource res = {
4389                 .size = 0,
4390                 .reformat_type =
4391                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
4392                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4393                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
4394         };
4395
4396         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4397                 return rte_flow_error_set(error, EINVAL,
4398                                           RTE_FLOW_ERROR_TYPE_ACTION,
4399                                           NULL, "can't create L2 decap action");
4400         return 0;
4401 }
4402
4403 /**
4404  * Convert raw decap/encap (L3 tunnel) action to DV specification.
4405  *
4406  * @param[in] dev
4407  *   Pointer to rte_eth_dev structure.
4408  * @param[in] action
4409  *   Pointer to action structure.
4410  * @param[in, out] dev_flow
4411  *   Pointer to the mlx5_flow.
4412  * @param[in] attr
4413  *   Pointer to the flow attributes.
4414  * @param[out] error
4415  *   Pointer to the error structure.
4416  *
4417  * @return
4418  *   0 on success, a negative errno value otherwise and rte_errno is set.
4419  */
4420 static int
4421 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
4422                                 const struct rte_flow_action *action,
4423                                 struct mlx5_flow *dev_flow,
4424                                 const struct rte_flow_attr *attr,
4425                                 struct rte_flow_error *error)
4426 {
4427         const struct rte_flow_action_raw_encap *encap_data;
4428         struct mlx5_flow_dv_encap_decap_resource res;
4429
4430         memset(&res, 0, sizeof(res));
4431         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
4432         res.size = encap_data->size;
4433         memcpy(res.buf, encap_data->data, res.size);
4434         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
4435                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
4436                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
4437         if (attr->transfer)
4438                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4439         else
4440                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4441                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4442         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4443                 return rte_flow_error_set(error, EINVAL,
4444                                           RTE_FLOW_ERROR_TYPE_ACTION,
4445                                           NULL, "can't create encap action");
4446         return 0;
4447 }
4448
4449 /**
4450  * Create action push VLAN.
4451  *
4452  * @param[in] dev
4453  *   Pointer to rte_eth_dev structure.
4454  * @param[in] attr
4455  *   Pointer to the flow attributes.
4456  * @param[in] vlan
4457  *   Pointer to the vlan to push to the Ethernet header.
4458  * @param[in, out] dev_flow
4459  *   Pointer to the mlx5_flow.
4460  * @param[out] error
4461  *   Pointer to the error structure.
4462  *
4463  * @return
4464  *   0 on success, a negative errno value otherwise and rte_errno is set.
4465  */
4466 static int
4467 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
4468                                 const struct rte_flow_attr *attr,
4469                                 const struct rte_vlan_hdr *vlan,
4470                                 struct mlx5_flow *dev_flow,
4471                                 struct rte_flow_error *error)
4472 {
4473         struct mlx5_flow_dv_push_vlan_action_resource res;
4474
4475         memset(&res, 0, sizeof(res));
4476         res.vlan_tag =
4477                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
4478                                  vlan->vlan_tci);
4479         if (attr->transfer)
4480                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4481         else
4482                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4483                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4484         return flow_dv_push_vlan_action_resource_register
4485                                             (dev, &res, dev_flow, error);
4486 }
4487
4488 /**
4489  * Validate the modify-header actions.
4490  *
4491  * @param[in] action_flags
4492  *   Holds the actions detected until now.
4493  * @param[in] action
4494  *   Pointer to the modify action.
4495  * @param[out] error
4496  *   Pointer to error structure.
4497  *
4498  * @return
4499  *   0 on success, a negative errno value otherwise and rte_errno is set.
4500  */
4501 static int
4502 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
4503                                    const struct rte_flow_action *action,
4504                                    struct rte_flow_error *error)
4505 {
4506         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
4507                 return rte_flow_error_set(error, EINVAL,
4508                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4509                                           NULL, "action configuration not set");
4510         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
4511                 return rte_flow_error_set(error, EINVAL,
4512                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4513                                           "can't have encap action before"
4514                                           " modify action");
4515         return 0;
4516 }
4517
4518 /**
4519  * Validate the modify-header MAC address actions.
4520  *
4521  * @param[in] action_flags
4522  *   Holds the actions detected until now.
4523  * @param[in] action
4524  *   Pointer to the modify action.
4525  * @param[in] item_flags
4526  *   Holds the items detected.
4527  * @param[out] error
4528  *   Pointer to error structure.
4529  *
4530  * @return
4531  *   0 on success, a negative errno value otherwise and rte_errno is set.
4532  */
4533 static int
4534 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
4535                                    const struct rte_flow_action *action,
4536                                    const uint64_t item_flags,
4537                                    struct rte_flow_error *error)
4538 {
4539         int ret = 0;
4540
4541         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4542         if (!ret) {
4543                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
4544                         return rte_flow_error_set(error, EINVAL,
4545                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4546                                                   NULL,
4547                                                   "no L2 item in pattern");
4548         }
4549         return ret;
4550 }
4551
4552 /**
4553  * Validate the modify-header IPv4 address actions.
4554  *
4555  * @param[in] action_flags
4556  *   Holds the actions detected until now.
4557  * @param[in] action
4558  *   Pointer to the modify action.
4559  * @param[in] item_flags
4560  *   Holds the items detected.
4561  * @param[out] error
4562  *   Pointer to error structure.
4563  *
4564  * @return
4565  *   0 on success, a negative errno value otherwise and rte_errno is set.
4566  */
4567 static int
4568 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
4569                                     const struct rte_flow_action *action,
4570                                     const uint64_t item_flags,
4571                                     struct rte_flow_error *error)
4572 {
4573         int ret = 0;
4574         uint64_t layer;
4575
4576         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4577         if (!ret) {
4578                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4579                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
4580                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
4581                 if (!(item_flags & layer))
4582                         return rte_flow_error_set(error, EINVAL,
4583                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4584                                                   NULL,
4585                                                   "no ipv4 item in pattern");
4586         }
4587         return ret;
4588 }
4589
4590 /**
4591  * Validate the modify-header IPv6 address actions.
4592  *
4593  * @param[in] action_flags
4594  *   Holds the actions detected until now.
4595  * @param[in] action
4596  *   Pointer to the modify action.
4597  * @param[in] item_flags
4598  *   Holds the items detected.
4599  * @param[out] error
4600  *   Pointer to error structure.
4601  *
4602  * @return
4603  *   0 on success, a negative errno value otherwise and rte_errno is set.
4604  */
4605 static int
4606 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
4607                                     const struct rte_flow_action *action,
4608                                     const uint64_t item_flags,
4609                                     struct rte_flow_error *error)
4610 {
4611         int ret = 0;
4612         uint64_t layer;
4613
4614         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4615         if (!ret) {
4616                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4617                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
4618                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
4619                 if (!(item_flags & layer))
4620                         return rte_flow_error_set(error, EINVAL,
4621                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4622                                                   NULL,
4623                                                   "no ipv6 item in pattern");
4624         }
4625         return ret;
4626 }
4627
4628 /**
4629  * Validate the modify-header TP actions.
4630  *
4631  * @param[in] action_flags
4632  *   Holds the actions detected until now.
4633  * @param[in] action
4634  *   Pointer to the modify action.
4635  * @param[in] item_flags
4636  *   Holds the items detected.
4637  * @param[out] error
4638  *   Pointer to error structure.
4639  *
4640  * @return
4641  *   0 on success, a negative errno value otherwise and rte_errno is set.
4642  */
4643 static int
4644 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
4645                                   const struct rte_flow_action *action,
4646                                   const uint64_t item_flags,
4647                                   struct rte_flow_error *error)
4648 {
4649         int ret = 0;
4650         uint64_t layer;
4651
4652         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4653         if (!ret) {
4654                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4655                                  MLX5_FLOW_LAYER_INNER_L4 :
4656                                  MLX5_FLOW_LAYER_OUTER_L4;
4657                 if (!(item_flags & layer))
4658                         return rte_flow_error_set(error, EINVAL,
4659                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4660                                                   NULL, "no transport layer "
4661                                                   "in pattern");
4662         }
4663         return ret;
4664 }
4665
4666 /**
4667  * Validate the modify-header actions of increment/decrement
4668  * TCP Sequence-number.
4669  *
4670  * @param[in] action_flags
4671  *   Holds the actions detected until now.
4672  * @param[in] action
4673  *   Pointer to the modify action.
4674  * @param[in] item_flags
4675  *   Holds the items detected.
4676  * @param[out] error
4677  *   Pointer to error structure.
4678  *
4679  * @return
4680  *   0 on success, a negative errno value otherwise and rte_errno is set.
4681  */
4682 static int
4683 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
4684                                        const struct rte_flow_action *action,
4685                                        const uint64_t item_flags,
4686                                        struct rte_flow_error *error)
4687 {
4688         int ret = 0;
4689         uint64_t layer;
4690
4691         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4692         if (!ret) {
4693                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4694                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4695                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4696                 if (!(item_flags & layer))
4697                         return rte_flow_error_set(error, EINVAL,
4698                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4699                                                   NULL, "no TCP item in"
4700                                                   " pattern");
4701                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
4702                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
4703                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
4704                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
4705                         return rte_flow_error_set(error, EINVAL,
4706                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4707                                                   NULL,
4708                                                   "cannot decrease and increase"
4709                                                   " TCP sequence number"
4710                                                   " at the same time");
4711         }
4712         return ret;
4713 }
4714
4715 /**
4716  * Validate the modify-header actions of increment/decrement
4717  * TCP Acknowledgment number.
4718  *
4719  * @param[in] action_flags
4720  *   Holds the actions detected until now.
4721  * @param[in] action
4722  *   Pointer to the modify action.
4723  * @param[in] item_flags
4724  *   Holds the items detected.
4725  * @param[out] error
4726  *   Pointer to error structure.
4727  *
4728  * @return
4729  *   0 on success, a negative errno value otherwise and rte_errno is set.
4730  */
4731 static int
4732 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
4733                                        const struct rte_flow_action *action,
4734                                        const uint64_t item_flags,
4735                                        struct rte_flow_error *error)
4736 {
4737         int ret = 0;
4738         uint64_t layer;
4739
4740         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4741         if (!ret) {
4742                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4743                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4744                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4745                 if (!(item_flags & layer))
4746                         return rte_flow_error_set(error, EINVAL,
4747                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4748                                                   NULL, "no TCP item in"
4749                                                   " pattern");
4750                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
4751                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
4752                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
4753                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
4754                         return rte_flow_error_set(error, EINVAL,
4755                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4756                                                   NULL,
4757                                                   "cannot decrease and increase"
4758                                                   " TCP acknowledgment number"
4759                                                   " at the same time");
4760         }
4761         return ret;
4762 }
4763
4764 /**
4765  * Validate the modify-header TTL actions.
4766  *
4767  * @param[in] action_flags
4768  *   Holds the actions detected until now.
4769  * @param[in] action
4770  *   Pointer to the modify action.
4771  * @param[in] item_flags
4772  *   Holds the items detected.
4773  * @param[out] error
4774  *   Pointer to error structure.
4775  *
4776  * @return
4777  *   0 on success, a negative errno value otherwise and rte_errno is set.
4778  */
4779 static int
4780 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
4781                                    const struct rte_flow_action *action,
4782                                    const uint64_t item_flags,
4783                                    struct rte_flow_error *error)
4784 {
4785         int ret = 0;
4786         uint64_t layer;
4787
4788         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4789         if (!ret) {
4790                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4791                                  MLX5_FLOW_LAYER_INNER_L3 :
4792                                  MLX5_FLOW_LAYER_OUTER_L3;
4793                 if (!(item_flags & layer))
4794                         return rte_flow_error_set(error, EINVAL,
4795                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4796                                                   NULL,
4797                                                   "no IP protocol in pattern");
4798         }
4799         return ret;
4800 }
4801
4802 /**
4803  * Validate the generic modify field actions.
4804  * @param[in] dev
4805  *   Pointer to the rte_eth_dev structure.
4806  * @param[in] action_flags
4807  *   Holds the actions detected until now.
4808  * @param[in] action
4809  *   Pointer to the modify action.
4810  * @param[in] attr
4811  *   Pointer to the flow attributes.
4812  * @param[out] error
4813  *   Pointer to error structure.
4814  *
4815  * @return
4816  *   Number of header fields to modify (0 or more) on success,
4817  *   a negative errno value otherwise and rte_errno is set.
4818  */
4819 static int
4820 flow_dv_validate_action_modify_field(struct rte_eth_dev *dev,
4821                                    const uint64_t action_flags,
4822                                    const struct rte_flow_action *action,
4823                                    const struct rte_flow_attr *attr,
4824                                    struct rte_flow_error *error)
4825 {
4826         int ret = 0;
4827         struct mlx5_priv *priv = dev->data->dev_private;
4828         struct mlx5_sh_config *config = &priv->sh->config;
4829         const struct rte_flow_action_modify_field *action_modify_field =
4830                 action->conf;
4831         uint32_t dst_width = mlx5_flow_item_field_width(dev,
4832                                 action_modify_field->dst.field,
4833                                 -1, attr, error);
4834         uint32_t src_width = mlx5_flow_item_field_width(dev,
4835                                 action_modify_field->src.field,
4836                                 dst_width, attr, error);
4837
4838         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4839         if (ret)
4840                 return ret;
4841
4842         if (action_modify_field->width == 0)
4843                 return rte_flow_error_set(error, EINVAL,
4844                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4845                                 "no bits are requested to be modified");
4846         else if (action_modify_field->width > dst_width ||
4847                  action_modify_field->width > src_width)
4848                 return rte_flow_error_set(error, EINVAL,
4849                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4850                                 "cannot modify more bits than"
4851                                 " the width of a field");
4852         if (action_modify_field->dst.field != RTE_FLOW_FIELD_VALUE &&
4853             action_modify_field->dst.field != RTE_FLOW_FIELD_POINTER) {
4854                 if ((action_modify_field->dst.offset +
4855                      action_modify_field->width > dst_width) ||
4856                     (action_modify_field->dst.offset % 32))
4857                         return rte_flow_error_set(error, EINVAL,
4858                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4859                                         "destination offset is too big"
4860                                         " or not aligned to 4 bytes");
4861                 if (action_modify_field->dst.level &&
4862                     action_modify_field->dst.field != RTE_FLOW_FIELD_TAG)
4863                         return rte_flow_error_set(error, ENOTSUP,
4864                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4865                                         "inner header fields modification"
4866                                         " is not supported");
4867         }
4868         if (action_modify_field->src.field != RTE_FLOW_FIELD_VALUE &&
4869             action_modify_field->src.field != RTE_FLOW_FIELD_POINTER) {
4870                 if (!attr->transfer && !attr->group)
4871                         return rte_flow_error_set(error, ENOTSUP,
4872                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4873                                         "modify field action is not"
4874                                         " supported for group 0");
4875                 if ((action_modify_field->src.offset +
4876                      action_modify_field->width > src_width) ||
4877                     (action_modify_field->src.offset % 32))
4878                         return rte_flow_error_set(error, EINVAL,
4879                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4880                                         "source offset is too big"
4881                                         " or not aligned to 4 bytes");
4882                 if (action_modify_field->src.level &&
4883                     action_modify_field->src.field != RTE_FLOW_FIELD_TAG)
4884                         return rte_flow_error_set(error, ENOTSUP,
4885                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4886                                         "inner header fields modification"
4887                                         " is not supported");
4888         }
4889         if ((action_modify_field->dst.field ==
4890              action_modify_field->src.field) &&
4891             (action_modify_field->dst.level ==
4892              action_modify_field->src.level))
4893                 return rte_flow_error_set(error, EINVAL,
4894                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4895                                 "source and destination fields"
4896                                 " cannot be the same");
4897         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VALUE ||
4898             action_modify_field->dst.field == RTE_FLOW_FIELD_POINTER ||
4899             action_modify_field->dst.field == RTE_FLOW_FIELD_MARK)
4900                 return rte_flow_error_set(error, EINVAL,
4901                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4902                                 "mark, immediate value or a pointer to it"
4903                                 " cannot be used as a destination");
4904         if (action_modify_field->dst.field == RTE_FLOW_FIELD_START ||
4905             action_modify_field->src.field == RTE_FLOW_FIELD_START)
4906                 return rte_flow_error_set(error, ENOTSUP,
4907                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4908                                 "modifications of an arbitrary"
4909                                 " place in a packet is not supported");
4910         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VLAN_TYPE ||
4911             action_modify_field->src.field == RTE_FLOW_FIELD_VLAN_TYPE)
4912                 return rte_flow_error_set(error, ENOTSUP,
4913                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4914                                 "modifications of the 802.1Q Tag"
4915                                 " Identifier is not supported");
4916         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VXLAN_VNI ||
4917             action_modify_field->src.field == RTE_FLOW_FIELD_VXLAN_VNI)
4918                 return rte_flow_error_set(error, ENOTSUP,
4919                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4920                                 "modifications of the VXLAN Network"
4921                                 " Identifier is not supported");
4922         if (action_modify_field->dst.field == RTE_FLOW_FIELD_GENEVE_VNI ||
4923             action_modify_field->src.field == RTE_FLOW_FIELD_GENEVE_VNI)
4924                 return rte_flow_error_set(error, ENOTSUP,
4925                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4926                                 "modifications of the GENEVE Network"
4927                                 " Identifier is not supported");
4928         if (action_modify_field->dst.field == RTE_FLOW_FIELD_MARK ||
4929             action_modify_field->src.field == RTE_FLOW_FIELD_MARK)
4930                 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4931                     !mlx5_flow_ext_mreg_supported(dev))
4932                         return rte_flow_error_set(error, ENOTSUP,
4933                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4934                                         "cannot modify mark in legacy mode"
4935                                         " or without extensive registers");
4936         if (action_modify_field->dst.field == RTE_FLOW_FIELD_META ||
4937             action_modify_field->src.field == RTE_FLOW_FIELD_META) {
4938                 if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
4939                     !mlx5_flow_ext_mreg_supported(dev))
4940                         return rte_flow_error_set(error, ENOTSUP,
4941                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4942                                         "cannot modify meta without"
4943                                         " extensive registers support");
4944                 ret = flow_dv_get_metadata_reg(dev, attr, error);
4945                 if (ret < 0 || ret == REG_NON)
4946                         return rte_flow_error_set(error, ENOTSUP,
4947                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4948                                         "cannot modify meta without"
4949                                         " extensive registers available");
4950         }
4951         if (action_modify_field->operation != RTE_FLOW_MODIFY_SET)
4952                 return rte_flow_error_set(error, ENOTSUP,
4953                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4954                                 "add and sub operations"
4955                                 " are not supported");
4956         return (action_modify_field->width / 32) +
4957                !!(action_modify_field->width % 32);
4958 }
4959
4960 /**
4961  * Validate jump action.
4962  *
4963  * @param[in] action
4964  *   Pointer to the jump action.
4965  * @param[in] action_flags
4966  *   Holds the actions detected until now.
4967  * @param[in] attributes
4968  *   Pointer to flow attributes
4969  * @param[in] external
4970  *   Action belongs to flow rule created by request external to PMD.
4971  * @param[out] error
4972  *   Pointer to error structure.
4973  *
4974  * @return
4975  *   0 on success, a negative errno value otherwise and rte_errno is set.
4976  */
4977 static int
4978 flow_dv_validate_action_jump(struct rte_eth_dev *dev,
4979                              const struct mlx5_flow_tunnel *tunnel,
4980                              const struct rte_flow_action *action,
4981                              uint64_t action_flags,
4982                              const struct rte_flow_attr *attributes,
4983                              bool external, struct rte_flow_error *error)
4984 {
4985         uint32_t target_group, table = 0;
4986         int ret = 0;
4987         struct flow_grp_info grp_info = {
4988                 .external = !!external,
4989                 .transfer = !!attributes->transfer,
4990                 .fdb_def_rule = 1,
4991                 .std_tbl_fix = 0
4992         };
4993         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
4994                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4995                 return rte_flow_error_set(error, EINVAL,
4996                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4997                                           "can't have 2 fate actions in"
4998                                           " same flow");
4999         if (!action->conf)
5000                 return rte_flow_error_set(error, EINVAL,
5001                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
5002                                           NULL, "action configuration not set");
5003         target_group =
5004                 ((const struct rte_flow_action_jump *)action->conf)->group;
5005         ret = mlx5_flow_group_to_table(dev, tunnel, target_group, &table,
5006                                        &grp_info, error);
5007         if (ret)
5008                 return ret;
5009         if (attributes->group == target_group &&
5010             !(action_flags & (MLX5_FLOW_ACTION_TUNNEL_SET |
5011                               MLX5_FLOW_ACTION_TUNNEL_MATCH)))
5012                 return rte_flow_error_set(error, EINVAL,
5013                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5014                                           "target group must be other than"
5015                                           " the current flow group");
5016         if (table == 0)
5017                 return rte_flow_error_set(error, EINVAL,
5018                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
5019                                           NULL, "root table shouldn't be destination");
5020         return 0;
5021 }
5022
5023 /*
5024  * Validate action PORT_ID / REPRESENTED_PORT.
5025  *
5026  * @param[in] dev
5027  *   Pointer to rte_eth_dev structure.
5028  * @param[in] action_flags
5029  *   Bit-fields that holds the actions detected until now.
5030  * @param[in] action
5031  *   PORT_ID / REPRESENTED_PORT action structure.
5032  * @param[in] attr
5033  *   Attributes of flow that includes this action.
5034  * @param[out] error
5035  *   Pointer to error structure.
5036  *
5037  * @return
5038  *   0 on success, a negative errno value otherwise and rte_errno is set.
5039  */
5040 static int
5041 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
5042                                 uint64_t action_flags,
5043                                 const struct rte_flow_action *action,
5044                                 const struct rte_flow_attr *attr,
5045                                 struct rte_flow_error *error)
5046 {
5047         const struct rte_flow_action_port_id *port_id;
5048         const struct rte_flow_action_ethdev *ethdev;
5049         struct mlx5_priv *act_priv;
5050         struct mlx5_priv *dev_priv;
5051         uint16_t port;
5052
5053         if (!attr->transfer)
5054                 return rte_flow_error_set(error, ENOTSUP,
5055                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5056                                           NULL,
5057                                           "port action is valid in transfer"
5058                                           " mode only");
5059         if (!action || !action->conf)
5060                 return rte_flow_error_set(error, ENOTSUP,
5061                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
5062                                           NULL,
5063                                           "port action parameters must be"
5064                                           " specified");
5065         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
5066                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5067                 return rte_flow_error_set(error, EINVAL,
5068                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5069                                           "can have only one fate actions in"
5070                                           " a flow");
5071         dev_priv = mlx5_dev_to_eswitch_info(dev);
5072         if (!dev_priv)
5073                 return rte_flow_error_set(error, rte_errno,
5074                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5075                                           NULL,
5076                                           "failed to obtain E-Switch info");
5077         switch (action->type) {
5078         case RTE_FLOW_ACTION_TYPE_PORT_ID:
5079                 port_id = action->conf;
5080                 port = port_id->original ? dev->data->port_id : port_id->id;
5081                 break;
5082         case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
5083                 ethdev = action->conf;
5084                 port = ethdev->port_id;
5085                 break;
5086         default:
5087                 MLX5_ASSERT(false);
5088                 return rte_flow_error_set
5089                                 (error, EINVAL,
5090                                  RTE_FLOW_ERROR_TYPE_ACTION, action,
5091                                  "unknown E-Switch action");
5092         }
5093         act_priv = mlx5_port_to_eswitch_info(port, false);
5094         if (!act_priv)
5095                 return rte_flow_error_set
5096                                 (error, rte_errno,
5097                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, action->conf,
5098                                  "failed to obtain E-Switch port id for port");
5099         if (act_priv->domain_id != dev_priv->domain_id)
5100                 return rte_flow_error_set
5101                                 (error, EINVAL,
5102                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5103                                  "port does not belong to"
5104                                  " E-Switch being configured");
5105         return 0;
5106 }
5107
5108 /**
5109  * Get the maximum number of modify header actions.
5110  *
5111  * @param dev
5112  *   Pointer to rte_eth_dev structure.
5113  * @param root
5114  *   Whether action is on root table.
5115  *
5116  * @return
5117  *   Max number of modify header actions device can support.
5118  */
5119 static inline unsigned int
5120 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
5121                               bool root)
5122 {
5123         /*
5124          * There's no way to directly query the max capacity from FW.
5125          * The maximal value on root table should be assumed to be supported.
5126          */
5127         if (!root)
5128                 return MLX5_MAX_MODIFY_NUM;
5129         else
5130                 return MLX5_ROOT_TBL_MODIFY_NUM;
5131 }
5132
5133 /**
5134  * Validate the meter action.
5135  *
5136  * @param[in] dev
5137  *   Pointer to rte_eth_dev structure.
5138  * @param[in] action_flags
5139  *   Bit-fields that holds the actions detected until now.
5140  * @param[in] item_flags
5141  *   Holds the items detected.
5142  * @param[in] action
5143  *   Pointer to the meter action.
5144  * @param[in] attr
5145  *   Attributes of flow that includes this action.
5146  * @param[in] port_id_item
5147  *   Pointer to item indicating port id.
5148  * @param[out] error
5149  *   Pointer to error structure.
5150  *
5151  * @return
5152  *   0 on success, a negative errno value otherwise and rte_errno is set.
5153  */
5154 static int
5155 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
5156                                 uint64_t action_flags, uint64_t item_flags,
5157                                 const struct rte_flow_action *action,
5158                                 const struct rte_flow_attr *attr,
5159                                 const struct rte_flow_item *port_id_item,
5160                                 bool *def_policy,
5161                                 struct rte_flow_error *error)
5162 {
5163         struct mlx5_priv *priv = dev->data->dev_private;
5164         const struct rte_flow_action_meter *am = action->conf;
5165         struct mlx5_flow_meter_info *fm;
5166         struct mlx5_flow_meter_policy *mtr_policy;
5167         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
5168
5169         if (!am)
5170                 return rte_flow_error_set(error, EINVAL,
5171                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5172                                           "meter action conf is NULL");
5173
5174         if (action_flags & MLX5_FLOW_ACTION_METER)
5175                 return rte_flow_error_set(error, ENOTSUP,
5176                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5177                                           "meter chaining not support");
5178         if (action_flags & MLX5_FLOW_ACTION_JUMP)
5179                 return rte_flow_error_set(error, ENOTSUP,
5180                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5181                                           "meter with jump not support");
5182         if (!priv->mtr_en)
5183                 return rte_flow_error_set(error, ENOTSUP,
5184                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5185                                           NULL,
5186                                           "meter action not supported");
5187         fm = mlx5_flow_meter_find(priv, am->mtr_id, NULL);
5188         if (!fm)
5189                 return rte_flow_error_set(error, EINVAL,
5190                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5191                                           "Meter not found");
5192         /* aso meter can always be shared by different domains */
5193         if (fm->ref_cnt && !priv->sh->meter_aso_en &&
5194             !(fm->transfer == attr->transfer ||
5195               (!fm->ingress && !attr->ingress && attr->egress) ||
5196               (!fm->egress && !attr->egress && attr->ingress)))
5197                 return rte_flow_error_set(error, EINVAL,
5198                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5199                         "Flow attributes domain are either invalid "
5200                         "or have a domain conflict with current "
5201                         "meter attributes");
5202         if (fm->def_policy) {
5203                 if (!((attr->transfer &&
5204                         mtrmng->def_policy[MLX5_MTR_DOMAIN_TRANSFER]) ||
5205                         (attr->egress &&
5206                         mtrmng->def_policy[MLX5_MTR_DOMAIN_EGRESS]) ||
5207                         (attr->ingress &&
5208                         mtrmng->def_policy[MLX5_MTR_DOMAIN_INGRESS])))
5209                         return rte_flow_error_set(error, EINVAL,
5210                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5211                                           "Flow attributes domain "
5212                                           "have a conflict with current "
5213                                           "meter domain attributes");
5214                 *def_policy = true;
5215         } else {
5216                 mtr_policy = mlx5_flow_meter_policy_find(dev,
5217                                                 fm->policy_id, NULL);
5218                 if (!mtr_policy)
5219                         return rte_flow_error_set(error, EINVAL,
5220                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5221                                           "Invalid policy id for meter ");
5222                 if (!((attr->transfer && mtr_policy->transfer) ||
5223                         (attr->egress && mtr_policy->egress) ||
5224                         (attr->ingress && mtr_policy->ingress)))
5225                         return rte_flow_error_set(error, EINVAL,
5226                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5227                                           "Flow attributes domain "
5228                                           "have a conflict with current "
5229                                           "meter domain attributes");
5230                 if (attr->transfer && mtr_policy->dev) {
5231                         /**
5232                          * When policy has fate action of port_id,
5233                          * the flow should have the same src port as policy.
5234                          */
5235                         struct mlx5_priv *policy_port_priv =
5236                                         mtr_policy->dev->data->dev_private;
5237                         int32_t flow_src_port = priv->representor_id;
5238
5239                         if (port_id_item) {
5240                                 const struct rte_flow_item_port_id *spec =
5241                                                         port_id_item->spec;
5242                                 struct mlx5_priv *port_priv =
5243                                         mlx5_port_to_eswitch_info(spec->id,
5244                                                                   false);
5245                                 if (!port_priv)
5246                                         return rte_flow_error_set(error,
5247                                                 rte_errno,
5248                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5249                                                 spec,
5250                                                 "Failed to get port info.");
5251                                 flow_src_port = port_priv->representor_id;
5252                         }
5253                         if (flow_src_port != policy_port_priv->representor_id)
5254                                 return rte_flow_error_set(error,
5255                                                 rte_errno,
5256                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5257                                                 NULL,
5258                                                 "Flow and meter policy "
5259                                                 "have different src port.");
5260                 } else if (mtr_policy->is_rss) {
5261                         struct mlx5_flow_meter_policy *fp;
5262                         struct mlx5_meter_policy_action_container *acg;
5263                         struct mlx5_meter_policy_action_container *acy;
5264                         const struct rte_flow_action *rss_act;
5265                         int ret;
5266
5267                         fp = mlx5_flow_meter_hierarchy_get_final_policy(dev,
5268                                                                 mtr_policy);
5269                         if (fp == NULL)
5270                                 return rte_flow_error_set(error, EINVAL,
5271                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5272                                                   "Unable to get the final "
5273                                                   "policy in the hierarchy");
5274                         acg = &fp->act_cnt[RTE_COLOR_GREEN];
5275                         acy = &fp->act_cnt[RTE_COLOR_YELLOW];
5276                         MLX5_ASSERT(acg->fate_action ==
5277                                     MLX5_FLOW_FATE_SHARED_RSS ||
5278                                     acy->fate_action ==
5279                                     MLX5_FLOW_FATE_SHARED_RSS);
5280                         if (acg->fate_action == MLX5_FLOW_FATE_SHARED_RSS)
5281                                 rss_act = acg->rss;
5282                         else
5283                                 rss_act = acy->rss;
5284                         ret = mlx5_flow_validate_action_rss(rss_act,
5285                                         action_flags, dev, attr,
5286                                         item_flags, error);
5287                         if (ret)
5288                                 return ret;
5289                 }
5290                 *def_policy = false;
5291         }
5292         return 0;
5293 }
5294
5295 /**
5296  * Validate the age action.
5297  *
5298  * @param[in] action_flags
5299  *   Holds the actions detected until now.
5300  * @param[in] action
5301  *   Pointer to the age action.
5302  * @param[in] dev
5303  *   Pointer to the Ethernet device structure.
5304  * @param[out] error
5305  *   Pointer to error structure.
5306  *
5307  * @return
5308  *   0 on success, a negative errno value otherwise and rte_errno is set.
5309  */
5310 static int
5311 flow_dv_validate_action_age(uint64_t action_flags,
5312                             const struct rte_flow_action *action,
5313                             struct rte_eth_dev *dev,
5314                             struct rte_flow_error *error)
5315 {
5316         struct mlx5_priv *priv = dev->data->dev_private;
5317         const struct rte_flow_action_age *age = action->conf;
5318
5319         if (!priv->sh->cdev->config.devx ||
5320             (priv->sh->cmng.counter_fallback && !priv->sh->aso_age_mng))
5321                 return rte_flow_error_set(error, ENOTSUP,
5322                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5323                                           NULL,
5324                                           "age action not supported");
5325         if (!(action->conf))
5326                 return rte_flow_error_set(error, EINVAL,
5327                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5328                                           "configuration cannot be null");
5329         if (!(age->timeout))
5330                 return rte_flow_error_set(error, EINVAL,
5331                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5332                                           "invalid timeout value 0");
5333         if (action_flags & MLX5_FLOW_ACTION_AGE)
5334                 return rte_flow_error_set(error, EINVAL,
5335                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5336                                           "duplicate age actions set");
5337         return 0;
5338 }
5339
5340 /**
5341  * Validate the modify-header IPv4 DSCP actions.
5342  *
5343  * @param[in] action_flags
5344  *   Holds the actions detected until now.
5345  * @param[in] action
5346  *   Pointer to the modify action.
5347  * @param[in] item_flags
5348  *   Holds the items detected.
5349  * @param[out] error
5350  *   Pointer to error structure.
5351  *
5352  * @return
5353  *   0 on success, a negative errno value otherwise and rte_errno is set.
5354  */
5355 static int
5356 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
5357                                          const struct rte_flow_action *action,
5358                                          const uint64_t item_flags,
5359                                          struct rte_flow_error *error)
5360 {
5361         int ret = 0;
5362
5363         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5364         if (!ret) {
5365                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
5366                         return rte_flow_error_set(error, EINVAL,
5367                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5368                                                   NULL,
5369                                                   "no ipv4 item in pattern");
5370         }
5371         return ret;
5372 }
5373
5374 /**
5375  * Validate the modify-header IPv6 DSCP actions.
5376  *
5377  * @param[in] action_flags
5378  *   Holds the actions detected until now.
5379  * @param[in] action
5380  *   Pointer to the modify action.
5381  * @param[in] item_flags
5382  *   Holds the items detected.
5383  * @param[out] error
5384  *   Pointer to error structure.
5385  *
5386  * @return
5387  *   0 on success, a negative errno value otherwise and rte_errno is set.
5388  */
5389 static int
5390 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
5391                                          const struct rte_flow_action *action,
5392                                          const uint64_t item_flags,
5393                                          struct rte_flow_error *error)
5394 {
5395         int ret = 0;
5396
5397         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5398         if (!ret) {
5399                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
5400                         return rte_flow_error_set(error, EINVAL,
5401                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5402                                                   NULL,
5403                                                   "no ipv6 item in pattern");
5404         }
5405         return ret;
5406 }
5407
5408 int
5409 flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
5410                         struct mlx5_list_entry *entry, void *cb_ctx)
5411 {
5412         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5413         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5414         struct mlx5_flow_dv_modify_hdr_resource *resource =
5415                                   container_of(entry, typeof(*resource), entry);
5416         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5417
5418         key_len += ref->actions_num * sizeof(ref->actions[0]);
5419         return ref->actions_num != resource->actions_num ||
5420                memcmp(&ref->ft_type, &resource->ft_type, key_len);
5421 }
5422
5423 static struct mlx5_indexed_pool *
5424 flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
5425 {
5426         struct mlx5_indexed_pool *ipool = __atomic_load_n
5427                                      (&sh->mdh_ipools[index], __ATOMIC_SEQ_CST);
5428
5429         if (!ipool) {
5430                 struct mlx5_indexed_pool *expected = NULL;
5431                 struct mlx5_indexed_pool_config cfg =
5432                     (struct mlx5_indexed_pool_config) {
5433                        .size = sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
5434                                                                    (index + 1) *
5435                                            sizeof(struct mlx5_modification_cmd),
5436                        .trunk_size = 64,
5437                        .grow_trunk = 3,
5438                        .grow_shift = 2,
5439                        .need_lock = 1,
5440                        .release_mem_en = !!sh->config.reclaim_mode,
5441                        .per_core_cache =
5442                                        sh->config.reclaim_mode ? 0 : (1 << 16),
5443                        .malloc = mlx5_malloc,
5444                        .free = mlx5_free,
5445                        .type = "mlx5_modify_action_resource",
5446                 };
5447
5448                 cfg.size = RTE_ALIGN(cfg.size, sizeof(ipool));
5449                 ipool = mlx5_ipool_create(&cfg);
5450                 if (!ipool)
5451                         return NULL;
5452                 if (!__atomic_compare_exchange_n(&sh->mdh_ipools[index],
5453                                                  &expected, ipool, false,
5454                                                  __ATOMIC_SEQ_CST,
5455                                                  __ATOMIC_SEQ_CST)) {
5456                         mlx5_ipool_destroy(ipool);
5457                         ipool = __atomic_load_n(&sh->mdh_ipools[index],
5458                                                 __ATOMIC_SEQ_CST);
5459                 }
5460         }
5461         return ipool;
5462 }
5463
5464 struct mlx5_list_entry *
5465 flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
5466 {
5467         struct mlx5_dev_ctx_shared *sh = tool_ctx;
5468         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5469         struct mlx5dv_dr_domain *ns;
5470         struct mlx5_flow_dv_modify_hdr_resource *entry;
5471         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5472         struct mlx5_indexed_pool *ipool = flow_dv_modify_ipool_get(sh,
5473                                                           ref->actions_num - 1);
5474         int ret;
5475         uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
5476         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5477         uint32_t idx;
5478
5479         if (unlikely(!ipool)) {
5480                 rte_flow_error_set(ctx->error, ENOMEM,
5481                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5482                                    NULL, "cannot allocate modify ipool");
5483                 return NULL;
5484         }
5485         entry = mlx5_ipool_zmalloc(ipool, &idx);
5486         if (!entry) {
5487                 rte_flow_error_set(ctx->error, ENOMEM,
5488                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5489                                    "cannot allocate resource memory");
5490                 return NULL;
5491         }
5492         rte_memcpy(&entry->ft_type,
5493                    RTE_PTR_ADD(ref, offsetof(typeof(*ref), ft_type)),
5494                    key_len + data_len);
5495         if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
5496                 ns = sh->fdb_domain;
5497         else if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
5498                 ns = sh->tx_domain;
5499         else
5500                 ns = sh->rx_domain;
5501         ret = mlx5_flow_os_create_flow_action_modify_header
5502                                         (sh->cdev->ctx, ns, entry,
5503                                          data_len, &entry->action);
5504         if (ret) {
5505                 mlx5_ipool_free(sh->mdh_ipools[ref->actions_num - 1], idx);
5506                 rte_flow_error_set(ctx->error, ENOMEM,
5507                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5508                                    NULL, "cannot create modification action");
5509                 return NULL;
5510         }
5511         entry->idx = idx;
5512         return &entry->entry;
5513 }
5514
5515 struct mlx5_list_entry *
5516 flow_dv_modify_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
5517                         void *cb_ctx)
5518 {
5519         struct mlx5_dev_ctx_shared *sh = tool_ctx;
5520         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5521         struct mlx5_flow_dv_modify_hdr_resource *entry;
5522         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5523         uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
5524         uint32_t idx;
5525
5526         entry = mlx5_ipool_malloc(sh->mdh_ipools[ref->actions_num - 1],
5527                                   &idx);
5528         if (!entry) {
5529                 rte_flow_error_set(ctx->error, ENOMEM,
5530                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5531                                    "cannot allocate resource memory");
5532                 return NULL;
5533         }
5534         memcpy(entry, oentry, sizeof(*entry) + data_len);
5535         entry->idx = idx;
5536         return &entry->entry;
5537 }
5538
5539 void
5540 flow_dv_modify_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
5541 {
5542         struct mlx5_dev_ctx_shared *sh = tool_ctx;
5543         struct mlx5_flow_dv_modify_hdr_resource *res =
5544                 container_of(entry, typeof(*res), entry);
5545
5546         mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
5547 }
5548
5549 /**
5550  * Validate the sample action.
5551  *
5552  * @param[in, out] action_flags
5553  *   Holds the actions detected until now.
5554  * @param[in] action
5555  *   Pointer to the sample action.
5556  * @param[in] dev
5557  *   Pointer to the Ethernet device structure.
5558  * @param[in] attr
5559  *   Attributes of flow that includes this action.
5560  * @param[in] item_flags
5561  *   Holds the items detected.
5562  * @param[in] rss
5563  *   Pointer to the RSS action.
5564  * @param[out] sample_rss
5565  *   Pointer to the RSS action in sample action list.
5566  * @param[out] count
5567  *   Pointer to the COUNT action in sample action list.
5568  * @param[out] fdb_mirror_limit
5569  *   Pointer to the FDB mirror limitation flag.
5570  * @param[out] error
5571  *   Pointer to error structure.
5572  *
5573  * @return
5574  *   0 on success, a negative errno value otherwise and rte_errno is set.
5575  */
5576 static int
5577 flow_dv_validate_action_sample(uint64_t *action_flags,
5578                                const struct rte_flow_action *action,
5579                                struct rte_eth_dev *dev,
5580                                const struct rte_flow_attr *attr,
5581                                uint64_t item_flags,
5582                                const struct rte_flow_action_rss *rss,
5583                                const struct rte_flow_action_rss **sample_rss,
5584                                const struct rte_flow_action_count **count,
5585                                int *fdb_mirror_limit,
5586                                struct rte_flow_error *error)
5587 {
5588         struct mlx5_priv *priv = dev->data->dev_private;
5589         struct mlx5_sh_config *dev_conf = &priv->sh->config;
5590         const struct rte_flow_action_sample *sample = action->conf;
5591         const struct rte_flow_action *act;
5592         uint64_t sub_action_flags = 0;
5593         uint16_t queue_index = 0xFFFF;
5594         int actions_n = 0;
5595         int ret;
5596
5597         if (!sample)
5598                 return rte_flow_error_set(error, EINVAL,
5599                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5600                                           "configuration cannot be NULL");
5601         if (sample->ratio == 0)
5602                 return rte_flow_error_set(error, EINVAL,
5603                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5604                                           "ratio value starts from 1");
5605         if (!priv->sh->cdev->config.devx ||
5606             (sample->ratio > 0 && !priv->sampler_en))
5607                 return rte_flow_error_set(error, ENOTSUP,
5608                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5609                                           NULL,
5610                                           "sample action not supported");
5611         if (*action_flags & MLX5_FLOW_ACTION_SAMPLE)
5612                 return rte_flow_error_set(error, EINVAL,
5613                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5614                                           "Multiple sample actions not "
5615                                           "supported");
5616         if (*action_flags & MLX5_FLOW_ACTION_METER)
5617                 return rte_flow_error_set(error, EINVAL,
5618                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5619                                           "wrong action order, meter should "
5620                                           "be after sample action");
5621         if (*action_flags & MLX5_FLOW_ACTION_JUMP)
5622                 return rte_flow_error_set(error, EINVAL,
5623                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5624                                           "wrong action order, jump should "
5625                                           "be after sample action");
5626         if (*action_flags & MLX5_FLOW_ACTION_CT)
5627                 return rte_flow_error_set(error, EINVAL,
5628                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5629                                           "Sample after CT not supported");
5630         act = sample->actions;
5631         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
5632                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5633                         return rte_flow_error_set(error, ENOTSUP,
5634                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5635                                                   act, "too many actions");
5636                 switch (act->type) {
5637                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5638                         ret = mlx5_flow_validate_action_queue(act,
5639                                                               sub_action_flags,
5640                                                               dev,
5641                                                               attr, error);
5642                         if (ret < 0)
5643                                 return ret;
5644                         queue_index = ((const struct rte_flow_action_queue *)
5645                                                         (act->conf))->index;
5646                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
5647                         ++actions_n;
5648                         break;
5649                 case RTE_FLOW_ACTION_TYPE_RSS:
5650                         *sample_rss = act->conf;
5651                         ret = mlx5_flow_validate_action_rss(act,
5652                                                             sub_action_flags,
5653                                                             dev, attr,
5654                                                             item_flags,
5655                                                             error);
5656                         if (ret < 0)
5657                                 return ret;
5658                         if (rss && *sample_rss &&
5659                             ((*sample_rss)->level != rss->level ||
5660                             (*sample_rss)->types != rss->types))
5661                                 return rte_flow_error_set(error, ENOTSUP,
5662                                         RTE_FLOW_ERROR_TYPE_ACTION,
5663                                         NULL,
5664                                         "Can't use the different RSS types "
5665                                         "or level in the same flow");
5666                         if (*sample_rss != NULL && (*sample_rss)->queue_num)
5667                                 queue_index = (*sample_rss)->queue[0];
5668                         sub_action_flags |= MLX5_FLOW_ACTION_RSS;
5669                         ++actions_n;
5670                         break;
5671                 case RTE_FLOW_ACTION_TYPE_MARK:
5672                         ret = flow_dv_validate_action_mark(dev, act,
5673                                                            sub_action_flags,
5674                                                            attr, error);
5675                         if (ret < 0)
5676                                 return ret;
5677                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
5678                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
5679                                                 MLX5_FLOW_ACTION_MARK_EXT;
5680                         else
5681                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
5682                         ++actions_n;
5683                         break;
5684                 case RTE_FLOW_ACTION_TYPE_COUNT:
5685                         ret = flow_dv_validate_action_count
5686                                 (dev, false, *action_flags | sub_action_flags,
5687                                  attr, error);
5688                         if (ret < 0)
5689                                 return ret;
5690                         *count = act->conf;
5691                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
5692                         *action_flags |= MLX5_FLOW_ACTION_COUNT;
5693                         ++actions_n;
5694                         break;
5695                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5696                 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
5697                         ret = flow_dv_validate_action_port_id(dev,
5698                                                               sub_action_flags,
5699                                                               act,
5700                                                               attr,
5701                                                               error);
5702                         if (ret)
5703                                 return ret;
5704                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5705                         ++actions_n;
5706                         break;
5707                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5708                         ret = flow_dv_validate_action_raw_encap_decap
5709                                 (dev, NULL, act->conf, attr, &sub_action_flags,
5710                                  &actions_n, action, item_flags, error);
5711                         if (ret < 0)
5712                                 return ret;
5713                         ++actions_n;
5714                         break;
5715                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5716                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5717                         ret = flow_dv_validate_action_l2_encap(dev,
5718                                                                sub_action_flags,
5719                                                                act, attr,
5720                                                                error);
5721                         if (ret < 0)
5722                                 return ret;
5723                         sub_action_flags |= MLX5_FLOW_ACTION_ENCAP;
5724                         ++actions_n;
5725                         break;
5726                 default:
5727                         return rte_flow_error_set(error, ENOTSUP,
5728                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5729                                                   NULL,
5730                                                   "Doesn't support optional "
5731                                                   "action");
5732                 }
5733         }
5734         if (attr->ingress && !attr->transfer) {
5735                 if (!(sub_action_flags & (MLX5_FLOW_ACTION_QUEUE |
5736                                           MLX5_FLOW_ACTION_RSS)))
5737                         return rte_flow_error_set(error, EINVAL,
5738                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5739                                                   NULL,
5740                                                   "Ingress must has a dest "
5741                                                   "QUEUE for Sample");
5742         } else if (attr->egress && !attr->transfer) {
5743                 return rte_flow_error_set(error, ENOTSUP,
5744                                           RTE_FLOW_ERROR_TYPE_ACTION,
5745                                           NULL,
5746                                           "Sample Only support Ingress "
5747                                           "or E-Switch");
5748         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
5749                 MLX5_ASSERT(attr->transfer);
5750                 if (sample->ratio > 1)
5751                         return rte_flow_error_set(error, ENOTSUP,
5752                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5753                                                   NULL,
5754                                                   "E-Switch doesn't support "
5755                                                   "any optional action "
5756                                                   "for sampling");
5757                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
5758                         return rte_flow_error_set(error, ENOTSUP,
5759                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5760                                                   NULL,
5761                                                   "unsupported action QUEUE");
5762                 if (sub_action_flags & MLX5_FLOW_ACTION_RSS)
5763                         return rte_flow_error_set(error, ENOTSUP,
5764                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5765                                                   NULL,
5766                                                   "unsupported action QUEUE");
5767                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
5768                         return rte_flow_error_set(error, EINVAL,
5769                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5770                                                   NULL,
5771                                                   "E-Switch must has a dest "
5772                                                   "port for mirroring");
5773                 if (!priv->sh->cdev->config.hca_attr.reg_c_preserve &&
5774                      priv->representor_id != UINT16_MAX)
5775                         *fdb_mirror_limit = 1;
5776         }
5777         /* Continue validation for Xcap actions.*/
5778         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
5779             (queue_index == 0xFFFF || !mlx5_rxq_is_hairpin(dev, queue_index))) {
5780                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
5781                      MLX5_FLOW_XCAP_ACTIONS)
5782                         return rte_flow_error_set(error, ENOTSUP,
5783                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5784                                                   NULL, "encap and decap "
5785                                                   "combination aren't "
5786                                                   "supported");
5787                 if (!attr->transfer && attr->ingress && (sub_action_flags &
5788                                                         MLX5_FLOW_ACTION_ENCAP))
5789                         return rte_flow_error_set(error, ENOTSUP,
5790                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5791                                                   NULL, "encap is not supported"
5792                                                   " for ingress traffic");
5793         }
5794         return 0;
5795 }
5796
5797 /**
5798  * Find existing modify-header resource or create and register a new one.
5799  *
5800  * @param dev[in, out]
5801  *   Pointer to rte_eth_dev structure.
5802  * @param[in, out] resource
5803  *   Pointer to modify-header resource.
5804  * @parm[in, out] dev_flow
5805  *   Pointer to the dev_flow.
5806  * @param[out] error
5807  *   pointer to error structure.
5808  *
5809  * @return
5810  *   0 on success otherwise -errno and errno is set.
5811  */
5812 static int
5813 flow_dv_modify_hdr_resource_register
5814                         (struct rte_eth_dev *dev,
5815                          struct mlx5_flow_dv_modify_hdr_resource *resource,
5816                          struct mlx5_flow *dev_flow,
5817                          struct rte_flow_error *error)
5818 {
5819         struct mlx5_priv *priv = dev->data->dev_private;
5820         struct mlx5_dev_ctx_shared *sh = priv->sh;
5821         uint32_t key_len = sizeof(*resource) -
5822                            offsetof(typeof(*resource), ft_type) +
5823                            resource->actions_num * sizeof(resource->actions[0]);
5824         struct mlx5_list_entry *entry;
5825         struct mlx5_flow_cb_ctx ctx = {
5826                 .error = error,
5827                 .data = resource,
5828         };
5829         struct mlx5_hlist *modify_cmds;
5830         uint64_t key64;
5831
5832         modify_cmds = flow_dv_hlist_prepare(sh, &sh->modify_cmds,
5833                                 "hdr_modify",
5834                                 MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
5835                                 true, false, sh,
5836                                 flow_dv_modify_create_cb,
5837                                 flow_dv_modify_match_cb,
5838                                 flow_dv_modify_remove_cb,
5839                                 flow_dv_modify_clone_cb,
5840                                 flow_dv_modify_clone_free_cb,
5841                                 error);
5842         if (unlikely(!modify_cmds))
5843                 return -rte_errno;
5844         resource->root = !dev_flow->dv.group;
5845         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
5846                                                                 resource->root))
5847                 return rte_flow_error_set(error, EOVERFLOW,
5848                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5849                                           "too many modify header items");
5850         key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0);
5851         entry = mlx5_hlist_register(modify_cmds, key64, &ctx);
5852         if (!entry)
5853                 return -rte_errno;
5854         resource = container_of(entry, typeof(*resource), entry);
5855         dev_flow->handle->dvh.modify_hdr = resource;
5856         return 0;
5857 }
5858
5859 /**
5860  * Get DV flow counter by index.
5861  *
5862  * @param[in] dev
5863  *   Pointer to the Ethernet device structure.
5864  * @param[in] idx
5865  *   mlx5 flow counter index in the container.
5866  * @param[out] ppool
5867  *   mlx5 flow counter pool in the container.
5868  *
5869  * @return
5870  *   Pointer to the counter, NULL otherwise.
5871  */
5872 static struct mlx5_flow_counter *
5873 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
5874                            uint32_t idx,
5875                            struct mlx5_flow_counter_pool **ppool)
5876 {
5877         struct mlx5_priv *priv = dev->data->dev_private;
5878         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5879         struct mlx5_flow_counter_pool *pool;
5880
5881         /* Decrease to original index and clear shared bit. */
5882         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
5883         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cmng->n);
5884         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
5885         MLX5_ASSERT(pool);
5886         if (ppool)
5887                 *ppool = pool;
5888         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
5889 }
5890
5891 /**
5892  * Check the devx counter belongs to the pool.
5893  *
5894  * @param[in] pool
5895  *   Pointer to the counter pool.
5896  * @param[in] id
5897  *   The counter devx ID.
5898  *
5899  * @return
5900  *   True if counter belongs to the pool, false otherwise.
5901  */
5902 static bool
5903 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
5904 {
5905         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
5906                    MLX5_COUNTERS_PER_POOL;
5907
5908         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
5909                 return true;
5910         return false;
5911 }
5912
5913 /**
5914  * Get a pool by devx counter ID.
5915  *
5916  * @param[in] cmng
5917  *   Pointer to the counter management.
5918  * @param[in] id
5919  *   The counter devx ID.
5920  *
5921  * @return
5922  *   The counter pool pointer if exists, NULL otherwise,
5923  */
5924 static struct mlx5_flow_counter_pool *
5925 flow_dv_find_pool_by_id(struct mlx5_flow_counter_mng *cmng, int id)
5926 {
5927         uint32_t i;
5928         struct mlx5_flow_counter_pool *pool = NULL;
5929
5930         rte_spinlock_lock(&cmng->pool_update_sl);
5931         /* Check last used pool. */
5932         if (cmng->last_pool_idx != POOL_IDX_INVALID &&
5933             flow_dv_is_counter_in_pool(cmng->pools[cmng->last_pool_idx], id)) {
5934                 pool = cmng->pools[cmng->last_pool_idx];
5935                 goto out;
5936         }
5937         /* ID out of range means no suitable pool in the container. */
5938         if (id > cmng->max_id || id < cmng->min_id)
5939                 goto out;
5940         /*
5941          * Find the pool from the end of the container, since mostly counter
5942          * ID is sequence increasing, and the last pool should be the needed
5943          * one.
5944          */
5945         i = cmng->n_valid;
5946         while (i--) {
5947                 struct mlx5_flow_counter_pool *pool_tmp = cmng->pools[i];
5948
5949                 if (flow_dv_is_counter_in_pool(pool_tmp, id)) {
5950                         pool = pool_tmp;
5951                         break;
5952                 }
5953         }
5954 out:
5955         rte_spinlock_unlock(&cmng->pool_update_sl);
5956         return pool;
5957 }
5958
5959 /**
5960  * Resize a counter container.
5961  *
5962  * @param[in] dev
5963  *   Pointer to the Ethernet device structure.
5964  *
5965  * @return
5966  *   0 on success, otherwise negative errno value and rte_errno is set.
5967  */
5968 static int
5969 flow_dv_container_resize(struct rte_eth_dev *dev)
5970 {
5971         struct mlx5_priv *priv = dev->data->dev_private;
5972         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5973         void *old_pools = cmng->pools;
5974         uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
5975         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
5976         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
5977
5978         if (!pools) {
5979                 rte_errno = ENOMEM;
5980                 return -ENOMEM;
5981         }
5982         if (old_pools)
5983                 memcpy(pools, old_pools, cmng->n *
5984                                        sizeof(struct mlx5_flow_counter_pool *));
5985         cmng->n = resize;
5986         cmng->pools = pools;
5987         if (old_pools)
5988                 mlx5_free(old_pools);
5989         return 0;
5990 }
5991
5992 /**
5993  * Query a devx flow counter.
5994  *
5995  * @param[in] dev
5996  *   Pointer to the Ethernet device structure.
5997  * @param[in] counter
5998  *   Index to the flow counter.
5999  * @param[out] pkts
6000  *   The statistics value of packets.
6001  * @param[out] bytes
6002  *   The statistics value of bytes.
6003  *
6004  * @return
6005  *   0 on success, otherwise a negative errno value and rte_errno is set.
6006  */
6007 static inline int
6008 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
6009                      uint64_t *bytes)
6010 {
6011         struct mlx5_priv *priv = dev->data->dev_private;
6012         struct mlx5_flow_counter_pool *pool = NULL;
6013         struct mlx5_flow_counter *cnt;
6014         int offset;
6015
6016         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
6017         MLX5_ASSERT(pool);
6018         if (priv->sh->cmng.counter_fallback)
6019                 return mlx5_devx_cmd_flow_counter_query(cnt->dcs_when_active, 0,
6020                                         0, pkts, bytes, 0, NULL, NULL, 0);
6021         rte_spinlock_lock(&pool->sl);
6022         if (!pool->raw) {
6023                 *pkts = 0;
6024                 *bytes = 0;
6025         } else {
6026                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
6027                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
6028                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
6029         }
6030         rte_spinlock_unlock(&pool->sl);
6031         return 0;
6032 }
6033
6034 /**
6035  * Create and initialize a new counter pool.
6036  *
6037  * @param[in] dev
6038  *   Pointer to the Ethernet device structure.
6039  * @param[out] dcs
6040  *   The devX counter handle.
6041  * @param[in] age
6042  *   Whether the pool is for counter that was allocated for aging.
6043  * @param[in/out] cont_cur
6044  *   Pointer to the container pointer, it will be update in pool resize.
6045  *
6046  * @return
6047  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
6048  */
6049 static struct mlx5_flow_counter_pool *
6050 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
6051                     uint32_t age)
6052 {
6053         struct mlx5_priv *priv = dev->data->dev_private;
6054         struct mlx5_flow_counter_pool *pool;
6055         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6056         bool fallback = priv->sh->cmng.counter_fallback;
6057         uint32_t size = sizeof(*pool);
6058
6059         size += MLX5_COUNTERS_PER_POOL * MLX5_CNT_SIZE;
6060         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * MLX5_AGE_SIZE);
6061         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
6062         if (!pool) {
6063                 rte_errno = ENOMEM;
6064                 return NULL;
6065         }
6066         pool->raw = NULL;
6067         pool->is_aged = !!age;
6068         pool->query_gen = 0;
6069         pool->min_dcs = dcs;
6070         rte_spinlock_init(&pool->sl);
6071         rte_spinlock_init(&pool->csl);
6072         TAILQ_INIT(&pool->counters[0]);
6073         TAILQ_INIT(&pool->counters[1]);
6074         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
6075         rte_spinlock_lock(&cmng->pool_update_sl);
6076         pool->index = cmng->n_valid;
6077         if (pool->index == cmng->n && flow_dv_container_resize(dev)) {
6078                 mlx5_free(pool);
6079                 rte_spinlock_unlock(&cmng->pool_update_sl);
6080                 return NULL;
6081         }
6082         cmng->pools[pool->index] = pool;
6083         cmng->n_valid++;
6084         if (unlikely(fallback)) {
6085                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
6086
6087                 if (base < cmng->min_id)
6088                         cmng->min_id = base;
6089                 if (base > cmng->max_id)
6090                         cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
6091                 cmng->last_pool_idx = pool->index;
6092         }
6093         rte_spinlock_unlock(&cmng->pool_update_sl);
6094         return pool;
6095 }
6096
6097 /**
6098  * Prepare a new counter and/or a new counter pool.
6099  *
6100  * @param[in] dev
6101  *   Pointer to the Ethernet device structure.
6102  * @param[out] cnt_free
6103  *   Where to put the pointer of a new counter.
6104  * @param[in] age
6105  *   Whether the pool is for counter that was allocated for aging.
6106  *
6107  * @return
6108  *   The counter pool pointer and @p cnt_free is set on success,
6109  *   NULL otherwise and rte_errno is set.
6110  */
6111 static struct mlx5_flow_counter_pool *
6112 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
6113                              struct mlx5_flow_counter **cnt_free,
6114                              uint32_t age)
6115 {
6116         struct mlx5_priv *priv = dev->data->dev_private;
6117         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6118         struct mlx5_flow_counter_pool *pool;
6119         struct mlx5_counters tmp_tq;
6120         struct mlx5_devx_obj *dcs = NULL;
6121         struct mlx5_flow_counter *cnt;
6122         enum mlx5_counter_type cnt_type =
6123                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
6124         bool fallback = priv->sh->cmng.counter_fallback;
6125         uint32_t i;
6126
6127         if (fallback) {
6128                 /* bulk_bitmap must be 0 for single counter allocation. */
6129                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->cdev->ctx, 0);
6130                 if (!dcs)
6131                         return NULL;
6132                 pool = flow_dv_find_pool_by_id(cmng, dcs->id);
6133                 if (!pool) {
6134                         pool = flow_dv_pool_create(dev, dcs, age);
6135                         if (!pool) {
6136                                 mlx5_devx_cmd_destroy(dcs);
6137                                 return NULL;
6138                         }
6139                 }
6140                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
6141                 cnt = MLX5_POOL_GET_CNT(pool, i);
6142                 cnt->pool = pool;
6143                 cnt->dcs_when_free = dcs;
6144                 *cnt_free = cnt;
6145                 return pool;
6146         }
6147         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->cdev->ctx, 0x4);
6148         if (!dcs) {
6149                 rte_errno = ENODATA;
6150                 return NULL;
6151         }
6152         pool = flow_dv_pool_create(dev, dcs, age);
6153         if (!pool) {
6154                 mlx5_devx_cmd_destroy(dcs);
6155                 return NULL;
6156         }
6157         TAILQ_INIT(&tmp_tq);
6158         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
6159                 cnt = MLX5_POOL_GET_CNT(pool, i);
6160                 cnt->pool = pool;
6161                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
6162         }
6163         rte_spinlock_lock(&cmng->csl[cnt_type]);
6164         TAILQ_CONCAT(&cmng->counters[cnt_type], &tmp_tq, next);
6165         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6166         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
6167         (*cnt_free)->pool = pool;
6168         return pool;
6169 }
6170
6171 /**
6172  * Allocate a flow counter.
6173  *
6174  * @param[in] dev
6175  *   Pointer to the Ethernet device structure.
6176  * @param[in] age
6177  *   Whether the counter was allocated for aging.
6178  *
6179  * @return
6180  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6181  */
6182 static uint32_t
6183 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t age)
6184 {
6185         struct mlx5_priv *priv = dev->data->dev_private;
6186         struct mlx5_flow_counter_pool *pool = NULL;
6187         struct mlx5_flow_counter *cnt_free = NULL;
6188         bool fallback = priv->sh->cmng.counter_fallback;
6189         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6190         enum mlx5_counter_type cnt_type =
6191                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
6192         uint32_t cnt_idx;
6193
6194         if (!priv->sh->cdev->config.devx) {
6195                 rte_errno = ENOTSUP;
6196                 return 0;
6197         }
6198         /* Get free counters from container. */
6199         rte_spinlock_lock(&cmng->csl[cnt_type]);
6200         cnt_free = TAILQ_FIRST(&cmng->counters[cnt_type]);
6201         if (cnt_free)
6202                 TAILQ_REMOVE(&cmng->counters[cnt_type], cnt_free, next);
6203         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6204         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, age))
6205                 goto err;
6206         pool = cnt_free->pool;
6207         if (fallback)
6208                 cnt_free->dcs_when_active = cnt_free->dcs_when_free;
6209         /* Create a DV counter action only in the first time usage. */
6210         if (!cnt_free->action) {
6211                 uint16_t offset;
6212                 struct mlx5_devx_obj *dcs;
6213                 int ret;
6214
6215                 if (!fallback) {
6216                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
6217                         dcs = pool->min_dcs;
6218                 } else {
6219                         offset = 0;
6220                         dcs = cnt_free->dcs_when_free;
6221                 }
6222                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
6223                                                             &cnt_free->action);
6224                 if (ret) {
6225                         rte_errno = errno;
6226                         goto err;
6227                 }
6228         }
6229         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
6230                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
6231         /* Update the counter reset values. */
6232         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
6233                                  &cnt_free->bytes))
6234                 goto err;
6235         if (!fallback && !priv->sh->cmng.query_thread_on)
6236                 /* Start the asynchronous batch query by the host thread. */
6237                 mlx5_set_query_alarm(priv->sh);
6238         /*
6239          * When the count action isn't shared (by ID), shared_info field is
6240          * used for indirect action API's refcnt.
6241          * When the counter action is not shared neither by ID nor by indirect
6242          * action API, shared info must be 1.
6243          */
6244         cnt_free->shared_info.refcnt = 1;
6245         return cnt_idx;
6246 err:
6247         if (cnt_free) {
6248                 cnt_free->pool = pool;
6249                 if (fallback)
6250                         cnt_free->dcs_when_free = cnt_free->dcs_when_active;
6251                 rte_spinlock_lock(&cmng->csl[cnt_type]);
6252                 TAILQ_INSERT_TAIL(&cmng->counters[cnt_type], cnt_free, next);
6253                 rte_spinlock_unlock(&cmng->csl[cnt_type]);
6254         }
6255         return 0;
6256 }
6257
6258 /**
6259  * Get age param from counter index.
6260  *
6261  * @param[in] dev
6262  *   Pointer to the Ethernet device structure.
6263  * @param[in] counter
6264  *   Index to the counter handler.
6265  *
6266  * @return
6267  *   The aging parameter specified for the counter index.
6268  */
6269 static struct mlx5_age_param*
6270 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
6271                                 uint32_t counter)
6272 {
6273         struct mlx5_flow_counter *cnt;
6274         struct mlx5_flow_counter_pool *pool = NULL;
6275
6276         flow_dv_counter_get_by_idx(dev, counter, &pool);
6277         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
6278         cnt = MLX5_POOL_GET_CNT(pool, counter);
6279         return MLX5_CNT_TO_AGE(cnt);
6280 }
6281
6282 /**
6283  * Remove a flow counter from aged counter list.
6284  *
6285  * @param[in] dev
6286  *   Pointer to the Ethernet device structure.
6287  * @param[in] counter
6288  *   Index to the counter handler.
6289  * @param[in] cnt
6290  *   Pointer to the counter handler.
6291  */
6292 static void
6293 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
6294                                 uint32_t counter, struct mlx5_flow_counter *cnt)
6295 {
6296         struct mlx5_age_info *age_info;
6297         struct mlx5_age_param *age_param;
6298         struct mlx5_priv *priv = dev->data->dev_private;
6299         uint16_t expected = AGE_CANDIDATE;
6300
6301         age_info = GET_PORT_AGE_INFO(priv);
6302         age_param = flow_dv_counter_idx_get_age(dev, counter);
6303         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
6304                                          AGE_FREE, false, __ATOMIC_RELAXED,
6305                                          __ATOMIC_RELAXED)) {
6306                 /**
6307                  * We need the lock even it is age timeout,
6308                  * since counter may still in process.
6309                  */
6310                 rte_spinlock_lock(&age_info->aged_sl);
6311                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
6312                 rte_spinlock_unlock(&age_info->aged_sl);
6313                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
6314         }
6315 }
6316
6317 /**
6318  * Release a flow counter.
6319  *
6320  * @param[in] dev
6321  *   Pointer to the Ethernet device structure.
6322  * @param[in] counter
6323  *   Index to the counter handler.
6324  */
6325 static void
6326 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter)
6327 {
6328         struct mlx5_priv *priv = dev->data->dev_private;
6329         struct mlx5_flow_counter_pool *pool = NULL;
6330         struct mlx5_flow_counter *cnt;
6331         enum mlx5_counter_type cnt_type;
6332
6333         if (!counter)
6334                 return;
6335         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
6336         MLX5_ASSERT(pool);
6337         if (pool->is_aged) {
6338                 flow_dv_counter_remove_from_age(dev, counter, cnt);
6339         } else {
6340                 /*
6341                  * If the counter action is shared by indirect action API,
6342                  * the atomic function reduces its references counter.
6343                  * If after the reduction the action is still referenced, the
6344                  * function returns here and does not release it.
6345                  * When the counter action is not shared by
6346                  * indirect action API, shared info is 1 before the reduction,
6347                  * so this condition is failed and function doesn't return here.
6348                  */
6349                 if (__atomic_sub_fetch(&cnt->shared_info.refcnt, 1,
6350                                        __ATOMIC_RELAXED))
6351                         return;
6352         }
6353         cnt->pool = pool;
6354         /*
6355          * Put the counter back to list to be updated in none fallback mode.
6356          * Currently, we are using two list alternately, while one is in query,
6357          * add the freed counter to the other list based on the pool query_gen
6358          * value. After query finishes, add counter the list to the global
6359          * container counter list. The list changes while query starts. In
6360          * this case, lock will not be needed as query callback and release
6361          * function both operate with the different list.
6362          */
6363         if (!priv->sh->cmng.counter_fallback) {
6364                 rte_spinlock_lock(&pool->csl);
6365                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
6366                 rte_spinlock_unlock(&pool->csl);
6367         } else {
6368                 cnt->dcs_when_free = cnt->dcs_when_active;
6369                 cnt_type = pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
6370                                            MLX5_COUNTER_TYPE_ORIGIN;
6371                 rte_spinlock_lock(&priv->sh->cmng.csl[cnt_type]);
6372                 TAILQ_INSERT_TAIL(&priv->sh->cmng.counters[cnt_type],
6373                                   cnt, next);
6374                 rte_spinlock_unlock(&priv->sh->cmng.csl[cnt_type]);
6375         }
6376 }
6377
6378 /**
6379  * Resize a meter id container.
6380  *
6381  * @param[in] dev
6382  *   Pointer to the Ethernet device structure.
6383  *
6384  * @return
6385  *   0 on success, otherwise negative errno value and rte_errno is set.
6386  */
6387 static int
6388 flow_dv_mtr_container_resize(struct rte_eth_dev *dev)
6389 {
6390         struct mlx5_priv *priv = dev->data->dev_private;
6391         struct mlx5_aso_mtr_pools_mng *pools_mng =
6392                                 &priv->sh->mtrmng->pools_mng;
6393         void *old_pools = pools_mng->pools;
6394         uint32_t resize = pools_mng->n + MLX5_MTRS_CONTAINER_RESIZE;
6395         uint32_t mem_size = sizeof(struct mlx5_aso_mtr_pool *) * resize;
6396         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
6397
6398         if (!pools) {
6399                 rte_errno = ENOMEM;
6400                 return -ENOMEM;
6401         }
6402         if (!pools_mng->n)
6403                 if (mlx5_aso_queue_init(priv->sh, ASO_OPC_MOD_POLICER)) {
6404                         mlx5_free(pools);
6405                         return -ENOMEM;
6406                 }
6407         if (old_pools)
6408                 memcpy(pools, old_pools, pools_mng->n *
6409                                        sizeof(struct mlx5_aso_mtr_pool *));
6410         pools_mng->n = resize;
6411         pools_mng->pools = pools;
6412         if (old_pools)
6413                 mlx5_free(old_pools);
6414         return 0;
6415 }
6416
6417 /**
6418  * Prepare a new meter and/or a new meter pool.
6419  *
6420  * @param[in] dev
6421  *   Pointer to the Ethernet device structure.
6422  * @param[out] mtr_free
6423  *   Where to put the pointer of a new meter.g.
6424  *
6425  * @return
6426  *   The meter pool pointer and @mtr_free is set on success,
6427  *   NULL otherwise and rte_errno is set.
6428  */
6429 static struct mlx5_aso_mtr_pool *
6430 flow_dv_mtr_pool_create(struct rte_eth_dev *dev, struct mlx5_aso_mtr **mtr_free)
6431 {
6432         struct mlx5_priv *priv = dev->data->dev_private;
6433         struct mlx5_aso_mtr_pools_mng *pools_mng = &priv->sh->mtrmng->pools_mng;
6434         struct mlx5_aso_mtr_pool *pool = NULL;
6435         struct mlx5_devx_obj *dcs = NULL;
6436         uint32_t i;
6437         uint32_t log_obj_size;
6438
6439         log_obj_size = rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1);
6440         dcs = mlx5_devx_cmd_create_flow_meter_aso_obj(priv->sh->cdev->ctx,
6441                                                       priv->sh->cdev->pdn,
6442                                                       log_obj_size);
6443         if (!dcs) {
6444                 rte_errno = ENODATA;
6445                 return NULL;
6446         }
6447         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
6448         if (!pool) {
6449                 rte_errno = ENOMEM;
6450                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6451                 return NULL;
6452         }
6453         pool->devx_obj = dcs;
6454         rte_rwlock_write_lock(&pools_mng->resize_mtrwl);
6455         pool->index = pools_mng->n_valid;
6456         if (pool->index == pools_mng->n && flow_dv_mtr_container_resize(dev)) {
6457                 mlx5_free(pool);
6458                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6459                 rte_rwlock_write_unlock(&pools_mng->resize_mtrwl);
6460                 return NULL;
6461         }
6462         pools_mng->pools[pool->index] = pool;
6463         pools_mng->n_valid++;
6464         rte_rwlock_write_unlock(&pools_mng->resize_mtrwl);
6465         for (i = 1; i < MLX5_ASO_MTRS_PER_POOL; ++i) {
6466                 pool->mtrs[i].offset = i;
6467                 LIST_INSERT_HEAD(&pools_mng->meters, &pool->mtrs[i], next);
6468         }
6469         pool->mtrs[0].offset = 0;
6470         *mtr_free = &pool->mtrs[0];
6471         return pool;
6472 }
6473
6474 /**
6475  * Release a flow meter into pool.
6476  *
6477  * @param[in] dev
6478  *   Pointer to the Ethernet device structure.
6479  * @param[in] mtr_idx
6480  *   Index to aso flow meter.
6481  */
6482 static void
6483 flow_dv_aso_mtr_release_to_pool(struct rte_eth_dev *dev, uint32_t mtr_idx)
6484 {
6485         struct mlx5_priv *priv = dev->data->dev_private;
6486         struct mlx5_aso_mtr_pools_mng *pools_mng =
6487                                 &priv->sh->mtrmng->pools_mng;
6488         struct mlx5_aso_mtr *aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
6489
6490         MLX5_ASSERT(aso_mtr);
6491         rte_spinlock_lock(&pools_mng->mtrsl);
6492         memset(&aso_mtr->fm, 0, sizeof(struct mlx5_flow_meter_info));
6493         aso_mtr->state = ASO_METER_FREE;
6494         LIST_INSERT_HEAD(&pools_mng->meters, aso_mtr, next);
6495         rte_spinlock_unlock(&pools_mng->mtrsl);
6496 }
6497
6498 /**
6499  * Allocate a aso flow meter.
6500  *
6501  * @param[in] dev
6502  *   Pointer to the Ethernet device structure.
6503  *
6504  * @return
6505  *   Index to aso flow meter on success, 0 otherwise and rte_errno is set.
6506  */
6507 static uint32_t
6508 flow_dv_mtr_alloc(struct rte_eth_dev *dev)
6509 {
6510         struct mlx5_priv *priv = dev->data->dev_private;
6511         struct mlx5_aso_mtr *mtr_free = NULL;
6512         struct mlx5_aso_mtr_pools_mng *pools_mng =
6513                                 &priv->sh->mtrmng->pools_mng;
6514         struct mlx5_aso_mtr_pool *pool;
6515         uint32_t mtr_idx = 0;
6516
6517         if (!priv->sh->cdev->config.devx) {
6518                 rte_errno = ENOTSUP;
6519                 return 0;
6520         }
6521         /* Allocate the flow meter memory. */
6522         /* Get free meters from management. */
6523         rte_spinlock_lock(&pools_mng->mtrsl);
6524         mtr_free = LIST_FIRST(&pools_mng->meters);
6525         if (mtr_free)
6526                 LIST_REMOVE(mtr_free, next);
6527         if (!mtr_free && !flow_dv_mtr_pool_create(dev, &mtr_free)) {
6528                 rte_spinlock_unlock(&pools_mng->mtrsl);
6529                 return 0;
6530         }
6531         mtr_free->state = ASO_METER_WAIT;
6532         rte_spinlock_unlock(&pools_mng->mtrsl);
6533         pool = container_of(mtr_free,
6534                         struct mlx5_aso_mtr_pool,
6535                         mtrs[mtr_free->offset]);
6536         mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset);
6537         if (!mtr_free->fm.meter_action_g) {
6538 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
6539                 struct rte_flow_error error;
6540                 uint8_t reg_id;
6541
6542                 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &error);
6543                 mtr_free->fm.meter_action_g =
6544                         mlx5_glue->dv_create_flow_action_aso
6545                                                 (priv->sh->rx_domain,
6546                                                  pool->devx_obj->obj,
6547                                                  mtr_free->offset,
6548                                                  (1 << MLX5_FLOW_COLOR_GREEN),
6549                                                  reg_id - REG_C_0);
6550 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
6551                 if (!mtr_free->fm.meter_action_g) {
6552                         flow_dv_aso_mtr_release_to_pool(dev, mtr_idx);
6553                         return 0;
6554                 }
6555         }
6556         return mtr_idx;
6557 }
6558
6559 /**
6560  * Verify the @p attributes will be correctly understood by the NIC and store
6561  * them in the @p flow if everything is correct.
6562  *
6563  * @param[in] dev
6564  *   Pointer to dev struct.
6565  * @param[in] attributes
6566  *   Pointer to flow attributes
6567  * @param[in] external
6568  *   This flow rule is created by request external to PMD.
6569  * @param[out] error
6570  *   Pointer to error structure.
6571  *
6572  * @return
6573  *   - 0 on success and non root table.
6574  *   - 1 on success and root table.
6575  *   - a negative errno value otherwise and rte_errno is set.
6576  */
6577 static int
6578 flow_dv_validate_attributes(struct rte_eth_dev *dev,
6579                             const struct mlx5_flow_tunnel *tunnel,
6580                             const struct rte_flow_attr *attributes,
6581                             const struct flow_grp_info *grp_info,
6582                             struct rte_flow_error *error)
6583 {
6584         struct mlx5_priv *priv = dev->data->dev_private;
6585         uint32_t lowest_priority = mlx5_get_lowest_priority(dev, attributes);
6586         int ret = 0;
6587
6588 #ifndef HAVE_MLX5DV_DR
6589         RTE_SET_USED(tunnel);
6590         RTE_SET_USED(grp_info);
6591         if (attributes->group)
6592                 return rte_flow_error_set(error, ENOTSUP,
6593                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6594                                           NULL,
6595                                           "groups are not supported");
6596 #else
6597         uint32_t table = 0;
6598
6599         ret = mlx5_flow_group_to_table(dev, tunnel, attributes->group, &table,
6600                                        grp_info, error);
6601         if (ret)
6602                 return ret;
6603         if (!table)
6604                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
6605 #endif
6606         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
6607             attributes->priority > lowest_priority)
6608                 return rte_flow_error_set(error, ENOTSUP,
6609                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
6610                                           NULL,
6611                                           "priority out of range");
6612         if (attributes->transfer) {
6613                 if (!priv->sh->config.dv_esw_en)
6614                         return rte_flow_error_set
6615                                 (error, ENOTSUP,
6616                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6617                                  "E-Switch dr is not supported");
6618                 if (attributes->egress)
6619                         return rte_flow_error_set
6620                                 (error, ENOTSUP,
6621                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
6622                                  "egress is not supported");
6623         }
6624         if (!(attributes->egress ^ attributes->ingress))
6625                 return rte_flow_error_set(error, ENOTSUP,
6626                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
6627                                           "must specify exactly one of "
6628                                           "ingress or egress");
6629         return ret;
6630 }
6631
6632 static int
6633 validate_integrity_bits(const struct rte_flow_item_integrity *mask,
6634                         int64_t pattern_flags, uint64_t l3_flags,
6635                         uint64_t l4_flags, uint64_t ip4_flag,
6636                         struct rte_flow_error *error)
6637 {
6638         if (mask->l3_ok && !(pattern_flags & l3_flags))
6639                 return rte_flow_error_set(error, EINVAL,
6640                                           RTE_FLOW_ERROR_TYPE_ITEM,
6641                                           NULL, "missing L3 protocol");
6642
6643         if (mask->ipv4_csum_ok && !(pattern_flags & ip4_flag))
6644                 return rte_flow_error_set(error, EINVAL,
6645                                           RTE_FLOW_ERROR_TYPE_ITEM,
6646                                           NULL, "missing IPv4 protocol");
6647
6648         if ((mask->l4_ok || mask->l4_csum_ok) && !(pattern_flags & l4_flags))
6649                 return rte_flow_error_set(error, EINVAL,
6650                                           RTE_FLOW_ERROR_TYPE_ITEM,
6651                                           NULL, "missing L4 protocol");
6652
6653         return 0;
6654 }
6655
6656 static int
6657 flow_dv_validate_item_integrity_post(const struct
6658                                      rte_flow_item *integrity_items[2],
6659                                      int64_t pattern_flags,
6660                                      struct rte_flow_error *error)
6661 {
6662         const struct rte_flow_item_integrity *mask;
6663         int ret;
6664
6665         if (pattern_flags & MLX5_FLOW_ITEM_OUTER_INTEGRITY) {
6666                 mask = (typeof(mask))integrity_items[0]->mask;
6667                 ret = validate_integrity_bits(mask, pattern_flags,
6668                                               MLX5_FLOW_LAYER_OUTER_L3,
6669                                               MLX5_FLOW_LAYER_OUTER_L4,
6670                                               MLX5_FLOW_LAYER_OUTER_L3_IPV4,
6671                                               error);
6672                 if (ret)
6673                         return ret;
6674         }
6675         if (pattern_flags & MLX5_FLOW_ITEM_INNER_INTEGRITY) {
6676                 mask = (typeof(mask))integrity_items[1]->mask;
6677                 ret = validate_integrity_bits(mask, pattern_flags,
6678                                               MLX5_FLOW_LAYER_INNER_L3,
6679                                               MLX5_FLOW_LAYER_INNER_L4,
6680                                               MLX5_FLOW_LAYER_INNER_L3_IPV4,
6681                                               error);
6682                 if (ret)
6683                         return ret;
6684         }
6685         return 0;
6686 }
6687
6688 static int
6689 flow_dv_validate_item_integrity(struct rte_eth_dev *dev,
6690                                 const struct rte_flow_item *integrity_item,
6691                                 uint64_t pattern_flags, uint64_t *last_item,
6692                                 const struct rte_flow_item *integrity_items[2],
6693                                 struct rte_flow_error *error)
6694 {
6695         struct mlx5_priv *priv = dev->data->dev_private;
6696         const struct rte_flow_item_integrity *mask = (typeof(mask))
6697                                                      integrity_item->mask;
6698         const struct rte_flow_item_integrity *spec = (typeof(spec))
6699                                                      integrity_item->spec;
6700
6701         if (!priv->sh->cdev->config.hca_attr.pkt_integrity_match)
6702                 return rte_flow_error_set(error, ENOTSUP,
6703                                           RTE_FLOW_ERROR_TYPE_ITEM,
6704                                           integrity_item,
6705                                           "packet integrity integrity_item not supported");
6706         if (!spec)
6707                 return rte_flow_error_set(error, ENOTSUP,
6708                                           RTE_FLOW_ERROR_TYPE_ITEM,
6709                                           integrity_item,
6710                                           "no spec for integrity item");
6711         if (!mask)
6712                 mask = &rte_flow_item_integrity_mask;
6713         if (!mlx5_validate_integrity_item(mask))
6714                 return rte_flow_error_set(error, ENOTSUP,
6715                                           RTE_FLOW_ERROR_TYPE_ITEM,
6716                                           integrity_item,
6717                                           "unsupported integrity filter");
6718         if (spec->level > 1) {
6719                 if (pattern_flags & MLX5_FLOW_ITEM_INNER_INTEGRITY)
6720                         return rte_flow_error_set
6721                                 (error, ENOTSUP,
6722                                  RTE_FLOW_ERROR_TYPE_ITEM,
6723                                  NULL, "multiple inner integrity items not supported");
6724                 integrity_items[1] = integrity_item;
6725                 *last_item |= MLX5_FLOW_ITEM_INNER_INTEGRITY;
6726         } else {
6727                 if (pattern_flags & MLX5_FLOW_ITEM_OUTER_INTEGRITY)
6728                         return rte_flow_error_set
6729                                 (error, ENOTSUP,
6730                                  RTE_FLOW_ERROR_TYPE_ITEM,
6731                                  NULL, "multiple outer integrity items not supported");
6732                 integrity_items[0] = integrity_item;
6733                 *last_item |= MLX5_FLOW_ITEM_OUTER_INTEGRITY;
6734         }
6735         return 0;
6736 }
6737
6738 static int
6739 flow_dv_validate_item_flex(struct rte_eth_dev *dev,
6740                            const struct rte_flow_item *item,
6741                            uint64_t item_flags,
6742                            uint64_t *last_item,
6743                            bool is_inner,
6744                            struct rte_flow_error *error)
6745 {
6746         const struct rte_flow_item_flex *flow_spec = item->spec;
6747         const struct rte_flow_item_flex *flow_mask = item->mask;
6748         struct mlx5_flex_item *flex;
6749
6750         if (!flow_spec)
6751                 return rte_flow_error_set(error, EINVAL,
6752                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
6753                                           "flex flow item spec cannot be NULL");
6754         if (!flow_mask)
6755                 return rte_flow_error_set(error, EINVAL,
6756                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
6757                                           "flex flow item mask cannot be NULL");
6758         if (item->last)
6759                 return rte_flow_error_set(error, ENOTSUP,
6760                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
6761                                           "flex flow item last not supported");
6762         if (mlx5_flex_acquire_index(dev, flow_spec->handle, false) < 0)
6763                 return rte_flow_error_set(error, EINVAL,
6764                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
6765                                           "invalid flex flow item handle");
6766         flex = (struct mlx5_flex_item *)flow_spec->handle;
6767         switch (flex->tunnel_mode) {
6768         case FLEX_TUNNEL_MODE_SINGLE:
6769                 if (item_flags &
6770                     (MLX5_FLOW_ITEM_OUTER_FLEX | MLX5_FLOW_ITEM_INNER_FLEX))
6771                         rte_flow_error_set(error, EINVAL,
6772                                            RTE_FLOW_ERROR_TYPE_ITEM,
6773                                            NULL, "multiple flex items not supported");
6774                 break;
6775         case FLEX_TUNNEL_MODE_OUTER:
6776                 if (is_inner)
6777                         rte_flow_error_set(error, EINVAL,
6778                                            RTE_FLOW_ERROR_TYPE_ITEM,
6779                                            NULL, "inner flex item was not configured");
6780                 if (item_flags & MLX5_FLOW_ITEM_OUTER_FLEX)
6781                         rte_flow_error_set(error, ENOTSUP,
6782                                            RTE_FLOW_ERROR_TYPE_ITEM,
6783                                            NULL, "multiple flex items not supported");
6784                 break;
6785         case FLEX_TUNNEL_MODE_INNER:
6786                 if (!is_inner)
6787                         rte_flow_error_set(error, EINVAL,
6788                                            RTE_FLOW_ERROR_TYPE_ITEM,
6789                                            NULL, "outer flex item was not configured");
6790                 if (item_flags & MLX5_FLOW_ITEM_INNER_FLEX)
6791                         rte_flow_error_set(error, EINVAL,
6792                                            RTE_FLOW_ERROR_TYPE_ITEM,
6793                                            NULL, "multiple flex items not supported");
6794                 break;
6795         case FLEX_TUNNEL_MODE_MULTI:
6796                 if ((is_inner && (item_flags & MLX5_FLOW_ITEM_INNER_FLEX)) ||
6797                     (!is_inner && (item_flags & MLX5_FLOW_ITEM_OUTER_FLEX))) {
6798                         rte_flow_error_set(error, EINVAL,
6799                                            RTE_FLOW_ERROR_TYPE_ITEM,
6800                                            NULL, "multiple flex items not supported");
6801                 }
6802                 break;
6803         case FLEX_TUNNEL_MODE_TUNNEL:
6804                 if (is_inner || (item_flags & MLX5_FLOW_ITEM_FLEX_TUNNEL))
6805                         rte_flow_error_set(error, EINVAL,
6806                                            RTE_FLOW_ERROR_TYPE_ITEM,
6807                                            NULL, "multiple flex tunnel items not supported");
6808                 break;
6809         default:
6810                 rte_flow_error_set(error, EINVAL,
6811                                    RTE_FLOW_ERROR_TYPE_ITEM,
6812                                    NULL, "invalid flex item configuration");
6813         }
6814         *last_item = flex->tunnel_mode == FLEX_TUNNEL_MODE_TUNNEL ?
6815                      MLX5_FLOW_ITEM_FLEX_TUNNEL : is_inner ?
6816                      MLX5_FLOW_ITEM_INNER_FLEX : MLX5_FLOW_ITEM_OUTER_FLEX;
6817         return 0;
6818 }
6819
6820 /**
6821  * Internal validation function. For validating both actions and items.
6822  *
6823  * @param[in] dev
6824  *   Pointer to the rte_eth_dev structure.
6825  * @param[in] attr
6826  *   Pointer to the flow attributes.
6827  * @param[in] items
6828  *   Pointer to the list of items.
6829  * @param[in] actions
6830  *   Pointer to the list of actions.
6831  * @param[in] external
6832  *   This flow rule is created by request external to PMD.
6833  * @param[in] hairpin
6834  *   Number of hairpin TX actions, 0 means classic flow.
6835  * @param[out] error
6836  *   Pointer to the error structure.
6837  *
6838  * @return
6839  *   0 on success, a negative errno value otherwise and rte_errno is set.
6840  */
6841 static int
6842 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
6843                  const struct rte_flow_item items[],
6844                  const struct rte_flow_action actions[],
6845                  bool external, int hairpin, struct rte_flow_error *error)
6846 {
6847         int ret;
6848         uint64_t aso_mask, action_flags = 0;
6849         uint64_t item_flags = 0;
6850         uint64_t last_item = 0;
6851         uint8_t next_protocol = 0xff;
6852         uint16_t ether_type = 0;
6853         int actions_n = 0;
6854         uint8_t item_ipv6_proto = 0;
6855         int fdb_mirror_limit = 0;
6856         int modify_after_mirror = 0;
6857         const struct rte_flow_item *geneve_item = NULL;
6858         const struct rte_flow_item *gre_item = NULL;
6859         const struct rte_flow_item *gtp_item = NULL;
6860         const struct rte_flow_action_raw_decap *decap;
6861         const struct rte_flow_action_raw_encap *encap;
6862         const struct rte_flow_action_rss *rss = NULL;
6863         const struct rte_flow_action_rss *sample_rss = NULL;
6864         const struct rte_flow_action_count *sample_count = NULL;
6865         const struct rte_flow_item_tcp nic_tcp_mask = {
6866                 .hdr = {
6867                         .tcp_flags = 0xFF,
6868                         .src_port = RTE_BE16(UINT16_MAX),
6869                         .dst_port = RTE_BE16(UINT16_MAX),
6870                 }
6871         };
6872         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
6873                 .hdr = {
6874                         .src_addr =
6875                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6876                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6877                         .dst_addr =
6878                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6879                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6880                         .vtc_flow = RTE_BE32(0xffffffff),
6881                         .proto = 0xff,
6882                         .hop_limits = 0xff,
6883                 },
6884                 .has_frag_ext = 1,
6885         };
6886         const struct rte_flow_item_ecpri nic_ecpri_mask = {
6887                 .hdr = {
6888                         .common = {
6889                                 .u32 =
6890                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
6891                                         .type = 0xFF,
6892                                         }).u32),
6893                         },
6894                         .dummy[0] = 0xffffffff,
6895                 },
6896         };
6897         struct mlx5_priv *priv = dev->data->dev_private;
6898         struct mlx5_sh_config *dev_conf = &priv->sh->config;
6899         uint16_t queue_index = 0xFFFF;
6900         const struct rte_flow_item_vlan *vlan_m = NULL;
6901         uint32_t rw_act_num = 0;
6902         uint64_t is_root;
6903         const struct mlx5_flow_tunnel *tunnel;
6904         enum mlx5_tof_rule_type tof_rule_type;
6905         struct flow_grp_info grp_info = {
6906                 .external = !!external,
6907                 .transfer = !!attr->transfer,
6908                 .fdb_def_rule = !!priv->fdb_def_rule,
6909                 .std_tbl_fix = true,
6910         };
6911         const struct rte_eth_hairpin_conf *conf;
6912         const struct rte_flow_item *integrity_items[2] = {NULL, NULL};
6913         const struct rte_flow_item *port_id_item = NULL;
6914         bool def_policy = false;
6915         bool shared_count = false;
6916         uint16_t udp_dport = 0;
6917         uint32_t tag_id = 0;
6918         const struct rte_flow_action_age *non_shared_age = NULL;
6919         const struct rte_flow_action_count *count = NULL;
6920
6921         if (items == NULL)
6922                 return -1;
6923         tunnel = is_tunnel_offload_active(dev) ?
6924                  mlx5_get_tof(items, actions, &tof_rule_type) : NULL;
6925         if (tunnel) {
6926                 if (!dev_conf->dv_flow_en)
6927                         return rte_flow_error_set
6928                                 (error, ENOTSUP,
6929                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6930                                  NULL, "tunnel offload requires DV flow interface");
6931                 if (priv->representor)
6932                         return rte_flow_error_set
6933                                 (error, ENOTSUP,
6934                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6935                                  NULL, "decap not supported for VF representor");
6936                 if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_SET_RULE)
6937                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
6938                 else if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_MATCH_RULE)
6939                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_MATCH |
6940                                         MLX5_FLOW_ACTION_DECAP;
6941                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
6942                                         (dev, attr, tunnel, tof_rule_type);
6943         }
6944         ret = flow_dv_validate_attributes(dev, tunnel, attr, &grp_info, error);
6945         if (ret < 0)
6946                 return ret;
6947         is_root = (uint64_t)ret;
6948         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
6949                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
6950                 int type = items->type;
6951
6952                 if (!mlx5_flow_os_item_supported(type))
6953                         return rte_flow_error_set(error, ENOTSUP,
6954                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6955                                                   NULL, "item not supported");
6956                 switch (type) {
6957                 case RTE_FLOW_ITEM_TYPE_VOID:
6958                         break;
6959                 case RTE_FLOW_ITEM_TYPE_ESP:
6960                         ret = mlx5_flow_os_validate_item_esp(items, item_flags,
6961                                                           next_protocol,
6962                                                           error);
6963                         if (ret < 0)
6964                                 return ret;
6965                         last_item = MLX5_FLOW_ITEM_ESP;
6966                         break;
6967                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
6968                         ret = flow_dv_validate_item_port_id
6969                                         (dev, items, attr, item_flags, error);
6970                         if (ret < 0)
6971                                 return ret;
6972                         last_item = MLX5_FLOW_ITEM_PORT_ID;
6973                         port_id_item = items;
6974                         break;
6975                 case RTE_FLOW_ITEM_TYPE_ETH:
6976                         ret = mlx5_flow_validate_item_eth(items, item_flags,
6977                                                           true, error);
6978                         if (ret < 0)
6979                                 return ret;
6980                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
6981                                              MLX5_FLOW_LAYER_OUTER_L2;
6982                         if (items->mask != NULL && items->spec != NULL) {
6983                                 ether_type =
6984                                         ((const struct rte_flow_item_eth *)
6985                                          items->spec)->type;
6986                                 ether_type &=
6987                                         ((const struct rte_flow_item_eth *)
6988                                          items->mask)->type;
6989                                 ether_type = rte_be_to_cpu_16(ether_type);
6990                         } else {
6991                                 ether_type = 0;
6992                         }
6993                         break;
6994                 case RTE_FLOW_ITEM_TYPE_VLAN:
6995                         ret = flow_dv_validate_item_vlan(items, item_flags,
6996                                                          dev, error);
6997                         if (ret < 0)
6998                                 return ret;
6999                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
7000                                              MLX5_FLOW_LAYER_OUTER_VLAN;
7001                         if (items->mask != NULL && items->spec != NULL) {
7002                                 ether_type =
7003                                         ((const struct rte_flow_item_vlan *)
7004                                          items->spec)->inner_type;
7005                                 ether_type &=
7006                                         ((const struct rte_flow_item_vlan *)
7007                                          items->mask)->inner_type;
7008                                 ether_type = rte_be_to_cpu_16(ether_type);
7009                         } else {
7010                                 ether_type = 0;
7011                         }
7012                         /* Store outer VLAN mask for of_push_vlan action. */
7013                         if (!tunnel)
7014                                 vlan_m = items->mask;
7015                         break;
7016                 case RTE_FLOW_ITEM_TYPE_IPV4:
7017                         mlx5_flow_tunnel_ip_check(items, next_protocol,
7018                                                   &item_flags, &tunnel);
7019                         ret = flow_dv_validate_item_ipv4(dev, items, item_flags,
7020                                                          last_item, ether_type,
7021                                                          error);
7022                         if (ret < 0)
7023                                 return ret;
7024                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
7025                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
7026                         if (items->mask != NULL &&
7027                             ((const struct rte_flow_item_ipv4 *)
7028                              items->mask)->hdr.next_proto_id) {
7029                                 next_protocol =
7030                                         ((const struct rte_flow_item_ipv4 *)
7031                                          (items->spec))->hdr.next_proto_id;
7032                                 next_protocol &=
7033                                         ((const struct rte_flow_item_ipv4 *)
7034                                          (items->mask))->hdr.next_proto_id;
7035                         } else {
7036                                 /* Reset for inner layer. */
7037                                 next_protocol = 0xff;
7038                         }
7039                         break;
7040                 case RTE_FLOW_ITEM_TYPE_IPV6:
7041                         mlx5_flow_tunnel_ip_check(items, next_protocol,
7042                                                   &item_flags, &tunnel);
7043                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
7044                                                            last_item,
7045                                                            ether_type,
7046                                                            &nic_ipv6_mask,
7047                                                            error);
7048                         if (ret < 0)
7049                                 return ret;
7050                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
7051                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
7052                         if (items->mask != NULL &&
7053                             ((const struct rte_flow_item_ipv6 *)
7054                              items->mask)->hdr.proto) {
7055                                 item_ipv6_proto =
7056                                         ((const struct rte_flow_item_ipv6 *)
7057                                          items->spec)->hdr.proto;
7058                                 next_protocol =
7059                                         ((const struct rte_flow_item_ipv6 *)
7060                                          items->spec)->hdr.proto;
7061                                 next_protocol &=
7062                                         ((const struct rte_flow_item_ipv6 *)
7063                                          items->mask)->hdr.proto;
7064                         } else {
7065                                 /* Reset for inner layer. */
7066                                 next_protocol = 0xff;
7067                         }
7068                         break;
7069                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
7070                         ret = flow_dv_validate_item_ipv6_frag_ext(items,
7071                                                                   item_flags,
7072                                                                   error);
7073                         if (ret < 0)
7074                                 return ret;
7075                         last_item = tunnel ?
7076                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
7077                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
7078                         if (items->mask != NULL &&
7079                             ((const struct rte_flow_item_ipv6_frag_ext *)
7080                              items->mask)->hdr.next_header) {
7081                                 next_protocol =
7082                                 ((const struct rte_flow_item_ipv6_frag_ext *)
7083                                  items->spec)->hdr.next_header;
7084                                 next_protocol &=
7085                                 ((const struct rte_flow_item_ipv6_frag_ext *)
7086                                  items->mask)->hdr.next_header;
7087                         } else {
7088                                 /* Reset for inner layer. */
7089                                 next_protocol = 0xff;
7090                         }
7091                         break;
7092                 case RTE_FLOW_ITEM_TYPE_TCP:
7093                         ret = mlx5_flow_validate_item_tcp
7094                                                 (items, item_flags,
7095                                                  next_protocol,
7096                                                  &nic_tcp_mask,
7097                                                  error);
7098                         if (ret < 0)
7099                                 return ret;
7100                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
7101                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
7102                         break;
7103                 case RTE_FLOW_ITEM_TYPE_UDP:
7104                         ret = mlx5_flow_validate_item_udp(items, item_flags,
7105                                                           next_protocol,
7106                                                           error);
7107                         const struct rte_flow_item_udp *spec = items->spec;
7108                         const struct rte_flow_item_udp *mask = items->mask;
7109                         if (!mask)
7110                                 mask = &rte_flow_item_udp_mask;
7111                         if (spec != NULL)
7112                                 udp_dport = rte_be_to_cpu_16
7113                                                 (spec->hdr.dst_port &
7114                                                  mask->hdr.dst_port);
7115                         if (ret < 0)
7116                                 return ret;
7117                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
7118                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
7119                         break;
7120                 case RTE_FLOW_ITEM_TYPE_GRE:
7121                         ret = mlx5_flow_validate_item_gre(items, item_flags,
7122                                                           next_protocol, error);
7123                         if (ret < 0)
7124                                 return ret;
7125                         gre_item = items;
7126                         last_item = MLX5_FLOW_LAYER_GRE;
7127                         break;
7128                 case RTE_FLOW_ITEM_TYPE_GRE_OPTION:
7129                         ret = mlx5_flow_validate_item_gre_option(dev, items, item_flags,
7130                                                           attr, gre_item, error);
7131                         if (ret < 0)
7132                                 return ret;
7133                         last_item = MLX5_FLOW_LAYER_GRE;
7134                         break;
7135                 case RTE_FLOW_ITEM_TYPE_NVGRE:
7136                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
7137                                                             next_protocol,
7138                                                             error);
7139                         if (ret < 0)
7140                                 return ret;
7141                         last_item = MLX5_FLOW_LAYER_NVGRE;
7142                         break;
7143                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
7144                         ret = mlx5_flow_validate_item_gre_key
7145                                 (items, item_flags, gre_item, error);
7146                         if (ret < 0)
7147                                 return ret;
7148                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
7149                         break;
7150                 case RTE_FLOW_ITEM_TYPE_VXLAN:
7151                         ret = mlx5_flow_validate_item_vxlan(dev, udp_dport,
7152                                                             items, item_flags,
7153                                                             attr, error);
7154                         if (ret < 0)
7155                                 return ret;
7156                         last_item = MLX5_FLOW_LAYER_VXLAN;
7157                         break;
7158                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
7159                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
7160                                                                 item_flags, dev,
7161                                                                 error);
7162                         if (ret < 0)
7163                                 return ret;
7164                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
7165                         break;
7166                 case RTE_FLOW_ITEM_TYPE_GENEVE:
7167                         ret = mlx5_flow_validate_item_geneve(items,
7168                                                              item_flags, dev,
7169                                                              error);
7170                         if (ret < 0)
7171                                 return ret;
7172                         geneve_item = items;
7173                         last_item = MLX5_FLOW_LAYER_GENEVE;
7174                         break;
7175                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
7176                         ret = mlx5_flow_validate_item_geneve_opt(items,
7177                                                                  last_item,
7178                                                                  geneve_item,
7179                                                                  dev,
7180                                                                  error);
7181                         if (ret < 0)
7182                                 return ret;
7183                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
7184                         break;
7185                 case RTE_FLOW_ITEM_TYPE_MPLS:
7186                         ret = mlx5_flow_validate_item_mpls(dev, items,
7187                                                            item_flags,
7188                                                            last_item, error);
7189                         if (ret < 0)
7190                                 return ret;
7191                         last_item = MLX5_FLOW_LAYER_MPLS;
7192                         break;
7193
7194                 case RTE_FLOW_ITEM_TYPE_MARK:
7195                         ret = flow_dv_validate_item_mark(dev, items, attr,
7196                                                          error);
7197                         if (ret < 0)
7198                                 return ret;
7199                         last_item = MLX5_FLOW_ITEM_MARK;
7200                         break;
7201                 case RTE_FLOW_ITEM_TYPE_META:
7202                         ret = flow_dv_validate_item_meta(dev, items, attr,
7203                                                          error);
7204                         if (ret < 0)
7205                                 return ret;
7206                         last_item = MLX5_FLOW_ITEM_METADATA;
7207                         break;
7208                 case RTE_FLOW_ITEM_TYPE_ICMP:
7209                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
7210                                                            next_protocol,
7211                                                            error);
7212                         if (ret < 0)
7213                                 return ret;
7214                         last_item = MLX5_FLOW_LAYER_ICMP;
7215                         break;
7216                 case RTE_FLOW_ITEM_TYPE_ICMP6:
7217                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
7218                                                             next_protocol,
7219                                                             error);
7220                         if (ret < 0)
7221                                 return ret;
7222                         item_ipv6_proto = IPPROTO_ICMPV6;
7223                         last_item = MLX5_FLOW_LAYER_ICMP6;
7224                         break;
7225                 case RTE_FLOW_ITEM_TYPE_TAG:
7226                         ret = flow_dv_validate_item_tag(dev, items,
7227                                                         attr, error);
7228                         if (ret < 0)
7229                                 return ret;
7230                         last_item = MLX5_FLOW_ITEM_TAG;
7231                         break;
7232                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
7233                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
7234                         break;
7235                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
7236                         break;
7237                 case RTE_FLOW_ITEM_TYPE_GTP:
7238                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
7239                                                         error);
7240                         if (ret < 0)
7241                                 return ret;
7242                         gtp_item = items;
7243                         last_item = MLX5_FLOW_LAYER_GTP;
7244                         break;
7245                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
7246                         ret = flow_dv_validate_item_gtp_psc(items, last_item,
7247                                                             gtp_item, attr,
7248                                                             error);
7249                         if (ret < 0)
7250                                 return ret;
7251                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
7252                         break;
7253                 case RTE_FLOW_ITEM_TYPE_ECPRI:
7254                         /* Capacity will be checked in the translate stage. */
7255                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
7256                                                             last_item,
7257                                                             ether_type,
7258                                                             &nic_ecpri_mask,
7259                                                             error);
7260                         if (ret < 0)
7261                                 return ret;
7262                         last_item = MLX5_FLOW_LAYER_ECPRI;
7263                         break;
7264                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
7265                         ret = flow_dv_validate_item_integrity(dev, items,
7266                                                               item_flags,
7267                                                               &last_item,
7268                                                               integrity_items,
7269                                                               error);
7270                         if (ret < 0)
7271                                 return ret;
7272                         break;
7273                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
7274                         ret = flow_dv_validate_item_aso_ct(dev, items,
7275                                                            &item_flags, error);
7276                         if (ret < 0)
7277                                 return ret;
7278                         break;
7279                 case MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL:
7280                         /* tunnel offload item was processed before
7281                          * list it here as a supported type
7282                          */
7283                         break;
7284                 case RTE_FLOW_ITEM_TYPE_FLEX:
7285                         ret = flow_dv_validate_item_flex(dev, items, item_flags,
7286                                                          &last_item,
7287                                                          tunnel != 0, error);
7288                         if (ret < 0)
7289                                 return ret;
7290                         break;
7291                 default:
7292                         return rte_flow_error_set(error, ENOTSUP,
7293                                                   RTE_FLOW_ERROR_TYPE_ITEM,
7294                                                   NULL, "item not supported");
7295                 }
7296                 item_flags |= last_item;
7297         }
7298         if (item_flags & MLX5_FLOW_ITEM_INTEGRITY) {
7299                 ret = flow_dv_validate_item_integrity_post(integrity_items,
7300                                                            item_flags, error);
7301                 if (ret)
7302                         return ret;
7303         }
7304         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
7305                 int type = actions->type;
7306
7307                 if (!mlx5_flow_os_action_supported(type))
7308                         return rte_flow_error_set(error, ENOTSUP,
7309                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7310                                                   actions,
7311                                                   "action not supported");
7312                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
7313                         return rte_flow_error_set(error, ENOTSUP,
7314                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7315                                                   actions, "too many actions");
7316                 if (action_flags &
7317                         MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
7318                         return rte_flow_error_set(error, ENOTSUP,
7319                                 RTE_FLOW_ERROR_TYPE_ACTION,
7320                                 NULL, "meter action with policy "
7321                                 "must be the last action");
7322                 switch (type) {
7323                 case RTE_FLOW_ACTION_TYPE_VOID:
7324                         break;
7325                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
7326                 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
7327                         ret = flow_dv_validate_action_port_id(dev,
7328                                                               action_flags,
7329                                                               actions,
7330                                                               attr,
7331                                                               error);
7332                         if (ret)
7333                                 return ret;
7334                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
7335                         ++actions_n;
7336                         break;
7337                 case RTE_FLOW_ACTION_TYPE_FLAG:
7338                         ret = flow_dv_validate_action_flag(dev, action_flags,
7339                                                            attr, error);
7340                         if (ret < 0)
7341                                 return ret;
7342                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7343                                 /* Count all modify-header actions as one. */
7344                                 if (!(action_flags &
7345                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7346                                         ++actions_n;
7347                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
7348                                                 MLX5_FLOW_ACTION_MARK_EXT;
7349                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7350                                         modify_after_mirror = 1;
7351
7352                         } else {
7353                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
7354                                 ++actions_n;
7355                         }
7356                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7357                         break;
7358                 case RTE_FLOW_ACTION_TYPE_MARK:
7359                         ret = flow_dv_validate_action_mark(dev, actions,
7360                                                            action_flags,
7361                                                            attr, error);
7362                         if (ret < 0)
7363                                 return ret;
7364                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7365                                 /* Count all modify-header actions as one. */
7366                                 if (!(action_flags &
7367                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7368                                         ++actions_n;
7369                                 action_flags |= MLX5_FLOW_ACTION_MARK |
7370                                                 MLX5_FLOW_ACTION_MARK_EXT;
7371                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7372                                         modify_after_mirror = 1;
7373                         } else {
7374                                 action_flags |= MLX5_FLOW_ACTION_MARK;
7375                                 ++actions_n;
7376                         }
7377                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7378                         break;
7379                 case RTE_FLOW_ACTION_TYPE_SET_META:
7380                         ret = flow_dv_validate_action_set_meta(dev, actions,
7381                                                                action_flags,
7382                                                                attr, error);
7383                         if (ret < 0)
7384                                 return ret;
7385                         /* Count all modify-header actions as one action. */
7386                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7387                                 ++actions_n;
7388                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7389                                 modify_after_mirror = 1;
7390                         action_flags |= MLX5_FLOW_ACTION_SET_META;
7391                         rw_act_num += MLX5_ACT_NUM_SET_META;
7392                         break;
7393                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
7394                         ret = flow_dv_validate_action_set_tag(dev, actions,
7395                                                               action_flags,
7396                                                               attr, error);
7397                         if (ret < 0)
7398                                 return ret;
7399                         /* Count all modify-header actions as one action. */
7400                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7401                                 ++actions_n;
7402                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7403                                 modify_after_mirror = 1;
7404                         tag_id = ((const struct rte_flow_action_set_tag *)
7405                                   actions->conf)->index;
7406                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
7407                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7408                         break;
7409                 case RTE_FLOW_ACTION_TYPE_DROP:
7410                         ret = mlx5_flow_validate_action_drop(action_flags,
7411                                                              attr, error);
7412                         if (ret < 0)
7413                                 return ret;
7414                         action_flags |= MLX5_FLOW_ACTION_DROP;
7415                         ++actions_n;
7416                         break;
7417                 case RTE_FLOW_ACTION_TYPE_QUEUE:
7418                         ret = mlx5_flow_validate_action_queue(actions,
7419                                                               action_flags, dev,
7420                                                               attr, error);
7421                         if (ret < 0)
7422                                 return ret;
7423                         queue_index = ((const struct rte_flow_action_queue *)
7424                                                         (actions->conf))->index;
7425                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
7426                         ++actions_n;
7427                         break;
7428                 case RTE_FLOW_ACTION_TYPE_RSS:
7429                         rss = actions->conf;
7430                         ret = mlx5_flow_validate_action_rss(actions,
7431                                                             action_flags, dev,
7432                                                             attr, item_flags,
7433                                                             error);
7434                         if (ret < 0)
7435                                 return ret;
7436                         if (rss && sample_rss &&
7437                             (sample_rss->level != rss->level ||
7438                             sample_rss->types != rss->types))
7439                                 return rte_flow_error_set(error, ENOTSUP,
7440                                         RTE_FLOW_ERROR_TYPE_ACTION,
7441                                         NULL,
7442                                         "Can't use the different RSS types "
7443                                         "or level in the same flow");
7444                         if (rss != NULL && rss->queue_num)
7445                                 queue_index = rss->queue[0];
7446                         action_flags |= MLX5_FLOW_ACTION_RSS;
7447                         ++actions_n;
7448                         break;
7449                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
7450                         ret =
7451                         mlx5_flow_validate_action_default_miss(action_flags,
7452                                         attr, error);
7453                         if (ret < 0)
7454                                 return ret;
7455                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
7456                         ++actions_n;
7457                         break;
7458                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
7459                         shared_count = true;
7460                         /* fall-through. */
7461                 case RTE_FLOW_ACTION_TYPE_COUNT:
7462                         ret = flow_dv_validate_action_count(dev, shared_count,
7463                                                             action_flags,
7464                                                             attr, error);
7465                         if (ret < 0)
7466                                 return ret;
7467                         count = actions->conf;
7468                         action_flags |= MLX5_FLOW_ACTION_COUNT;
7469                         ++actions_n;
7470                         break;
7471                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
7472                         if (flow_dv_validate_action_pop_vlan(dev,
7473                                                              action_flags,
7474                                                              actions,
7475                                                              item_flags, attr,
7476                                                              error))
7477                                 return -rte_errno;
7478                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7479                                 modify_after_mirror = 1;
7480                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
7481                         ++actions_n;
7482                         break;
7483                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
7484                         ret = flow_dv_validate_action_push_vlan(dev,
7485                                                                 action_flags,
7486                                                                 vlan_m,
7487                                                                 actions, attr,
7488                                                                 error);
7489                         if (ret < 0)
7490                                 return ret;
7491                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7492                                 modify_after_mirror = 1;
7493                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
7494                         ++actions_n;
7495                         break;
7496                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
7497                         ret = flow_dv_validate_action_set_vlan_pcp
7498                                                 (action_flags, actions, error);
7499                         if (ret < 0)
7500                                 return ret;
7501                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7502                                 modify_after_mirror = 1;
7503                         /* Count PCP with push_vlan command. */
7504                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
7505                         break;
7506                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
7507                         ret = flow_dv_validate_action_set_vlan_vid
7508                                                 (item_flags, action_flags,
7509                                                  actions, error);
7510                         if (ret < 0)
7511                                 return ret;
7512                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7513                                 modify_after_mirror = 1;
7514                         /* Count VID with push_vlan command. */
7515                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
7516                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
7517                         break;
7518                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
7519                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
7520                         ret = flow_dv_validate_action_l2_encap(dev,
7521                                                                action_flags,
7522                                                                actions, attr,
7523                                                                error);
7524                         if (ret < 0)
7525                                 return ret;
7526                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
7527                         ++actions_n;
7528                         break;
7529                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
7530                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
7531                         ret = flow_dv_validate_action_decap(dev, action_flags,
7532                                                             actions, item_flags,
7533                                                             attr, error);
7534                         if (ret < 0)
7535                                 return ret;
7536                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7537                                 modify_after_mirror = 1;
7538                         action_flags |= MLX5_FLOW_ACTION_DECAP;
7539                         ++actions_n;
7540                         break;
7541                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
7542                         ret = flow_dv_validate_action_raw_encap_decap
7543                                 (dev, NULL, actions->conf, attr, &action_flags,
7544                                  &actions_n, actions, item_flags, error);
7545                         if (ret < 0)
7546                                 return ret;
7547                         break;
7548                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
7549                         decap = actions->conf;
7550                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
7551                                 ;
7552                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
7553                                 encap = NULL;
7554                                 actions--;
7555                         } else {
7556                                 encap = actions->conf;
7557                         }
7558                         ret = flow_dv_validate_action_raw_encap_decap
7559                                            (dev,
7560                                             decap ? decap : &empty_decap, encap,
7561                                             attr, &action_flags, &actions_n,
7562                                             actions, item_flags, error);
7563                         if (ret < 0)
7564                                 return ret;
7565                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7566                             (action_flags & MLX5_FLOW_ACTION_DECAP))
7567                                 modify_after_mirror = 1;
7568                         break;
7569                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
7570                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
7571                         ret = flow_dv_validate_action_modify_mac(action_flags,
7572                                                                  actions,
7573                                                                  item_flags,
7574                                                                  error);
7575                         if (ret < 0)
7576                                 return ret;
7577                         /* Count all modify-header actions as one action. */
7578                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7579                                 ++actions_n;
7580                         action_flags |= actions->type ==
7581                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
7582                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
7583                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
7584                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7585                                 modify_after_mirror = 1;
7586                         /*
7587                          * Even if the source and destination MAC addresses have
7588                          * overlap in the header with 4B alignment, the convert
7589                          * function will handle them separately and 4 SW actions
7590                          * will be created. And 2 actions will be added each
7591                          * time no matter how many bytes of address will be set.
7592                          */
7593                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
7594                         break;
7595                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
7596                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
7597                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
7598                                                                   actions,
7599                                                                   item_flags,
7600                                                                   error);
7601                         if (ret < 0)
7602                                 return ret;
7603                         /* Count all modify-header actions as one action. */
7604                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7605                                 ++actions_n;
7606                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7607                                 modify_after_mirror = 1;
7608                         action_flags |= actions->type ==
7609                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
7610                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
7611                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
7612                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
7613                         break;
7614                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
7615                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
7616                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
7617                                                                   actions,
7618                                                                   item_flags,
7619                                                                   error);
7620                         if (ret < 0)
7621                                 return ret;
7622                         if (item_ipv6_proto == IPPROTO_ICMPV6)
7623                                 return rte_flow_error_set(error, ENOTSUP,
7624                                         RTE_FLOW_ERROR_TYPE_ACTION,
7625                                         actions,
7626                                         "Can't change header "
7627                                         "with ICMPv6 proto");
7628                         /* Count all modify-header actions as one action. */
7629                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7630                                 ++actions_n;
7631                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7632                                 modify_after_mirror = 1;
7633                         action_flags |= actions->type ==
7634                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
7635                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
7636                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
7637                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
7638                         break;
7639                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
7640                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
7641                         ret = flow_dv_validate_action_modify_tp(action_flags,
7642                                                                 actions,
7643                                                                 item_flags,
7644                                                                 error);
7645                         if (ret < 0)
7646                                 return ret;
7647                         /* Count all modify-header actions as one action. */
7648                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7649                                 ++actions_n;
7650                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7651                                 modify_after_mirror = 1;
7652                         action_flags |= actions->type ==
7653                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
7654                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
7655                                                 MLX5_FLOW_ACTION_SET_TP_DST;
7656                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
7657                         break;
7658                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
7659                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
7660                         ret = flow_dv_validate_action_modify_ttl(action_flags,
7661                                                                  actions,
7662                                                                  item_flags,
7663                                                                  error);
7664                         if (ret < 0)
7665                                 return ret;
7666                         /* Count all modify-header actions as one action. */
7667                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7668                                 ++actions_n;
7669                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7670                                 modify_after_mirror = 1;
7671                         action_flags |= actions->type ==
7672                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
7673                                                 MLX5_FLOW_ACTION_SET_TTL :
7674                                                 MLX5_FLOW_ACTION_DEC_TTL;
7675                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
7676                         break;
7677                 case RTE_FLOW_ACTION_TYPE_JUMP:
7678                         ret = flow_dv_validate_action_jump(dev, tunnel, actions,
7679                                                            action_flags,
7680                                                            attr, external,
7681                                                            error);
7682                         if (ret)
7683                                 return ret;
7684                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7685                             fdb_mirror_limit)
7686                                 return rte_flow_error_set(error, EINVAL,
7687                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7688                                                   NULL,
7689                                                   "sample and jump action combination is not supported");
7690                         ++actions_n;
7691                         action_flags |= MLX5_FLOW_ACTION_JUMP;
7692                         break;
7693                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
7694                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
7695                         ret = flow_dv_validate_action_modify_tcp_seq
7696                                                                 (action_flags,
7697                                                                  actions,
7698                                                                  item_flags,
7699                                                                  error);
7700                         if (ret < 0)
7701                                 return ret;
7702                         /* Count all modify-header actions as one action. */
7703                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7704                                 ++actions_n;
7705                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7706                                 modify_after_mirror = 1;
7707                         action_flags |= actions->type ==
7708                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
7709                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
7710                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
7711                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
7712                         break;
7713                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
7714                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
7715                         ret = flow_dv_validate_action_modify_tcp_ack
7716                                                                 (action_flags,
7717                                                                  actions,
7718                                                                  item_flags,
7719                                                                  error);
7720                         if (ret < 0)
7721                                 return ret;
7722                         /* Count all modify-header actions as one action. */
7723                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7724                                 ++actions_n;
7725                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7726                                 modify_after_mirror = 1;
7727                         action_flags |= actions->type ==
7728                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
7729                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
7730                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
7731                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
7732                         break;
7733                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
7734                         break;
7735                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
7736                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
7737                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7738                         break;
7739                 case RTE_FLOW_ACTION_TYPE_METER:
7740                         ret = mlx5_flow_validate_action_meter(dev,
7741                                                               action_flags,
7742                                                               item_flags,
7743                                                               actions, attr,
7744                                                               port_id_item,
7745                                                               &def_policy,
7746                                                               error);
7747                         if (ret < 0)
7748                                 return ret;
7749                         action_flags |= MLX5_FLOW_ACTION_METER;
7750                         if (!def_policy)
7751                                 action_flags |=
7752                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
7753                         ++actions_n;
7754                         /* Meter action will add one more TAG action. */
7755                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7756                         break;
7757                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
7758                         if (!attr->transfer && !attr->group)
7759                                 return rte_flow_error_set(error, ENOTSUP,
7760                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7761                                                                            NULL,
7762                           "Shared ASO age action is not supported for group 0");
7763                         if (action_flags & MLX5_FLOW_ACTION_AGE)
7764                                 return rte_flow_error_set
7765                                                   (error, EINVAL,
7766                                                    RTE_FLOW_ERROR_TYPE_ACTION,
7767                                                    NULL,
7768                                                    "duplicate age actions set");
7769                         action_flags |= MLX5_FLOW_ACTION_AGE;
7770                         ++actions_n;
7771                         break;
7772                 case RTE_FLOW_ACTION_TYPE_AGE:
7773                         non_shared_age = actions->conf;
7774                         ret = flow_dv_validate_action_age(action_flags,
7775                                                           actions, dev,
7776                                                           error);
7777                         if (ret < 0)
7778                                 return ret;
7779                         /*
7780                          * Validate the regular AGE action (using counter)
7781                          * mutual exclusion with indirect counter actions.
7782                          */
7783                         if (!flow_hit_aso_supported(priv->sh, attr)) {
7784                                 if (shared_count)
7785                                         return rte_flow_error_set
7786                                                 (error, EINVAL,
7787                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7788                                                 NULL,
7789                                                 "old age and indirect count combination is not supported");
7790                                 if (sample_count)
7791                                         return rte_flow_error_set
7792                                                 (error, EINVAL,
7793                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7794                                                 NULL,
7795                                                 "old age action and count must be in the same sub flow");
7796                         }
7797                         action_flags |= MLX5_FLOW_ACTION_AGE;
7798                         ++actions_n;
7799                         break;
7800                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
7801                         ret = flow_dv_validate_action_modify_ipv4_dscp
7802                                                          (action_flags,
7803                                                           actions,
7804                                                           item_flags,
7805                                                           error);
7806                         if (ret < 0)
7807                                 return ret;
7808                         /* Count all modify-header actions as one action. */
7809                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7810                                 ++actions_n;
7811                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7812                                 modify_after_mirror = 1;
7813                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
7814                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7815                         break;
7816                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
7817                         ret = flow_dv_validate_action_modify_ipv6_dscp
7818                                                                 (action_flags,
7819                                                                  actions,
7820                                                                  item_flags,
7821                                                                  error);
7822                         if (ret < 0)
7823                                 return ret;
7824                         /* Count all modify-header actions as one action. */
7825                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7826                                 ++actions_n;
7827                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7828                                 modify_after_mirror = 1;
7829                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
7830                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7831                         break;
7832                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
7833                         ret = flow_dv_validate_action_sample(&action_flags,
7834                                                              actions, dev,
7835                                                              attr, item_flags,
7836                                                              rss, &sample_rss,
7837                                                              &sample_count,
7838                                                              &fdb_mirror_limit,
7839                                                              error);
7840                         if (ret < 0)
7841                                 return ret;
7842                         if ((action_flags & MLX5_FLOW_ACTION_SET_TAG) &&
7843                             tag_id == 0 && priv->mtr_color_reg == REG_NON)
7844                                 return rte_flow_error_set(error, EINVAL,
7845                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7846                                         "sample after tag action causes metadata tag index 0 corruption");
7847                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
7848                         ++actions_n;
7849                         break;
7850                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
7851                         ret = flow_dv_validate_action_modify_field(dev,
7852                                                                    action_flags,
7853                                                                    actions,
7854                                                                    attr,
7855                                                                    error);
7856                         if (ret < 0)
7857                                 return ret;
7858                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7859                                 modify_after_mirror = 1;
7860                         /* Count all modify-header actions as one action. */
7861                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7862                                 ++actions_n;
7863                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
7864                         rw_act_num += ret;
7865                         break;
7866                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
7867                         ret = flow_dv_validate_action_aso_ct(dev, action_flags,
7868                                                              item_flags, attr,
7869                                                              error);
7870                         if (ret < 0)
7871                                 return ret;
7872                         action_flags |= MLX5_FLOW_ACTION_CT;
7873                         break;
7874                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
7875                         /* tunnel offload action was processed before
7876                          * list it here as a supported type
7877                          */
7878                         break;
7879                 default:
7880                         return rte_flow_error_set(error, ENOTSUP,
7881                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7882                                                   actions,
7883                                                   "action not supported");
7884                 }
7885         }
7886         /*
7887          * Validate actions in flow rules
7888          * - Explicit decap action is prohibited by the tunnel offload API.
7889          * - Drop action in tunnel steer rule is prohibited by the API.
7890          * - Application cannot use MARK action because it's value can mask
7891          *   tunnel default miss notification.
7892          * - JUMP in tunnel match rule has no support in current PMD
7893          *   implementation.
7894          * - TAG & META are reserved for future uses.
7895          */
7896         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_SET) {
7897                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_DECAP    |
7898                                             MLX5_FLOW_ACTION_MARK     |
7899                                             MLX5_FLOW_ACTION_SET_TAG  |
7900                                             MLX5_FLOW_ACTION_SET_META |
7901                                             MLX5_FLOW_ACTION_DROP;
7902
7903                 if (action_flags & bad_actions_mask)
7904                         return rte_flow_error_set
7905                                         (error, EINVAL,
7906                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7907                                         "Invalid RTE action in tunnel "
7908                                         "set decap rule");
7909                 if (!(action_flags & MLX5_FLOW_ACTION_JUMP))
7910                         return rte_flow_error_set
7911                                         (error, EINVAL,
7912                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7913                                         "tunnel set decap rule must terminate "
7914                                         "with JUMP");
7915                 if (!attr->ingress)
7916                         return rte_flow_error_set
7917                                         (error, EINVAL,
7918                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7919                                         "tunnel flows for ingress traffic only");
7920         }
7921         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_MATCH) {
7922                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_JUMP    |
7923                                             MLX5_FLOW_ACTION_MARK    |
7924                                             MLX5_FLOW_ACTION_SET_TAG |
7925                                             MLX5_FLOW_ACTION_SET_META;
7926
7927                 if (action_flags & bad_actions_mask)
7928                         return rte_flow_error_set
7929                                         (error, EINVAL,
7930                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7931                                         "Invalid RTE action in tunnel "
7932                                         "set match rule");
7933         }
7934         /*
7935          * Validate the drop action mutual exclusion with other actions.
7936          * Drop action is mutually-exclusive with any other action, except for
7937          * Count action.
7938          * Drop action compatibility with tunnel offload was already validated.
7939          */
7940         if (action_flags & (MLX5_FLOW_ACTION_TUNNEL_MATCH |
7941                             MLX5_FLOW_ACTION_TUNNEL_MATCH));
7942         else if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
7943             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
7944                 return rte_flow_error_set(error, EINVAL,
7945                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7946                                           "Drop action is mutually-exclusive "
7947                                           "with any other action, except for "
7948                                           "Count action");
7949         /* Eswitch has few restrictions on using items and actions */
7950         if (attr->transfer) {
7951                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7952                     action_flags & MLX5_FLOW_ACTION_FLAG)
7953                         return rte_flow_error_set(error, ENOTSUP,
7954                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7955                                                   NULL,
7956                                                   "unsupported action FLAG");
7957                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7958                     action_flags & MLX5_FLOW_ACTION_MARK)
7959                         return rte_flow_error_set(error, ENOTSUP,
7960                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7961                                                   NULL,
7962                                                   "unsupported action MARK");
7963                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
7964                         return rte_flow_error_set(error, ENOTSUP,
7965                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7966                                                   NULL,
7967                                                   "unsupported action QUEUE");
7968                 if (action_flags & MLX5_FLOW_ACTION_RSS)
7969                         return rte_flow_error_set(error, ENOTSUP,
7970                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7971                                                   NULL,
7972                                                   "unsupported action RSS");
7973                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
7974                         return rte_flow_error_set(error, EINVAL,
7975                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7976                                                   actions,
7977                                                   "no fate action is found");
7978         } else {
7979                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
7980                         return rte_flow_error_set(error, EINVAL,
7981                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7982                                                   actions,
7983                                                   "no fate action is found");
7984         }
7985         /*
7986          * Continue validation for Xcap and VLAN actions.
7987          * If hairpin is working in explicit TX rule mode, there is no actions
7988          * splitting and the validation of hairpin ingress flow should be the
7989          * same as other standard flows.
7990          */
7991         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
7992                              MLX5_FLOW_VLAN_ACTIONS)) &&
7993             (queue_index == 0xFFFF || !mlx5_rxq_is_hairpin(dev, queue_index) ||
7994              ((conf = mlx5_rxq_get_hairpin_conf(dev, queue_index)) != NULL &&
7995              conf->tx_explicit != 0))) {
7996                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
7997                     MLX5_FLOW_XCAP_ACTIONS)
7998                         return rte_flow_error_set(error, ENOTSUP,
7999                                                   RTE_FLOW_ERROR_TYPE_ACTION,
8000                                                   NULL, "encap and decap "
8001                                                   "combination aren't supported");
8002                 /* Push VLAN is not supported in ingress except for NICs newer than CX5. */
8003                 if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) {
8004                         struct mlx5_dev_ctx_shared *sh = priv->sh;
8005                         bool direction_error = false;
8006
8007                         if (attr->transfer) {
8008                                 bool fdb_tx = priv->representor_id != UINT16_MAX;
8009                                 bool is_cx5 = sh->steering_format_version ==
8010                                     MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5;
8011
8012                                 if (!fdb_tx && is_cx5)
8013                                         direction_error = true;
8014                         } else if (attr->ingress) {
8015                                 direction_error = true;
8016                         }
8017                         if (direction_error)
8018                                 return rte_flow_error_set(error, ENOTSUP,
8019                                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
8020                                                           NULL,
8021                                                           "push VLAN action not supported "
8022                                                           "for ingress");
8023                 }
8024                 if (!attr->transfer && attr->ingress) {
8025                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
8026                                 return rte_flow_error_set
8027                                                 (error, ENOTSUP,
8028                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8029                                                  NULL, "encap is not supported"
8030                                                  " for ingress traffic");
8031                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
8032                                         MLX5_FLOW_VLAN_ACTIONS)
8033                                 return rte_flow_error_set
8034                                                 (error, ENOTSUP,
8035                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8036                                                  NULL, "no support for "
8037                                                  "multiple VLAN actions");
8038                 }
8039         }
8040         if (action_flags & MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY) {
8041                 if ((action_flags & (MLX5_FLOW_FATE_ACTIONS &
8042                         ~MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)) &&
8043                         attr->ingress)
8044                         return rte_flow_error_set
8045                                 (error, ENOTSUP,
8046                                 RTE_FLOW_ERROR_TYPE_ACTION,
8047                                 NULL, "fate action not supported for "
8048                                 "meter with policy");
8049                 if (attr->egress) {
8050                         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
8051                                 return rte_flow_error_set
8052                                         (error, ENOTSUP,
8053                                         RTE_FLOW_ERROR_TYPE_ACTION,
8054                                         NULL, "modify header action in egress "
8055                                         "cannot be done before meter action");
8056                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
8057                                 return rte_flow_error_set
8058                                         (error, ENOTSUP,
8059                                         RTE_FLOW_ERROR_TYPE_ACTION,
8060                                         NULL, "encap action in egress "
8061                                         "cannot be done before meter action");
8062                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
8063                                 return rte_flow_error_set
8064                                         (error, ENOTSUP,
8065                                         RTE_FLOW_ERROR_TYPE_ACTION,
8066                                         NULL, "push vlan action in egress "
8067                                         "cannot be done before meter action");
8068                 }
8069         }
8070         /*
8071          * Only support one ASO action in a single flow rule.
8072          * non-shared AGE + counter will fallback to use HW counter, no ASO hit object.
8073          * Group 0 uses HW counter for AGE too even if no counter action.
8074          */
8075         aso_mask = (action_flags & MLX5_FLOW_ACTION_METER && priv->sh->meter_aso_en) << 2 |
8076                    (action_flags & MLX5_FLOW_ACTION_CT && priv->sh->ct_aso_en) << 1 |
8077                    (action_flags & MLX5_FLOW_ACTION_AGE &&
8078                     !(non_shared_age && count) &&
8079                     (attr->group || (attr->transfer && priv->fdb_def_rule)) &&
8080                     priv->sh->flow_hit_aso_en);
8081         if (__builtin_popcountl(aso_mask) > 1)
8082                 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
8083                                           NULL, "unsupported combining AGE, METER, CT ASO actions in a single rule");
8084         /*
8085          * Hairpin flow will add one more TAG action in TX implicit mode.
8086          * In TX explicit mode, there will be no hairpin flow ID.
8087          */
8088         if (hairpin > 0)
8089                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
8090         /* extra metadata enabled: one more TAG action will be add. */
8091         if (dev_conf->dv_flow_en &&
8092             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
8093             mlx5_flow_ext_mreg_supported(dev))
8094                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
8095         if (rw_act_num >
8096                         flow_dv_modify_hdr_action_max(dev, is_root)) {
8097                 return rte_flow_error_set(error, ENOTSUP,
8098                                           RTE_FLOW_ERROR_TYPE_ACTION,
8099                                           NULL, "too many header modify"
8100                                           " actions to support");
8101         }
8102         /* Eswitch egress mirror and modify flow has limitation on CX5 */
8103         if (fdb_mirror_limit && modify_after_mirror)
8104                 return rte_flow_error_set(error, EINVAL,
8105                                 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
8106                                 "sample before modify action is not supported");
8107         /*
8108          * Validation the NIC Egress flow on representor, except implicit
8109          * hairpin default egress flow with TX_QUEUE item, other flows not
8110          * work due to metadata regC0 mismatch.
8111          */
8112         if ((!attr->transfer && attr->egress) && priv->representor &&
8113             !(item_flags & MLX5_FLOW_ITEM_TX_QUEUE))
8114                 return rte_flow_error_set(error, EINVAL,
8115                                           RTE_FLOW_ERROR_TYPE_ITEM,
8116                                           NULL,
8117                                           "NIC egress rules on representors"
8118                                           " is not supported");
8119         return 0;
8120 }
8121
8122 /**
8123  * Internal preparation function. Allocates the DV flow size,
8124  * this size is constant.
8125  *
8126  * @param[in] dev
8127  *   Pointer to the rte_eth_dev structure.
8128  * @param[in] attr
8129  *   Pointer to the flow attributes.
8130  * @param[in] items
8131  *   Pointer to the list of items.
8132  * @param[in] actions
8133  *   Pointer to the list of actions.
8134  * @param[out] error
8135  *   Pointer to the error structure.
8136  *
8137  * @return
8138  *   Pointer to mlx5_flow object on success,
8139  *   otherwise NULL and rte_errno is set.
8140  */
8141 static struct mlx5_flow *
8142 flow_dv_prepare(struct rte_eth_dev *dev,
8143                 const struct rte_flow_attr *attr __rte_unused,
8144                 const struct rte_flow_item items[] __rte_unused,
8145                 const struct rte_flow_action actions[] __rte_unused,
8146                 struct rte_flow_error *error)
8147 {
8148         uint32_t handle_idx = 0;
8149         struct mlx5_flow *dev_flow;
8150         struct mlx5_flow_handle *dev_handle;
8151         struct mlx5_priv *priv = dev->data->dev_private;
8152         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
8153
8154         MLX5_ASSERT(wks);
8155         wks->skip_matcher_reg = 0;
8156         wks->policy = NULL;
8157         wks->final_policy = NULL;
8158         /* In case of corrupting the memory. */
8159         if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
8160                 rte_flow_error_set(error, ENOSPC,
8161                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8162                                    "not free temporary device flow");
8163                 return NULL;
8164         }
8165         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
8166                                    &handle_idx);
8167         if (!dev_handle) {
8168                 rte_flow_error_set(error, ENOMEM,
8169                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8170                                    "not enough memory to create flow handle");
8171                 return NULL;
8172         }
8173         MLX5_ASSERT(wks->flow_idx < RTE_DIM(wks->flows));
8174         dev_flow = &wks->flows[wks->flow_idx++];
8175         memset(dev_flow, 0, sizeof(*dev_flow));
8176         dev_flow->handle = dev_handle;
8177         dev_flow->handle_idx = handle_idx;
8178         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
8179         dev_flow->ingress = attr->ingress;
8180         dev_flow->dv.transfer = attr->transfer;
8181         return dev_flow;
8182 }
8183
8184 #ifdef RTE_LIBRTE_MLX5_DEBUG
8185 /**
8186  * Sanity check for match mask and value. Similar to check_valid_spec() in
8187  * kernel driver. If unmasked bit is present in value, it returns failure.
8188  *
8189  * @param match_mask
8190  *   pointer to match mask buffer.
8191  * @param match_value
8192  *   pointer to match value buffer.
8193  *
8194  * @return
8195  *   0 if valid, -EINVAL otherwise.
8196  */
8197 static int
8198 flow_dv_check_valid_spec(void *match_mask, void *match_value)
8199 {
8200         uint8_t *m = match_mask;
8201         uint8_t *v = match_value;
8202         unsigned int i;
8203
8204         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
8205                 if (v[i] & ~m[i]) {
8206                         DRV_LOG(ERR,
8207                                 "match_value differs from match_criteria"
8208                                 " %p[%u] != %p[%u]",
8209                                 match_value, i, match_mask, i);
8210                         return -EINVAL;
8211                 }
8212         }
8213         return 0;
8214 }
8215 #endif
8216
8217 /**
8218  * Add match of ip_version.
8219  *
8220  * @param[in] group
8221  *   Flow group.
8222  * @param[in] headers_v
8223  *   Values header pointer.
8224  * @param[in] headers_m
8225  *   Masks header pointer.
8226  * @param[in] ip_version
8227  *   The IP version to set.
8228  */
8229 static inline void
8230 flow_dv_set_match_ip_version(uint32_t group,
8231                              void *headers_v,
8232                              void *headers_m,
8233                              uint8_t ip_version)
8234 {
8235         if (group == 0)
8236                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
8237         else
8238                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
8239                          ip_version);
8240         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
8241         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
8242         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
8243 }
8244
8245 /**
8246  * Add Ethernet item to matcher and to the value.
8247  *
8248  * @param[in, out] matcher
8249  *   Flow matcher.
8250  * @param[in, out] key
8251  *   Flow matcher value.
8252  * @param[in] item
8253  *   Flow pattern to translate.
8254  * @param[in] inner
8255  *   Item is inner pattern.
8256  */
8257 static void
8258 flow_dv_translate_item_eth(void *matcher, void *key,
8259                            const struct rte_flow_item *item, int inner,
8260                            uint32_t group)
8261 {
8262         const struct rte_flow_item_eth *eth_m = item->mask;
8263         const struct rte_flow_item_eth *eth_v = item->spec;
8264         const struct rte_flow_item_eth nic_mask = {
8265                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
8266                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
8267                 .type = RTE_BE16(0xffff),
8268                 .has_vlan = 0,
8269         };
8270         void *hdrs_m;
8271         void *hdrs_v;
8272         char *l24_v;
8273         unsigned int i;
8274
8275         if (!eth_v)
8276                 return;
8277         if (!eth_m)
8278                 eth_m = &nic_mask;
8279         if (inner) {
8280                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8281                                          inner_headers);
8282                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8283         } else {
8284                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8285                                          outer_headers);
8286                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8287         }
8288         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, dmac_47_16),
8289                &eth_m->dst, sizeof(eth_m->dst));
8290         /* The value must be in the range of the mask. */
8291         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, dmac_47_16);
8292         for (i = 0; i < sizeof(eth_m->dst); ++i)
8293                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
8294         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, smac_47_16),
8295                &eth_m->src, sizeof(eth_m->src));
8296         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, smac_47_16);
8297         /* The value must be in the range of the mask. */
8298         for (i = 0; i < sizeof(eth_m->dst); ++i)
8299                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
8300         /*
8301          * HW supports match on one Ethertype, the Ethertype following the last
8302          * VLAN tag of the packet (see PRM).
8303          * Set match on ethertype only if ETH header is not followed by VLAN.
8304          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8305          * ethertype, and use ip_version field instead.
8306          * eCPRI over Ether layer will use type value 0xAEFE.
8307          */
8308         if (eth_m->type == 0xFFFF) {
8309                 /* Set cvlan_tag mask for any single\multi\un-tagged case. */
8310                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8311                 switch (eth_v->type) {
8312                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8313                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8314                         return;
8315                 case RTE_BE16(RTE_ETHER_TYPE_QINQ):
8316                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8317                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8318                         return;
8319                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8320                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8321                         return;
8322                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8323                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8324                         return;
8325                 default:
8326                         break;
8327                 }
8328         }
8329         if (eth_m->has_vlan) {
8330                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8331                 if (eth_v->has_vlan) {
8332                         /*
8333                          * Here, when also has_more_vlan field in VLAN item is
8334                          * not set, only single-tagged packets will be matched.
8335                          */
8336                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8337                         return;
8338                 }
8339         }
8340         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8341                  rte_be_to_cpu_16(eth_m->type));
8342         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype);
8343         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
8344 }
8345
8346 /**
8347  * Add VLAN item to matcher and to the value.
8348  *
8349  * @param[in, out] dev_flow
8350  *   Flow descriptor.
8351  * @param[in, out] matcher
8352  *   Flow matcher.
8353  * @param[in, out] key
8354  *   Flow matcher value.
8355  * @param[in] item
8356  *   Flow pattern to translate.
8357  * @param[in] inner
8358  *   Item is inner pattern.
8359  */
8360 static void
8361 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
8362                             void *matcher, void *key,
8363                             const struct rte_flow_item *item,
8364                             int inner, uint32_t group)
8365 {
8366         const struct rte_flow_item_vlan *vlan_m = item->mask;
8367         const struct rte_flow_item_vlan *vlan_v = item->spec;
8368         void *hdrs_m;
8369         void *hdrs_v;
8370         uint16_t tci_m;
8371         uint16_t tci_v;
8372
8373         if (inner) {
8374                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8375                                          inner_headers);
8376                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8377         } else {
8378                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8379                                          outer_headers);
8380                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8381                 /*
8382                  * This is workaround, masks are not supported,
8383                  * and pre-validated.
8384                  */
8385                 if (vlan_v)
8386                         dev_flow->handle->vf_vlan.tag =
8387                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
8388         }
8389         /*
8390          * When VLAN item exists in flow, mark packet as tagged,
8391          * even if TCI is not specified.
8392          */
8393         if (!MLX5_GET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag)) {
8394                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8395                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8396         }
8397         if (!vlan_v)
8398                 return;
8399         if (!vlan_m)
8400                 vlan_m = &rte_flow_item_vlan_mask;
8401         tci_m = rte_be_to_cpu_16(vlan_m->tci);
8402         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
8403         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_vid, tci_m);
8404         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_vid, tci_v);
8405         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_cfi, tci_m >> 12);
8406         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_cfi, tci_v >> 12);
8407         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_prio, tci_m >> 13);
8408         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_prio, tci_v >> 13);
8409         /*
8410          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8411          * ethertype, and use ip_version field instead.
8412          */
8413         if (vlan_m->inner_type == 0xFFFF) {
8414                 switch (vlan_v->inner_type) {
8415                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8416                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8417                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8418                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8419                         return;
8420                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8421                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8422                         return;
8423                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8424                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8425                         return;
8426                 default:
8427                         break;
8428                 }
8429         }
8430         if (vlan_m->has_more_vlan && vlan_v->has_more_vlan) {
8431                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8432                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8433                 /* Only one vlan_tag bit can be set. */
8434                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8435                 return;
8436         }
8437         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8438                  rte_be_to_cpu_16(vlan_m->inner_type));
8439         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, ethertype,
8440                  rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
8441 }
8442
8443 /**
8444  * Add IPV4 item to matcher and to the value.
8445  *
8446  * @param[in, out] matcher
8447  *   Flow matcher.
8448  * @param[in, out] key
8449  *   Flow matcher value.
8450  * @param[in] item
8451  *   Flow pattern to translate.
8452  * @param[in] inner
8453  *   Item is inner pattern.
8454  * @param[in] group
8455  *   The group to insert the rule.
8456  */
8457 static void
8458 flow_dv_translate_item_ipv4(void *matcher, void *key,
8459                             const struct rte_flow_item *item,
8460                             int inner, uint32_t group)
8461 {
8462         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
8463         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
8464         const struct rte_flow_item_ipv4 nic_mask = {
8465                 .hdr = {
8466                         .src_addr = RTE_BE32(0xffffffff),
8467                         .dst_addr = RTE_BE32(0xffffffff),
8468                         .type_of_service = 0xff,
8469                         .next_proto_id = 0xff,
8470                         .time_to_live = 0xff,
8471                 },
8472         };
8473         void *headers_m;
8474         void *headers_v;
8475         char *l24_m;
8476         char *l24_v;
8477         uint8_t tos, ihl_m, ihl_v;
8478
8479         if (inner) {
8480                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8481                                          inner_headers);
8482                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8483         } else {
8484                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8485                                          outer_headers);
8486                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8487         }
8488         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
8489         if (!ipv4_v)
8490                 return;
8491         if (!ipv4_m)
8492                 ipv4_m = &nic_mask;
8493         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8494                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8495         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8496                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8497         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
8498         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
8499         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8500                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8501         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8502                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8503         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
8504         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
8505         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
8506         ihl_m = ipv4_m->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK;
8507         ihl_v = ipv4_v->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK;
8508         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_ihl, ihl_m);
8509         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_ihl, ihl_m & ihl_v);
8510         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
8511                  ipv4_m->hdr.type_of_service);
8512         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
8513         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
8514                  ipv4_m->hdr.type_of_service >> 2);
8515         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
8516         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8517                  ipv4_m->hdr.next_proto_id);
8518         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8519                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
8520         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8521                  ipv4_m->hdr.time_to_live);
8522         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8523                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
8524         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8525                  !!(ipv4_m->hdr.fragment_offset));
8526         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8527                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
8528 }
8529
8530 /**
8531  * Add IPV6 item to matcher and to the value.
8532  *
8533  * @param[in, out] matcher
8534  *   Flow matcher.
8535  * @param[in, out] key
8536  *   Flow matcher value.
8537  * @param[in] item
8538  *   Flow pattern to translate.
8539  * @param[in] inner
8540  *   Item is inner pattern.
8541  * @param[in] group
8542  *   The group to insert the rule.
8543  */
8544 static void
8545 flow_dv_translate_item_ipv6(void *matcher, void *key,
8546                             const struct rte_flow_item *item,
8547                             int inner, uint32_t group)
8548 {
8549         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
8550         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
8551         const struct rte_flow_item_ipv6 nic_mask = {
8552                 .hdr = {
8553                         .src_addr =
8554                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8555                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8556                         .dst_addr =
8557                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8558                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8559                         .vtc_flow = RTE_BE32(0xffffffff),
8560                         .proto = 0xff,
8561                         .hop_limits = 0xff,
8562                 },
8563         };
8564         void *headers_m;
8565         void *headers_v;
8566         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8567         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8568         char *l24_m;
8569         char *l24_v;
8570         uint32_t vtc_m;
8571         uint32_t vtc_v;
8572         int i;
8573         int size;
8574
8575         if (inner) {
8576                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8577                                          inner_headers);
8578                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8579         } else {
8580                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8581                                          outer_headers);
8582                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8583         }
8584         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
8585         if (!ipv6_v)
8586                 return;
8587         if (!ipv6_m)
8588                 ipv6_m = &nic_mask;
8589         size = sizeof(ipv6_m->hdr.dst_addr);
8590         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8591                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8592         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8593                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8594         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
8595         for (i = 0; i < size; ++i)
8596                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
8597         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8598                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8599         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8600                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8601         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
8602         for (i = 0; i < size; ++i)
8603                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
8604         /* TOS. */
8605         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
8606         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
8607         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
8608         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
8609         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
8610         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
8611         /* Label. */
8612         if (inner) {
8613                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
8614                          vtc_m);
8615                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
8616                          vtc_v);
8617         } else {
8618                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
8619                          vtc_m);
8620                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
8621                          vtc_v);
8622         }
8623         /* Protocol. */
8624         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8625                  ipv6_m->hdr.proto);
8626         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8627                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
8628         /* Hop limit. */
8629         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8630                  ipv6_m->hdr.hop_limits);
8631         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8632                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
8633         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8634                  !!(ipv6_m->has_frag_ext));
8635         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8636                  !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext));
8637 }
8638
8639 /**
8640  * Add IPV6 fragment extension item to matcher and to the value.
8641  *
8642  * @param[in, out] matcher
8643  *   Flow matcher.
8644  * @param[in, out] key
8645  *   Flow matcher value.
8646  * @param[in] item
8647  *   Flow pattern to translate.
8648  * @param[in] inner
8649  *   Item is inner pattern.
8650  */
8651 static void
8652 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key,
8653                                      const struct rte_flow_item *item,
8654                                      int inner)
8655 {
8656         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask;
8657         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec;
8658         const struct rte_flow_item_ipv6_frag_ext nic_mask = {
8659                 .hdr = {
8660                         .next_header = 0xff,
8661                         .frag_data = RTE_BE16(0xffff),
8662                 },
8663         };
8664         void *headers_m;
8665         void *headers_v;
8666
8667         if (inner) {
8668                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8669                                          inner_headers);
8670                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8671         } else {
8672                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8673                                          outer_headers);
8674                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8675         }
8676         /* IPv6 fragment extension item exists, so packet is IP fragment. */
8677         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
8678         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1);
8679         if (!ipv6_frag_ext_v)
8680                 return;
8681         if (!ipv6_frag_ext_m)
8682                 ipv6_frag_ext_m = &nic_mask;
8683         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8684                  ipv6_frag_ext_m->hdr.next_header);
8685         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8686                  ipv6_frag_ext_v->hdr.next_header &
8687                  ipv6_frag_ext_m->hdr.next_header);
8688 }
8689
8690 /**
8691  * Add TCP item to matcher and to the value.
8692  *
8693  * @param[in, out] matcher
8694  *   Flow matcher.
8695  * @param[in, out] key
8696  *   Flow matcher value.
8697  * @param[in] item
8698  *   Flow pattern to translate.
8699  * @param[in] inner
8700  *   Item is inner pattern.
8701  */
8702 static void
8703 flow_dv_translate_item_tcp(void *matcher, void *key,
8704                            const struct rte_flow_item *item,
8705                            int inner)
8706 {
8707         const struct rte_flow_item_tcp *tcp_m = item->mask;
8708         const struct rte_flow_item_tcp *tcp_v = item->spec;
8709         void *headers_m;
8710         void *headers_v;
8711
8712         if (inner) {
8713                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8714                                          inner_headers);
8715                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8716         } else {
8717                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8718                                          outer_headers);
8719                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8720         }
8721         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8722         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
8723         if (!tcp_v)
8724                 return;
8725         if (!tcp_m)
8726                 tcp_m = &rte_flow_item_tcp_mask;
8727         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
8728                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
8729         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
8730                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
8731         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
8732                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
8733         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
8734                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
8735         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
8736                  tcp_m->hdr.tcp_flags);
8737         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
8738                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
8739 }
8740
8741 /**
8742  * Add ESP item to matcher and to the value.
8743  *
8744  * @param[in, out] matcher
8745  *   Flow matcher.
8746  * @param[in, out] key
8747  *   Flow matcher value.
8748  * @param[in] item
8749  *   Flow pattern to translate.
8750  * @param[in] inner
8751  *   Item is inner pattern.
8752  */
8753 static void
8754 flow_dv_translate_item_esp(void *matcher, void *key,
8755                            const struct rte_flow_item *item,
8756                            int inner)
8757 {
8758         const struct rte_flow_item_esp *esp_m = item->mask;
8759         const struct rte_flow_item_esp *esp_v = item->spec;
8760         void *headers_m;
8761         void *headers_v;
8762         char *spi_m;
8763         char *spi_v;
8764
8765         if (inner) {
8766                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8767                                          inner_headers);
8768                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8769         } else {
8770                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8771                                          outer_headers);
8772                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8773         }
8774         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8775         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ESP);
8776         if (!esp_v)
8777                 return;
8778         if (!esp_m)
8779                 esp_m = &rte_flow_item_esp_mask;
8780         headers_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8781         headers_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8782         if (inner) {
8783                 spi_m = MLX5_ADDR_OF(fte_match_set_misc, headers_m, inner_esp_spi);
8784                 spi_v = MLX5_ADDR_OF(fte_match_set_misc, headers_v, inner_esp_spi);
8785         } else {
8786                 spi_m = MLX5_ADDR_OF(fte_match_set_misc, headers_m, outer_esp_spi);
8787                 spi_v = MLX5_ADDR_OF(fte_match_set_misc, headers_v, outer_esp_spi);
8788         }
8789         *(uint32_t *)spi_m = esp_m->hdr.spi;
8790         *(uint32_t *)spi_v = esp_m->hdr.spi & esp_v->hdr.spi;
8791 }
8792
8793 /**
8794  * Add UDP item to matcher and to the value.
8795  *
8796  * @param[in, out] matcher
8797  *   Flow matcher.
8798  * @param[in, out] key
8799  *   Flow matcher value.
8800  * @param[in] item
8801  *   Flow pattern to translate.
8802  * @param[in] inner
8803  *   Item is inner pattern.
8804  */
8805 static void
8806 flow_dv_translate_item_udp(void *matcher, void *key,
8807                            const struct rte_flow_item *item,
8808                            int inner)
8809 {
8810         const struct rte_flow_item_udp *udp_m = item->mask;
8811         const struct rte_flow_item_udp *udp_v = item->spec;
8812         void *headers_m;
8813         void *headers_v;
8814
8815         if (inner) {
8816                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8817                                          inner_headers);
8818                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8819         } else {
8820                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8821                                          outer_headers);
8822                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8823         }
8824         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8825         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
8826         if (!udp_v)
8827                 return;
8828         if (!udp_m)
8829                 udp_m = &rte_flow_item_udp_mask;
8830         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
8831                  rte_be_to_cpu_16(udp_m->hdr.src_port));
8832         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
8833                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
8834         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
8835                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
8836         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
8837                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
8838 }
8839
8840 /**
8841  * Add GRE optional Key item to matcher and to the value.
8842  *
8843  * @param[in, out] matcher
8844  *   Flow matcher.
8845  * @param[in, out] key
8846  *   Flow matcher value.
8847  * @param[in] item
8848  *   Flow pattern to translate.
8849  * @param[in] inner
8850  *   Item is inner pattern.
8851  */
8852 static void
8853 flow_dv_translate_item_gre_key(void *matcher, void *key,
8854                                    const struct rte_flow_item *item)
8855 {
8856         const rte_be32_t *key_m = item->mask;
8857         const rte_be32_t *key_v = item->spec;
8858         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8859         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8860         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
8861
8862         /* GRE K bit must be on and should already be validated */
8863         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
8864         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
8865         if (!key_v)
8866                 return;
8867         if (!key_m)
8868                 key_m = &gre_key_default_mask;
8869         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
8870                  rte_be_to_cpu_32(*key_m) >> 8);
8871         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
8872                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
8873         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
8874                  rte_be_to_cpu_32(*key_m) & 0xFF);
8875         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
8876                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
8877 }
8878
8879 /**
8880  * Add GRE item to matcher and to the value.
8881  *
8882  * @param[in, out] matcher
8883  *   Flow matcher.
8884  * @param[in, out] key
8885  *   Flow matcher value.
8886  * @param[in] item
8887  *   Flow pattern to translate.
8888  * @param[in] pattern_flags
8889  *   Accumulated pattern flags.
8890  */
8891 static void
8892 flow_dv_translate_item_gre(void *matcher, void *key,
8893                            const struct rte_flow_item *item,
8894                            uint64_t pattern_flags)
8895 {
8896         static const struct rte_flow_item_gre empty_gre = {0,};
8897         const struct rte_flow_item_gre *gre_m = item->mask;
8898         const struct rte_flow_item_gre *gre_v = item->spec;
8899         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
8900         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8901         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8902         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8903         struct {
8904                 union {
8905                         __extension__
8906                         struct {
8907                                 uint16_t version:3;
8908                                 uint16_t rsvd0:9;
8909                                 uint16_t s_present:1;
8910                                 uint16_t k_present:1;
8911                                 uint16_t rsvd_bit1:1;
8912                                 uint16_t c_present:1;
8913                         };
8914                         uint16_t value;
8915                 };
8916         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
8917         uint16_t protocol_m, protocol_v;
8918
8919         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8920         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
8921         if (!gre_v) {
8922                 gre_v = &empty_gre;
8923                 gre_m = &empty_gre;
8924         } else {
8925                 if (!gre_m)
8926                         gre_m = &rte_flow_item_gre_mask;
8927         }
8928         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
8929         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
8930         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
8931                  gre_crks_rsvd0_ver_m.c_present);
8932         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
8933                  gre_crks_rsvd0_ver_v.c_present &
8934                  gre_crks_rsvd0_ver_m.c_present);
8935         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
8936                  gre_crks_rsvd0_ver_m.k_present);
8937         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
8938                  gre_crks_rsvd0_ver_v.k_present &
8939                  gre_crks_rsvd0_ver_m.k_present);
8940         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
8941                  gre_crks_rsvd0_ver_m.s_present);
8942         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
8943                  gre_crks_rsvd0_ver_v.s_present &
8944                  gre_crks_rsvd0_ver_m.s_present);
8945         protocol_m = rte_be_to_cpu_16(gre_m->protocol);
8946         protocol_v = rte_be_to_cpu_16(gre_v->protocol);
8947         if (!protocol_m) {
8948                 /* Force next protocol to prevent matchers duplication */
8949                 protocol_v = mlx5_translate_tunnel_etypes(pattern_flags);
8950                 if (protocol_v)
8951                         protocol_m = 0xFFFF;
8952         }
8953         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, protocol_m);
8954         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
8955                  protocol_m & protocol_v);
8956 }
8957
8958 /**
8959  * Add GRE optional items to matcher and to the value.
8960  *
8961  * @param[in, out] matcher
8962  *   Flow matcher.
8963  * @param[in, out] key
8964  *   Flow matcher value.
8965  * @param[in] item
8966  *   Flow pattern to translate.
8967  * @param[in] gre_item
8968  *   Pointer to gre_item.
8969  * @param[in] pattern_flags
8970  *   Accumulated pattern flags.
8971  */
8972 static void
8973 flow_dv_translate_item_gre_option(void *matcher, void *key,
8974                                   const struct rte_flow_item *item,
8975                                   const struct rte_flow_item *gre_item,
8976                                   uint64_t pattern_flags)
8977 {
8978         const struct rte_flow_item_gre_opt *option_m = item->mask;
8979         const struct rte_flow_item_gre_opt *option_v = item->spec;
8980         const struct rte_flow_item_gre *gre_m = gre_item->mask;
8981         const struct rte_flow_item_gre *gre_v = gre_item->spec;
8982         static const struct rte_flow_item_gre empty_gre = {0};
8983         struct rte_flow_item gre_key_item;
8984         uint16_t c_rsvd0_ver_m, c_rsvd0_ver_v;
8985         uint16_t protocol_m, protocol_v;
8986         void *misc5_m;
8987         void *misc5_v;
8988
8989         /*
8990          * If only match key field, keep using misc for matching.
8991          * If need to match checksum or sequence, using misc5 and do
8992          * not need using misc.
8993          */
8994         if (!(option_m->sequence.sequence ||
8995               option_m->checksum_rsvd.checksum)) {
8996                 flow_dv_translate_item_gre(matcher, key, gre_item,
8997                                            pattern_flags);
8998                 gre_key_item.spec = &option_v->key.key;
8999                 gre_key_item.mask = &option_m->key.key;
9000                 flow_dv_translate_item_gre_key(matcher, key, &gre_key_item);
9001                 return;
9002         }
9003         if (!gre_v) {
9004                 gre_v = &empty_gre;
9005                 gre_m = &empty_gre;
9006         } else {
9007                 if (!gre_m)
9008                         gre_m = &rte_flow_item_gre_mask;
9009         }
9010         protocol_v = gre_v->protocol;
9011         protocol_m = gre_m->protocol;
9012         if (!protocol_m) {
9013                 /* Force next protocol to prevent matchers duplication */
9014                 uint16_t ether_type =
9015                         mlx5_translate_tunnel_etypes(pattern_flags);
9016                 if (ether_type) {
9017                         protocol_v = rte_be_to_cpu_16(ether_type);
9018                         protocol_m = UINT16_MAX;
9019                 }
9020         }
9021         c_rsvd0_ver_v = gre_v->c_rsvd0_ver;
9022         c_rsvd0_ver_m = gre_m->c_rsvd0_ver;
9023         if (option_m->sequence.sequence) {
9024                 c_rsvd0_ver_v |= RTE_BE16(0x1000);
9025                 c_rsvd0_ver_m |= RTE_BE16(0x1000);
9026         }
9027         if (option_m->key.key) {
9028                 c_rsvd0_ver_v |= RTE_BE16(0x2000);
9029                 c_rsvd0_ver_m |= RTE_BE16(0x2000);
9030         }
9031         if (option_m->checksum_rsvd.checksum) {
9032                 c_rsvd0_ver_v |= RTE_BE16(0x8000);
9033                 c_rsvd0_ver_m |= RTE_BE16(0x8000);
9034         }
9035         /*
9036          * Hardware parses GRE optional field into the fixed location,
9037          * do not need to adjust the tunnel dword indices.
9038          */
9039         misc5_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_5);
9040         misc5_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_5);
9041         MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_0,
9042                  rte_be_to_cpu_32((c_rsvd0_ver_v | protocol_v << 16) &
9043                                   (c_rsvd0_ver_m | protocol_m << 16)));
9044         MLX5_SET(fte_match_set_misc5, misc5_m, tunnel_header_0,
9045                  rte_be_to_cpu_32(c_rsvd0_ver_m | protocol_m << 16));
9046         MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_1,
9047                  rte_be_to_cpu_32(option_v->checksum_rsvd.checksum &
9048                                   option_m->checksum_rsvd.checksum));
9049         MLX5_SET(fte_match_set_misc5, misc5_m, tunnel_header_1,
9050                  rte_be_to_cpu_32(option_m->checksum_rsvd.checksum));
9051         MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_2,
9052                  rte_be_to_cpu_32(option_v->key.key & option_m->key.key));
9053         MLX5_SET(fte_match_set_misc5, misc5_m, tunnel_header_2,
9054                  rte_be_to_cpu_32(option_m->key.key));
9055         MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_3,
9056                  rte_be_to_cpu_32(option_v->sequence.sequence &
9057                                   option_m->sequence.sequence));
9058         MLX5_SET(fte_match_set_misc5, misc5_m, tunnel_header_3,
9059                  rte_be_to_cpu_32(option_m->sequence.sequence));
9060 }
9061
9062 /**
9063  * Add NVGRE item to matcher and to the value.
9064  *
9065  * @param[in, out] matcher
9066  *   Flow matcher.
9067  * @param[in, out] key
9068  *   Flow matcher value.
9069  * @param[in] item
9070  *   Flow pattern to translate.
9071  * @param[in] pattern_flags
9072  *   Accumulated pattern flags.
9073  */
9074 static void
9075 flow_dv_translate_item_nvgre(void *matcher, void *key,
9076                              const struct rte_flow_item *item,
9077                              unsigned long pattern_flags)
9078 {
9079         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
9080         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
9081         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9082         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9083         const char *tni_flow_id_m;
9084         const char *tni_flow_id_v;
9085         char *gre_key_m;
9086         char *gre_key_v;
9087         int size;
9088         int i;
9089
9090         /* For NVGRE, GRE header fields must be set with defined values. */
9091         const struct rte_flow_item_gre gre_spec = {
9092                 .c_rsvd0_ver = RTE_BE16(0x2000),
9093                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
9094         };
9095         const struct rte_flow_item_gre gre_mask = {
9096                 .c_rsvd0_ver = RTE_BE16(0xB000),
9097                 .protocol = RTE_BE16(UINT16_MAX),
9098         };
9099         const struct rte_flow_item gre_item = {
9100                 .spec = &gre_spec,
9101                 .mask = &gre_mask,
9102                 .last = NULL,
9103         };
9104         flow_dv_translate_item_gre(matcher, key, &gre_item, pattern_flags);
9105         if (!nvgre_v)
9106                 return;
9107         if (!nvgre_m)
9108                 nvgre_m = &rte_flow_item_nvgre_mask;
9109         tni_flow_id_m = (const char *)nvgre_m->tni;
9110         tni_flow_id_v = (const char *)nvgre_v->tni;
9111         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
9112         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
9113         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
9114         memcpy(gre_key_m, tni_flow_id_m, size);
9115         for (i = 0; i < size; ++i)
9116                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
9117 }
9118
9119 /**
9120  * Add VXLAN item to matcher and to the value.
9121  *
9122  * @param[in] dev
9123  *   Pointer to the Ethernet device structure.
9124  * @param[in] attr
9125  *   Flow rule attributes.
9126  * @param[in, out] matcher
9127  *   Flow matcher.
9128  * @param[in, out] key
9129  *   Flow matcher value.
9130  * @param[in] item
9131  *   Flow pattern to translate.
9132  * @param[in] inner
9133  *   Item is inner pattern.
9134  */
9135 static void
9136 flow_dv_translate_item_vxlan(struct rte_eth_dev *dev,
9137                              const struct rte_flow_attr *attr,
9138                              void *matcher, void *key,
9139                              const struct rte_flow_item *item,
9140                              int inner)
9141 {
9142         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
9143         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
9144         void *headers_m;
9145         void *headers_v;
9146         void *misc5_m;
9147         void *misc5_v;
9148         uint32_t *tunnel_header_v;
9149         uint32_t *tunnel_header_m;
9150         uint16_t dport;
9151         struct mlx5_priv *priv = dev->data->dev_private;
9152         const struct rte_flow_item_vxlan nic_mask = {
9153                 .vni = "\xff\xff\xff",
9154                 .rsvd1 = 0xff,
9155         };
9156
9157         if (inner) {
9158                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9159                                          inner_headers);
9160                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9161         } else {
9162                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9163                                          outer_headers);
9164                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9165         }
9166         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
9167                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
9168         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9169                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9170                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
9171         }
9172         dport = MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport);
9173         if (!vxlan_v)
9174                 return;
9175         if (!vxlan_m) {
9176                 if ((!attr->group && !priv->sh->tunnel_header_0_1) ||
9177                     (attr->group && !priv->sh->misc5_cap))
9178                         vxlan_m = &rte_flow_item_vxlan_mask;
9179                 else
9180                         vxlan_m = &nic_mask;
9181         }
9182         if ((priv->sh->steering_format_version ==
9183             MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 &&
9184             dport != MLX5_UDP_PORT_VXLAN) ||
9185             (!attr->group && !attr->transfer && !priv->sh->tunnel_header_0_1) ||
9186             ((attr->group || attr->transfer) && !priv->sh->misc5_cap)) {
9187                 void *misc_m;
9188                 void *misc_v;
9189                 char *vni_m;
9190                 char *vni_v;
9191                 int size;
9192                 int i;
9193                 misc_m = MLX5_ADDR_OF(fte_match_param,
9194                                       matcher, misc_parameters);
9195                 misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9196                 size = sizeof(vxlan_m->vni);
9197                 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
9198                 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
9199                 memcpy(vni_m, vxlan_m->vni, size);
9200                 for (i = 0; i < size; ++i)
9201                         vni_v[i] = vni_m[i] & vxlan_v->vni[i];
9202                 return;
9203         }
9204         misc5_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_5);
9205         misc5_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_5);
9206         tunnel_header_v = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5,
9207                                                    misc5_v,
9208                                                    tunnel_header_1);
9209         tunnel_header_m = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5,
9210                                                    misc5_m,
9211                                                    tunnel_header_1);
9212         *tunnel_header_v = (vxlan_v->vni[0] & vxlan_m->vni[0]) |
9213                            (vxlan_v->vni[1] & vxlan_m->vni[1]) << 8 |
9214                            (vxlan_v->vni[2] & vxlan_m->vni[2]) << 16;
9215         if (*tunnel_header_v)
9216                 *tunnel_header_m = vxlan_m->vni[0] |
9217                         vxlan_m->vni[1] << 8 |
9218                         vxlan_m->vni[2] << 16;
9219         else
9220                 *tunnel_header_m = 0x0;
9221         *tunnel_header_v |= (vxlan_v->rsvd1 & vxlan_m->rsvd1) << 24;
9222         if (vxlan_v->rsvd1 & vxlan_m->rsvd1)
9223                 *tunnel_header_m |= vxlan_m->rsvd1 << 24;
9224 }
9225
9226 /**
9227  * Add VXLAN-GPE item to matcher and to the value.
9228  *
9229  * @param[in, out] matcher
9230  *   Flow matcher.
9231  * @param[in, out] key
9232  *   Flow matcher value.
9233  * @param[in] item
9234  *   Flow pattern to translate.
9235  * @param[in] inner
9236  *   Item is inner pattern.
9237  */
9238
9239 static void
9240 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
9241                                  const struct rte_flow_item *item,
9242                                  const uint64_t pattern_flags)
9243 {
9244         static const struct rte_flow_item_vxlan_gpe dummy_vxlan_gpe_hdr = {0, };
9245         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
9246         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
9247         /* The item was validated to be on the outer side */
9248         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
9249         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9250         void *misc_m =
9251                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
9252         void *misc_v =
9253                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9254         char *vni_m =
9255                 MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
9256         char *vni_v =
9257                 MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
9258         int i, size = sizeof(vxlan_m->vni);
9259         uint8_t flags_m = 0xff;
9260         uint8_t flags_v = 0xc;
9261         uint8_t m_protocol, v_protocol;
9262
9263         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9264                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9265                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
9266                          MLX5_UDP_PORT_VXLAN_GPE);
9267         }
9268         if (!vxlan_v) {
9269                 vxlan_v = &dummy_vxlan_gpe_hdr;
9270                 vxlan_m = &dummy_vxlan_gpe_hdr;
9271         } else {
9272                 if (!vxlan_m)
9273                         vxlan_m = &rte_flow_item_vxlan_gpe_mask;
9274         }
9275         memcpy(vni_m, vxlan_m->vni, size);
9276         for (i = 0; i < size; ++i)
9277                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
9278         if (vxlan_m->flags) {
9279                 flags_m = vxlan_m->flags;
9280                 flags_v = vxlan_v->flags;
9281         }
9282         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
9283         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
9284         m_protocol = vxlan_m->protocol;
9285         v_protocol = vxlan_v->protocol;
9286         if (!m_protocol) {
9287                 /* Force next protocol to ensure next headers parsing. */
9288                 if (pattern_flags & MLX5_FLOW_LAYER_INNER_L2)
9289                         v_protocol = RTE_VXLAN_GPE_TYPE_ETH;
9290                 else if (pattern_flags & MLX5_FLOW_LAYER_INNER_L3_IPV4)
9291                         v_protocol = RTE_VXLAN_GPE_TYPE_IPV4;
9292                 else if (pattern_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)
9293                         v_protocol = RTE_VXLAN_GPE_TYPE_IPV6;
9294                 if (v_protocol)
9295                         m_protocol = 0xFF;
9296         }
9297         MLX5_SET(fte_match_set_misc3, misc_m,
9298                  outer_vxlan_gpe_next_protocol, m_protocol);
9299         MLX5_SET(fte_match_set_misc3, misc_v,
9300                  outer_vxlan_gpe_next_protocol, m_protocol & v_protocol);
9301 }
9302
9303 /**
9304  * Add Geneve item to matcher and to the value.
9305  *
9306  * @param[in, out] matcher
9307  *   Flow matcher.
9308  * @param[in, out] key
9309  *   Flow matcher value.
9310  * @param[in] item
9311  *   Flow pattern to translate.
9312  * @param[in] inner
9313  *   Item is inner pattern.
9314  */
9315
9316 static void
9317 flow_dv_translate_item_geneve(void *matcher, void *key,
9318                               const struct rte_flow_item *item,
9319                               uint64_t pattern_flags)
9320 {
9321         static const struct rte_flow_item_geneve empty_geneve = {0,};
9322         const struct rte_flow_item_geneve *geneve_m = item->mask;
9323         const struct rte_flow_item_geneve *geneve_v = item->spec;
9324         /* GENEVE flow item validation allows single tunnel item */
9325         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
9326         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9327         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9328         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9329         uint16_t gbhdr_m;
9330         uint16_t gbhdr_v;
9331         char *vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
9332         char *vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
9333         size_t size = sizeof(geneve_m->vni), i;
9334         uint16_t protocol_m, protocol_v;
9335
9336         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9337                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9338                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
9339                          MLX5_UDP_PORT_GENEVE);
9340         }
9341         if (!geneve_v) {
9342                 geneve_v = &empty_geneve;
9343                 geneve_m = &empty_geneve;
9344         } else {
9345                 if (!geneve_m)
9346                         geneve_m = &rte_flow_item_geneve_mask;
9347         }
9348         memcpy(vni_m, geneve_m->vni, size);
9349         for (i = 0; i < size; ++i)
9350                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
9351         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
9352         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
9353         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
9354                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
9355         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
9356                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
9357         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
9358                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
9359         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
9360                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
9361                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
9362         protocol_m = rte_be_to_cpu_16(geneve_m->protocol);
9363         protocol_v = rte_be_to_cpu_16(geneve_v->protocol);
9364         if (!protocol_m) {
9365                 /* Force next protocol to prevent matchers duplication */
9366                 protocol_v = mlx5_translate_tunnel_etypes(pattern_flags);
9367                 if (protocol_v)
9368                         protocol_m = 0xFFFF;
9369         }
9370         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type, protocol_m);
9371         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
9372                  protocol_m & protocol_v);
9373 }
9374
9375 /**
9376  * Create Geneve TLV option resource.
9377  *
9378  * @param dev[in, out]
9379  *   Pointer to rte_eth_dev structure.
9380  * @param[in, out] tag_be24
9381  *   Tag value in big endian then R-shift 8.
9382  * @parm[in, out] dev_flow
9383  *   Pointer to the dev_flow.
9384  * @param[out] error
9385  *   pointer to error structure.
9386  *
9387  * @return
9388  *   0 on success otherwise -errno and errno is set.
9389  */
9390
9391 int
9392 flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
9393                                              const struct rte_flow_item *item,
9394                                              struct rte_flow_error *error)
9395 {
9396         struct mlx5_priv *priv = dev->data->dev_private;
9397         struct mlx5_dev_ctx_shared *sh = priv->sh;
9398         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
9399                         sh->geneve_tlv_option_resource;
9400         struct mlx5_devx_obj *obj;
9401         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
9402         int ret = 0;
9403
9404         if (!geneve_opt_v)
9405                 return -1;
9406         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
9407         if (geneve_opt_resource != NULL) {
9408                 if (geneve_opt_resource->option_class ==
9409                         geneve_opt_v->option_class &&
9410                         geneve_opt_resource->option_type ==
9411                         geneve_opt_v->option_type &&
9412                         geneve_opt_resource->length ==
9413                         geneve_opt_v->option_len) {
9414                         /* We already have GENEVE TLV option obj allocated. */
9415                         __atomic_fetch_add(&geneve_opt_resource->refcnt, 1,
9416                                            __ATOMIC_RELAXED);
9417                 } else {
9418                         ret = rte_flow_error_set(error, ENOMEM,
9419                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9420                                 "Only one GENEVE TLV option supported");
9421                         goto exit;
9422                 }
9423         } else {
9424                 /* Create a GENEVE TLV object and resource. */
9425                 obj = mlx5_devx_cmd_create_geneve_tlv_option(sh->cdev->ctx,
9426                                 geneve_opt_v->option_class,
9427                                 geneve_opt_v->option_type,
9428                                 geneve_opt_v->option_len);
9429                 if (!obj) {
9430                         ret = rte_flow_error_set(error, ENODATA,
9431                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9432                                 "Failed to create GENEVE TLV Devx object");
9433                         goto exit;
9434                 }
9435                 sh->geneve_tlv_option_resource =
9436                                 mlx5_malloc(MLX5_MEM_ZERO,
9437                                                 sizeof(*geneve_opt_resource),
9438                                                 0, SOCKET_ID_ANY);
9439                 if (!sh->geneve_tlv_option_resource) {
9440                         claim_zero(mlx5_devx_cmd_destroy(obj));
9441                         ret = rte_flow_error_set(error, ENOMEM,
9442                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9443                                 "GENEVE TLV object memory allocation failed");
9444                         goto exit;
9445                 }
9446                 geneve_opt_resource = sh->geneve_tlv_option_resource;
9447                 geneve_opt_resource->obj = obj;
9448                 geneve_opt_resource->option_class = geneve_opt_v->option_class;
9449                 geneve_opt_resource->option_type = geneve_opt_v->option_type;
9450                 geneve_opt_resource->length = geneve_opt_v->option_len;
9451                 __atomic_store_n(&geneve_opt_resource->refcnt, 1,
9452                                 __ATOMIC_RELAXED);
9453         }
9454 exit:
9455         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
9456         return ret;
9457 }
9458
9459 /**
9460  * Add Geneve TLV option item to matcher.
9461  *
9462  * @param[in, out] dev
9463  *   Pointer to rte_eth_dev structure.
9464  * @param[in, out] matcher
9465  *   Flow matcher.
9466  * @param[in, out] key
9467  *   Flow matcher value.
9468  * @param[in] item
9469  *   Flow pattern to translate.
9470  * @param[out] error
9471  *   Pointer to error structure.
9472  */
9473 static int
9474 flow_dv_translate_item_geneve_opt(struct rte_eth_dev *dev, void *matcher,
9475                                   void *key, const struct rte_flow_item *item,
9476                                   struct rte_flow_error *error)
9477 {
9478         const struct rte_flow_item_geneve_opt *geneve_opt_m = item->mask;
9479         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
9480         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9481         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9482         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9483                         misc_parameters_3);
9484         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9485         rte_be32_t opt_data_key = 0, opt_data_mask = 0;
9486         int ret = 0;
9487
9488         if (!geneve_opt_v)
9489                 return -1;
9490         if (!geneve_opt_m)
9491                 geneve_opt_m = &rte_flow_item_geneve_opt_mask;
9492         ret = flow_dev_geneve_tlv_option_resource_register(dev, item,
9493                                                            error);
9494         if (ret) {
9495                 DRV_LOG(ERR, "Failed to create geneve_tlv_obj");
9496                 return ret;
9497         }
9498         /*
9499          * Set the option length in GENEVE header if not requested.
9500          * The GENEVE TLV option length is expressed by the option length field
9501          * in the GENEVE header.
9502          * If the option length was not requested but the GENEVE TLV option item
9503          * is present we set the option length field implicitly.
9504          */
9505         if (!MLX5_GET16(fte_match_set_misc, misc_m, geneve_opt_len)) {
9506                 MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
9507                          MLX5_GENEVE_OPTLEN_MASK);
9508                 MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
9509                          geneve_opt_v->option_len + 1);
9510         }
9511         MLX5_SET(fte_match_set_misc, misc_m, geneve_tlv_option_0_exist, 1);
9512         MLX5_SET(fte_match_set_misc, misc_v, geneve_tlv_option_0_exist, 1);
9513         /* Set the data. */
9514         if (geneve_opt_v->data) {
9515                 memcpy(&opt_data_key, geneve_opt_v->data,
9516                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
9517                                 sizeof(opt_data_key)));
9518                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
9519                                 sizeof(opt_data_key));
9520                 memcpy(&opt_data_mask, geneve_opt_m->data,
9521                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
9522                                 sizeof(opt_data_mask)));
9523                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
9524                                 sizeof(opt_data_mask));
9525                 MLX5_SET(fte_match_set_misc3, misc3_m,
9526                                 geneve_tlv_option_0_data,
9527                                 rte_be_to_cpu_32(opt_data_mask));
9528                 MLX5_SET(fte_match_set_misc3, misc3_v,
9529                                 geneve_tlv_option_0_data,
9530                         rte_be_to_cpu_32(opt_data_key & opt_data_mask));
9531         }
9532         return ret;
9533 }
9534
9535 /**
9536  * Add MPLS item to matcher and to the value.
9537  *
9538  * @param[in, out] matcher
9539  *   Flow matcher.
9540  * @param[in, out] key
9541  *   Flow matcher value.
9542  * @param[in] item
9543  *   Flow pattern to translate.
9544  * @param[in] prev_layer
9545  *   The protocol layer indicated in previous item.
9546  * @param[in] inner
9547  *   Item is inner pattern.
9548  */
9549 static void
9550 flow_dv_translate_item_mpls(void *matcher, void *key,
9551                             const struct rte_flow_item *item,
9552                             uint64_t prev_layer,
9553                             int inner)
9554 {
9555         const uint32_t *in_mpls_m = item->mask;
9556         const uint32_t *in_mpls_v = item->spec;
9557         uint32_t *out_mpls_m = 0;
9558         uint32_t *out_mpls_v = 0;
9559         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9560         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9561         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
9562                                      misc_parameters_2);
9563         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9564         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
9565         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9566
9567         switch (prev_layer) {
9568         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9569                 if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9570                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
9571                                  0xffff);
9572                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
9573                                  MLX5_UDP_PORT_MPLS);
9574                 }
9575                 break;
9576         case MLX5_FLOW_LAYER_GRE:
9577                 /* Fall-through. */
9578         case MLX5_FLOW_LAYER_GRE_KEY:
9579                 if (!MLX5_GET16(fte_match_set_misc, misc_v, gre_protocol)) {
9580                         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
9581                                  0xffff);
9582                         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
9583                                  RTE_ETHER_TYPE_MPLS);
9584                 }
9585                 break;
9586         default:
9587                 break;
9588         }
9589         if (!in_mpls_v)
9590                 return;
9591         if (!in_mpls_m)
9592                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
9593         switch (prev_layer) {
9594         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9595                 out_mpls_m =
9596                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9597                                                  outer_first_mpls_over_udp);
9598                 out_mpls_v =
9599                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9600                                                  outer_first_mpls_over_udp);
9601                 break;
9602         case MLX5_FLOW_LAYER_GRE:
9603                 out_mpls_m =
9604                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9605                                                  outer_first_mpls_over_gre);
9606                 out_mpls_v =
9607                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9608                                                  outer_first_mpls_over_gre);
9609                 break;
9610         default:
9611                 /* Inner MPLS not over GRE is not supported. */
9612                 if (!inner) {
9613                         out_mpls_m =
9614                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9615                                                          misc2_m,
9616                                                          outer_first_mpls);
9617                         out_mpls_v =
9618                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9619                                                          misc2_v,
9620                                                          outer_first_mpls);
9621                 }
9622                 break;
9623         }
9624         if (out_mpls_m && out_mpls_v) {
9625                 *out_mpls_m = *in_mpls_m;
9626                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
9627         }
9628 }
9629
9630 /**
9631  * Add metadata register item to matcher
9632  *
9633  * @param[in, out] matcher
9634  *   Flow matcher.
9635  * @param[in, out] key
9636  *   Flow matcher value.
9637  * @param[in] reg_type
9638  *   Type of device metadata register
9639  * @param[in] value
9640  *   Register value
9641  * @param[in] mask
9642  *   Register mask
9643  */
9644 static void
9645 flow_dv_match_meta_reg(void *matcher, void *key,
9646                        enum modify_reg reg_type,
9647                        uint32_t data, uint32_t mask)
9648 {
9649         void *misc2_m =
9650                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
9651         void *misc2_v =
9652                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9653         uint32_t temp;
9654
9655         data &= mask;
9656         switch (reg_type) {
9657         case REG_A:
9658                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
9659                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
9660                 break;
9661         case REG_B:
9662                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
9663                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
9664                 break;
9665         case REG_C_0:
9666                 /*
9667                  * The metadata register C0 field might be divided into
9668                  * source vport index and META item value, we should set
9669                  * this field according to specified mask, not as whole one.
9670                  */
9671                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
9672                 temp |= mask;
9673                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
9674                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
9675                 temp &= ~mask;
9676                 temp |= data;
9677                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
9678                 break;
9679         case REG_C_1:
9680                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
9681                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
9682                 break;
9683         case REG_C_2:
9684                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
9685                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
9686                 break;
9687         case REG_C_3:
9688                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
9689                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
9690                 break;
9691         case REG_C_4:
9692                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
9693                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
9694                 break;
9695         case REG_C_5:
9696                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
9697                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
9698                 break;
9699         case REG_C_6:
9700                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
9701                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
9702                 break;
9703         case REG_C_7:
9704                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
9705                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
9706                 break;
9707         default:
9708                 MLX5_ASSERT(false);
9709                 break;
9710         }
9711 }
9712
9713 /**
9714  * Add MARK item to matcher
9715  *
9716  * @param[in] dev
9717  *   The device to configure through.
9718  * @param[in, out] matcher
9719  *   Flow matcher.
9720  * @param[in, out] key
9721  *   Flow matcher value.
9722  * @param[in] item
9723  *   Flow pattern to translate.
9724  */
9725 static void
9726 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
9727                             void *matcher, void *key,
9728                             const struct rte_flow_item *item)
9729 {
9730         struct mlx5_priv *priv = dev->data->dev_private;
9731         const struct rte_flow_item_mark *mark;
9732         uint32_t value;
9733         uint32_t mask;
9734
9735         mark = item->mask ? (const void *)item->mask :
9736                             &rte_flow_item_mark_mask;
9737         mask = mark->id & priv->sh->dv_mark_mask;
9738         mark = (const void *)item->spec;
9739         MLX5_ASSERT(mark);
9740         value = mark->id & priv->sh->dv_mark_mask & mask;
9741         if (mask) {
9742                 enum modify_reg reg;
9743
9744                 /* Get the metadata register index for the mark. */
9745                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
9746                 MLX5_ASSERT(reg > 0);
9747                 if (reg == REG_C_0) {
9748                         struct mlx5_priv *priv = dev->data->dev_private;
9749                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9750                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9751
9752                         mask &= msk_c0;
9753                         mask <<= shl_c0;
9754                         value <<= shl_c0;
9755                 }
9756                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9757         }
9758 }
9759
9760 /**
9761  * Add META item to matcher
9762  *
9763  * @param[in] dev
9764  *   The devich to configure through.
9765  * @param[in, out] matcher
9766  *   Flow matcher.
9767  * @param[in, out] key
9768  *   Flow matcher value.
9769  * @param[in] attr
9770  *   Attributes of flow that includes this item.
9771  * @param[in] item
9772  *   Flow pattern to translate.
9773  */
9774 static void
9775 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
9776                             void *matcher, void *key,
9777                             const struct rte_flow_attr *attr,
9778                             const struct rte_flow_item *item)
9779 {
9780         const struct rte_flow_item_meta *meta_m;
9781         const struct rte_flow_item_meta *meta_v;
9782
9783         meta_m = (const void *)item->mask;
9784         if (!meta_m)
9785                 meta_m = &rte_flow_item_meta_mask;
9786         meta_v = (const void *)item->spec;
9787         if (meta_v) {
9788                 int reg;
9789                 uint32_t value = meta_v->data;
9790                 uint32_t mask = meta_m->data;
9791
9792                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
9793                 if (reg < 0)
9794                         return;
9795                 MLX5_ASSERT(reg != REG_NON);
9796                 if (reg == REG_C_0) {
9797                         struct mlx5_priv *priv = dev->data->dev_private;
9798                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9799                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9800
9801                         mask &= msk_c0;
9802                         mask <<= shl_c0;
9803                         value <<= shl_c0;
9804                 }
9805                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9806         }
9807 }
9808
9809 /**
9810  * Add vport metadata Reg C0 item to matcher
9811  *
9812  * @param[in, out] matcher
9813  *   Flow matcher.
9814  * @param[in, out] key
9815  *   Flow matcher value.
9816  * @param[in] reg
9817  *   Flow pattern to translate.
9818  */
9819 static void
9820 flow_dv_translate_item_meta_vport(void *matcher, void *key,
9821                                   uint32_t value, uint32_t mask)
9822 {
9823         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
9824 }
9825
9826 /**
9827  * Add tag item to matcher
9828  *
9829  * @param[in] dev
9830  *   The devich to configure through.
9831  * @param[in, out] matcher
9832  *   Flow matcher.
9833  * @param[in, out] key
9834  *   Flow matcher value.
9835  * @param[in] item
9836  *   Flow pattern to translate.
9837  */
9838 static void
9839 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
9840                                 void *matcher, void *key,
9841                                 const struct rte_flow_item *item)
9842 {
9843         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
9844         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
9845         uint32_t mask, value;
9846
9847         MLX5_ASSERT(tag_v);
9848         value = tag_v->data;
9849         mask = tag_m ? tag_m->data : UINT32_MAX;
9850         if (tag_v->id == REG_C_0) {
9851                 struct mlx5_priv *priv = dev->data->dev_private;
9852                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9853                 uint32_t shl_c0 = rte_bsf32(msk_c0);
9854
9855                 mask &= msk_c0;
9856                 mask <<= shl_c0;
9857                 value <<= shl_c0;
9858         }
9859         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
9860 }
9861
9862 /**
9863  * Add TAG item to matcher
9864  *
9865  * @param[in] dev
9866  *   The devich to configure through.
9867  * @param[in, out] matcher
9868  *   Flow matcher.
9869  * @param[in, out] key
9870  *   Flow matcher value.
9871  * @param[in] item
9872  *   Flow pattern to translate.
9873  */
9874 static void
9875 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
9876                            void *matcher, void *key,
9877                            const struct rte_flow_item *item)
9878 {
9879         const struct rte_flow_item_tag *tag_v = item->spec;
9880         const struct rte_flow_item_tag *tag_m = item->mask;
9881         enum modify_reg reg;
9882
9883         MLX5_ASSERT(tag_v);
9884         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
9885         /* Get the metadata register index for the tag. */
9886         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
9887         MLX5_ASSERT(reg > 0);
9888         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
9889 }
9890
9891 /**
9892  * Add source vport match to the specified matcher.
9893  *
9894  * @param[in, out] matcher
9895  *   Flow matcher.
9896  * @param[in, out] key
9897  *   Flow matcher value.
9898  * @param[in] port
9899  *   Source vport value to match
9900  * @param[in] mask
9901  *   Mask
9902  */
9903 static void
9904 flow_dv_translate_item_source_vport(void *matcher, void *key,
9905                                     int16_t port, uint16_t mask)
9906 {
9907         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9908         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9909
9910         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
9911         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
9912 }
9913
9914 /**
9915  * Translate port-id item to eswitch match on  port-id.
9916  *
9917  * @param[in] dev
9918  *   The devich to configure through.
9919  * @param[in, out] matcher
9920  *   Flow matcher.
9921  * @param[in, out] key
9922  *   Flow matcher value.
9923  * @param[in] item
9924  *   Flow pattern to translate.
9925  * @param[in]
9926  *   Flow attributes.
9927  *
9928  * @return
9929  *   0 on success, a negative errno value otherwise.
9930  */
9931 static int
9932 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
9933                                void *key, const struct rte_flow_item *item,
9934                                const struct rte_flow_attr *attr)
9935 {
9936         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
9937         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
9938         struct mlx5_priv *priv;
9939         uint16_t mask, id;
9940
9941         if (pid_v && pid_v->id == MLX5_PORT_ESW_MGR) {
9942                 flow_dv_translate_item_source_vport(matcher, key,
9943                         flow_dv_get_esw_manager_vport_id(dev), 0xffff);
9944                 return 0;
9945         }
9946         mask = pid_m ? pid_m->id : 0xffff;
9947         id = pid_v ? pid_v->id : dev->data->port_id;
9948         priv = mlx5_port_to_eswitch_info(id, item == NULL);
9949         if (!priv)
9950                 return -rte_errno;
9951         /*
9952          * Translate to vport field or to metadata, depending on mode.
9953          * Kernel can use either misc.source_port or half of C0 metadata
9954          * register.
9955          */
9956         if (priv->vport_meta_mask) {
9957                 /*
9958                  * Provide the hint for SW steering library
9959                  * to insert the flow into ingress domain and
9960                  * save the extra vport match.
9961                  */
9962                 if (mask == 0xffff && priv->vport_id == 0xffff &&
9963                     priv->pf_bond < 0 && attr->transfer)
9964                         flow_dv_translate_item_source_vport
9965                                 (matcher, key, priv->vport_id, mask);
9966                 /*
9967                  * We should always set the vport metadata register,
9968                  * otherwise the SW steering library can drop
9969                  * the rule if wire vport metadata value is not zero,
9970                  * it depends on kernel configuration.
9971                  */
9972                 flow_dv_translate_item_meta_vport(matcher, key,
9973                                                   priv->vport_meta_tag,
9974                                                   priv->vport_meta_mask);
9975         } else {
9976                 flow_dv_translate_item_source_vport(matcher, key,
9977                                                     priv->vport_id, mask);
9978         }
9979         return 0;
9980 }
9981
9982 /**
9983  * Add ICMP6 item to matcher and to the value.
9984  *
9985  * @param[in, out] matcher
9986  *   Flow matcher.
9987  * @param[in, out] key
9988  *   Flow matcher value.
9989  * @param[in] item
9990  *   Flow pattern to translate.
9991  * @param[in] inner
9992  *   Item is inner pattern.
9993  */
9994 static void
9995 flow_dv_translate_item_icmp6(void *matcher, void *key,
9996                               const struct rte_flow_item *item,
9997                               int inner)
9998 {
9999         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
10000         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
10001         void *headers_m;
10002         void *headers_v;
10003         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
10004                                      misc_parameters_3);
10005         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
10006         if (inner) {
10007                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
10008                                          inner_headers);
10009                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
10010         } else {
10011                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
10012                                          outer_headers);
10013                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
10014         }
10015         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
10016         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
10017         if (!icmp6_v)
10018                 return;
10019         if (!icmp6_m)
10020                 icmp6_m = &rte_flow_item_icmp6_mask;
10021         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
10022         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
10023                  icmp6_v->type & icmp6_m->type);
10024         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
10025         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
10026                  icmp6_v->code & icmp6_m->code);
10027 }
10028
10029 /**
10030  * Add ICMP item to matcher and to the value.
10031  *
10032  * @param[in, out] matcher
10033  *   Flow matcher.
10034  * @param[in, out] key
10035  *   Flow matcher value.
10036  * @param[in] item
10037  *   Flow pattern to translate.
10038  * @param[in] inner
10039  *   Item is inner pattern.
10040  */
10041 static void
10042 flow_dv_translate_item_icmp(void *matcher, void *key,
10043                             const struct rte_flow_item *item,
10044                             int inner)
10045 {
10046         const struct rte_flow_item_icmp *icmp_m = item->mask;
10047         const struct rte_flow_item_icmp *icmp_v = item->spec;
10048         uint32_t icmp_header_data_m = 0;
10049         uint32_t icmp_header_data_v = 0;
10050         void *headers_m;
10051         void *headers_v;
10052         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
10053                                      misc_parameters_3);
10054         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
10055         if (inner) {
10056                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
10057                                          inner_headers);
10058                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
10059         } else {
10060                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
10061                                          outer_headers);
10062                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
10063         }
10064         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
10065         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
10066         if (!icmp_v)
10067                 return;
10068         if (!icmp_m)
10069                 icmp_m = &rte_flow_item_icmp_mask;
10070         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
10071                  icmp_m->hdr.icmp_type);
10072         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
10073                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
10074         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
10075                  icmp_m->hdr.icmp_code);
10076         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
10077                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
10078         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
10079         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
10080         if (icmp_header_data_m) {
10081                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
10082                 icmp_header_data_v |=
10083                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
10084                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
10085                          icmp_header_data_m);
10086                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
10087                          icmp_header_data_v & icmp_header_data_m);
10088         }
10089 }
10090
10091 /**
10092  * Add GTP item to matcher and to the value.
10093  *
10094  * @param[in, out] matcher
10095  *   Flow matcher.
10096  * @param[in, out] key
10097  *   Flow matcher value.
10098  * @param[in] item
10099  *   Flow pattern to translate.
10100  * @param[in] inner
10101  *   Item is inner pattern.
10102  */
10103 static void
10104 flow_dv_translate_item_gtp(void *matcher, void *key,
10105                            const struct rte_flow_item *item, int inner)
10106 {
10107         const struct rte_flow_item_gtp *gtp_m = item->mask;
10108         const struct rte_flow_item_gtp *gtp_v = item->spec;
10109         void *headers_m;
10110         void *headers_v;
10111         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
10112                                      misc_parameters_3);
10113         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
10114         uint16_t dport = RTE_GTPU_UDP_PORT;
10115
10116         if (inner) {
10117                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
10118                                          inner_headers);
10119                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
10120         } else {
10121                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
10122                                          outer_headers);
10123                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
10124         }
10125         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
10126                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
10127                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
10128         }
10129         if (!gtp_v)
10130                 return;
10131         if (!gtp_m)
10132                 gtp_m = &rte_flow_item_gtp_mask;
10133         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
10134                  gtp_m->v_pt_rsv_flags);
10135         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
10136                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
10137         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
10138         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
10139                  gtp_v->msg_type & gtp_m->msg_type);
10140         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
10141                  rte_be_to_cpu_32(gtp_m->teid));
10142         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
10143                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
10144 }
10145
10146 /**
10147  * Add GTP PSC item to matcher.
10148  *
10149  * @param[in, out] matcher
10150  *   Flow matcher.
10151  * @param[in, out] key
10152  *   Flow matcher value.
10153  * @param[in] item
10154  *   Flow pattern to translate.
10155  */
10156 static int
10157 flow_dv_translate_item_gtp_psc(void *matcher, void *key,
10158                                const struct rte_flow_item *item)
10159 {
10160         const struct rte_flow_item_gtp_psc *gtp_psc_m = item->mask;
10161         const struct rte_flow_item_gtp_psc *gtp_psc_v = item->spec;
10162         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
10163                         misc_parameters_3);
10164         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
10165         union {
10166                 uint32_t w32;
10167                 struct {
10168                         uint16_t seq_num;
10169                         uint8_t npdu_num;
10170                         uint8_t next_ext_header_type;
10171                 };
10172         } dw_2;
10173         uint8_t gtp_flags;
10174
10175         /* Always set E-flag match on one, regardless of GTP item settings. */
10176         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_m, gtpu_msg_flags);
10177         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
10178         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags, gtp_flags);
10179         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_v, gtpu_msg_flags);
10180         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
10181         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags, gtp_flags);
10182         /*Set next extension header type. */
10183         dw_2.seq_num = 0;
10184         dw_2.npdu_num = 0;
10185         dw_2.next_ext_header_type = 0xff;
10186         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_dw_2,
10187                  rte_cpu_to_be_32(dw_2.w32));
10188         dw_2.seq_num = 0;
10189         dw_2.npdu_num = 0;
10190         dw_2.next_ext_header_type = 0x85;
10191         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_dw_2,
10192                  rte_cpu_to_be_32(dw_2.w32));
10193         if (gtp_psc_v) {
10194                 union {
10195                         uint32_t w32;
10196                         struct {
10197                                 uint8_t len;
10198                                 uint8_t type_flags;
10199                                 uint8_t qfi;
10200                                 uint8_t reserved;
10201                         };
10202                 } dw_0;
10203
10204                 /*Set extension header PDU type and Qos. */
10205                 if (!gtp_psc_m)
10206                         gtp_psc_m = &rte_flow_item_gtp_psc_mask;
10207                 dw_0.w32 = 0;
10208                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_m->hdr.type);
10209                 dw_0.qfi = gtp_psc_m->hdr.qfi;
10210                 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_first_ext_dw_0,
10211                          rte_cpu_to_be_32(dw_0.w32));
10212                 dw_0.w32 = 0;
10213                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_v->hdr.type &
10214                                                         gtp_psc_m->hdr.type);
10215                 dw_0.qfi = gtp_psc_v->hdr.qfi & gtp_psc_m->hdr.qfi;
10216                 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_first_ext_dw_0,
10217                          rte_cpu_to_be_32(dw_0.w32));
10218         }
10219         return 0;
10220 }
10221
10222 /**
10223  * Add eCPRI item to matcher and to the value.
10224  *
10225  * @param[in] dev
10226  *   The devich to configure through.
10227  * @param[in, out] matcher
10228  *   Flow matcher.
10229  * @param[in, out] key
10230  *   Flow matcher value.
10231  * @param[in] item
10232  *   Flow pattern to translate.
10233  * @param[in] last_item
10234  *   Last item flags.
10235  */
10236 static void
10237 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
10238                              void *key, const struct rte_flow_item *item,
10239                              uint64_t last_item)
10240 {
10241         struct mlx5_priv *priv = dev->data->dev_private;
10242         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
10243         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
10244         struct rte_ecpri_common_hdr common;
10245         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
10246                                      misc_parameters_4);
10247         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
10248         uint32_t *samples;
10249         void *dw_m;
10250         void *dw_v;
10251
10252         /*
10253          * In case of eCPRI over Ethernet, if EtherType is not specified,
10254          * match on eCPRI EtherType implicitly.
10255          */
10256         if (last_item & MLX5_FLOW_LAYER_OUTER_L2) {
10257                 void *hdrs_m, *hdrs_v, *l2m, *l2v;
10258
10259                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
10260                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
10261                 l2m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, ethertype);
10262                 l2v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype);
10263                 if (*(uint16_t *)l2m == 0 && *(uint16_t *)l2v == 0) {
10264                         *(uint16_t *)l2m = UINT16_MAX;
10265                         *(uint16_t *)l2v = RTE_BE16(RTE_ETHER_TYPE_ECPRI);
10266                 }
10267         }
10268         if (!ecpri_v)
10269                 return;
10270         if (!ecpri_m)
10271                 ecpri_m = &rte_flow_item_ecpri_mask;
10272         /*
10273          * Maximal four DW samples are supported in a single matching now.
10274          * Two are used now for a eCPRI matching:
10275          * 1. Type: one byte, mask should be 0x00ff0000 in network order
10276          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
10277          *    if any.
10278          */
10279         if (!ecpri_m->hdr.common.u32)
10280                 return;
10281         samples = priv->sh->ecpri_parser.ids;
10282         /* Need to take the whole DW as the mask to fill the entry. */
10283         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
10284                             prog_sample_field_value_0);
10285         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
10286                             prog_sample_field_value_0);
10287         /* Already big endian (network order) in the header. */
10288         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
10289         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32 & ecpri_m->hdr.common.u32;
10290         /* Sample#0, used for matching type, offset 0. */
10291         MLX5_SET(fte_match_set_misc4, misc4_m,
10292                  prog_sample_field_id_0, samples[0]);
10293         /* It makes no sense to set the sample ID in the mask field. */
10294         MLX5_SET(fte_match_set_misc4, misc4_v,
10295                  prog_sample_field_id_0, samples[0]);
10296         /*
10297          * Checking if message body part needs to be matched.
10298          * Some wildcard rules only matching type field should be supported.
10299          */
10300         if (ecpri_m->hdr.dummy[0]) {
10301                 common.u32 = rte_be_to_cpu_32(ecpri_v->hdr.common.u32);
10302                 switch (common.type) {
10303                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
10304                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
10305                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
10306                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
10307                                             prog_sample_field_value_1);
10308                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
10309                                             prog_sample_field_value_1);
10310                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
10311                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0] &
10312                                             ecpri_m->hdr.dummy[0];
10313                         /* Sample#1, to match message body, offset 4. */
10314                         MLX5_SET(fte_match_set_misc4, misc4_m,
10315                                  prog_sample_field_id_1, samples[1]);
10316                         MLX5_SET(fte_match_set_misc4, misc4_v,
10317                                  prog_sample_field_id_1, samples[1]);
10318                         break;
10319                 default:
10320                         /* Others, do not match any sample ID. */
10321                         break;
10322                 }
10323         }
10324 }
10325
10326 /*
10327  * Add connection tracking status item to matcher
10328  *
10329  * @param[in] dev
10330  *   The devich to configure through.
10331  * @param[in, out] matcher
10332  *   Flow matcher.
10333  * @param[in, out] key
10334  *   Flow matcher value.
10335  * @param[in] item
10336  *   Flow pattern to translate.
10337  */
10338 static void
10339 flow_dv_translate_item_aso_ct(struct rte_eth_dev *dev,
10340                               void *matcher, void *key,
10341                               const struct rte_flow_item *item)
10342 {
10343         uint32_t reg_value = 0;
10344         int reg_id;
10345         /* 8LSB 0b 11/0000/11, middle 4 bits are reserved. */
10346         uint32_t reg_mask = 0;
10347         const struct rte_flow_item_conntrack *spec = item->spec;
10348         const struct rte_flow_item_conntrack *mask = item->mask;
10349         uint32_t flags;
10350         struct rte_flow_error error;
10351
10352         if (!mask)
10353                 mask = &rte_flow_item_conntrack_mask;
10354         if (!spec || !mask->flags)
10355                 return;
10356         flags = spec->flags & mask->flags;
10357         /* The conflict should be checked in the validation. */
10358         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID)
10359                 reg_value |= MLX5_CT_SYNDROME_VALID;
10360         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
10361                 reg_value |= MLX5_CT_SYNDROME_STATE_CHANGE;
10362         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID)
10363                 reg_value |= MLX5_CT_SYNDROME_INVALID;
10364         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)
10365                 reg_value |= MLX5_CT_SYNDROME_TRAP;
10366         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
10367                 reg_value |= MLX5_CT_SYNDROME_BAD_PACKET;
10368         if (mask->flags & (RTE_FLOW_CONNTRACK_PKT_STATE_VALID |
10369                            RTE_FLOW_CONNTRACK_PKT_STATE_INVALID |
10370                            RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED))
10371                 reg_mask |= 0xc0;
10372         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
10373                 reg_mask |= MLX5_CT_SYNDROME_STATE_CHANGE;
10374         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
10375                 reg_mask |= MLX5_CT_SYNDROME_BAD_PACKET;
10376         /* The REG_C_x value could be saved during startup. */
10377         reg_id = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, &error);
10378         if (reg_id == REG_NON)
10379                 return;
10380         flow_dv_match_meta_reg(matcher, key, (enum modify_reg)reg_id,
10381                                reg_value, reg_mask);
10382 }
10383
10384 static void
10385 flow_dv_translate_item_flex(struct rte_eth_dev *dev, void *matcher, void *key,
10386                             const struct rte_flow_item *item,
10387                             struct mlx5_flow *dev_flow, bool is_inner)
10388 {
10389         const struct rte_flow_item_flex *spec =
10390                 (const struct rte_flow_item_flex *)item->spec;
10391         int index = mlx5_flex_acquire_index(dev, spec->handle, false);
10392
10393         MLX5_ASSERT(index >= 0 && index <= (int)(sizeof(uint32_t) * CHAR_BIT));
10394         if (index < 0)
10395                 return;
10396         if (!(dev_flow->handle->flex_item & RTE_BIT32(index))) {
10397                 /* Don't count both inner and outer flex items in one rule. */
10398                 if (mlx5_flex_acquire_index(dev, spec->handle, true) != index)
10399                         MLX5_ASSERT(false);
10400                 dev_flow->handle->flex_item |= (uint8_t)RTE_BIT32(index);
10401         }
10402         mlx5_flex_flow_translate_item(dev, matcher, key, item, is_inner);
10403 }
10404
10405 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
10406
10407 #define HEADER_IS_ZERO(match_criteria, headers)                              \
10408         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
10409                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
10410
10411 /**
10412  * Calculate flow matcher enable bitmap.
10413  *
10414  * @param match_criteria
10415  *   Pointer to flow matcher criteria.
10416  *
10417  * @return
10418  *   Bitmap of enabled fields.
10419  */
10420 static uint8_t
10421 flow_dv_matcher_enable(uint32_t *match_criteria)
10422 {
10423         uint8_t match_criteria_enable;
10424
10425         match_criteria_enable =
10426                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
10427                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
10428         match_criteria_enable |=
10429                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
10430                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
10431         match_criteria_enable |=
10432                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
10433                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
10434         match_criteria_enable |=
10435                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
10436                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
10437         match_criteria_enable |=
10438                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
10439                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
10440         match_criteria_enable |=
10441                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
10442                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
10443         match_criteria_enable |=
10444                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_5)) <<
10445                 MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT;
10446         return match_criteria_enable;
10447 }
10448
10449 static void
10450 __flow_dv_adjust_buf_size(size_t *size, uint8_t match_criteria)
10451 {
10452         /*
10453          * Check flow matching criteria first, subtract misc5/4 length if flow
10454          * doesn't own misc5/4 parameters. In some old rdma-core releases,
10455          * misc5/4 are not supported, and matcher creation failure is expected
10456          * w/o subtraction. If misc5 is provided, misc4 must be counted in since
10457          * misc5 is right after misc4.
10458          */
10459         if (!(match_criteria & (1 << MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT))) {
10460                 *size = MLX5_ST_SZ_BYTES(fte_match_param) -
10461                         MLX5_ST_SZ_BYTES(fte_match_set_misc5);
10462                 if (!(match_criteria & (1 <<
10463                         MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT))) {
10464                         *size -= MLX5_ST_SZ_BYTES(fte_match_set_misc4);
10465                 }
10466         }
10467 }
10468
10469 static struct mlx5_list_entry *
10470 flow_dv_matcher_clone_cb(void *tool_ctx __rte_unused,
10471                          struct mlx5_list_entry *entry, void *cb_ctx)
10472 {
10473         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10474         struct mlx5_flow_dv_matcher *ref = ctx->data;
10475         struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
10476                                                             typeof(*tbl), tbl);
10477         struct mlx5_flow_dv_matcher *resource = mlx5_malloc(MLX5_MEM_ANY,
10478                                                             sizeof(*resource),
10479                                                             0, SOCKET_ID_ANY);
10480
10481         if (!resource) {
10482                 rte_flow_error_set(ctx->error, ENOMEM,
10483                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10484                                    "cannot create matcher");
10485                 return NULL;
10486         }
10487         memcpy(resource, entry, sizeof(*resource));
10488         resource->tbl = &tbl->tbl;
10489         return &resource->entry;
10490 }
10491
10492 static void
10493 flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused,
10494                              struct mlx5_list_entry *entry)
10495 {
10496         mlx5_free(entry);
10497 }
10498
10499 struct mlx5_list_entry *
10500 flow_dv_tbl_create_cb(void *tool_ctx, void *cb_ctx)
10501 {
10502         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10503         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10504         struct rte_eth_dev *dev = ctx->dev;
10505         struct mlx5_flow_tbl_data_entry *tbl_data;
10506         struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data2;
10507         struct rte_flow_error *error = ctx->error;
10508         union mlx5_flow_tbl_key key = { .v64 = *(uint64_t *)(ctx->data) };
10509         struct mlx5_flow_tbl_resource *tbl;
10510         void *domain;
10511         uint32_t idx = 0;
10512         int ret;
10513
10514         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
10515         if (!tbl_data) {
10516                 rte_flow_error_set(error, ENOMEM,
10517                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10518                                    NULL,
10519                                    "cannot allocate flow table data entry");
10520                 return NULL;
10521         }
10522         tbl_data->idx = idx;
10523         tbl_data->tunnel = tt_prm->tunnel;
10524         tbl_data->group_id = tt_prm->group_id;
10525         tbl_data->external = !!tt_prm->external;
10526         tbl_data->tunnel_offload = is_tunnel_offload_active(dev);
10527         tbl_data->is_egress = !!key.is_egress;
10528         tbl_data->is_transfer = !!key.is_fdb;
10529         tbl_data->dummy = !!key.dummy;
10530         tbl_data->level = key.level;
10531         tbl_data->id = key.id;
10532         tbl = &tbl_data->tbl;
10533         if (key.dummy)
10534                 return &tbl_data->entry;
10535         if (key.is_fdb)
10536                 domain = sh->fdb_domain;
10537         else if (key.is_egress)
10538                 domain = sh->tx_domain;
10539         else
10540                 domain = sh->rx_domain;
10541         ret = mlx5_flow_os_create_flow_tbl(domain, key.level, &tbl->obj);
10542         if (ret) {
10543                 rte_flow_error_set(error, ENOMEM,
10544                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10545                                    NULL, "cannot create flow table object");
10546                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10547                 return NULL;
10548         }
10549         if (key.level != 0) {
10550                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
10551                                         (tbl->obj, &tbl_data->jump.action);
10552                 if (ret) {
10553                         rte_flow_error_set(error, ENOMEM,
10554                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10555                                            NULL,
10556                                            "cannot create flow jump action");
10557                         mlx5_flow_os_destroy_flow_tbl(tbl->obj);
10558                         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10559                         return NULL;
10560                 }
10561         }
10562         MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
10563               key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
10564               key.level, key.id);
10565         tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
10566                                               flow_dv_matcher_create_cb,
10567                                               flow_dv_matcher_match_cb,
10568                                               flow_dv_matcher_remove_cb,
10569                                               flow_dv_matcher_clone_cb,
10570                                               flow_dv_matcher_clone_free_cb);
10571         if (!tbl_data->matchers) {
10572                 rte_flow_error_set(error, ENOMEM,
10573                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10574                                    NULL,
10575                                    "cannot create tbl matcher list");
10576                 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
10577                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
10578                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10579                 return NULL;
10580         }
10581         return &tbl_data->entry;
10582 }
10583
10584 int
10585 flow_dv_tbl_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
10586                      void *cb_ctx)
10587 {
10588         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10589         struct mlx5_flow_tbl_data_entry *tbl_data =
10590                 container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10591         union mlx5_flow_tbl_key key = { .v64 =  *(uint64_t *)(ctx->data) };
10592
10593         return tbl_data->level != key.level ||
10594                tbl_data->id != key.id ||
10595                tbl_data->dummy != key.dummy ||
10596                tbl_data->is_transfer != !!key.is_fdb ||
10597                tbl_data->is_egress != !!key.is_egress;
10598 }
10599
10600 struct mlx5_list_entry *
10601 flow_dv_tbl_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
10602                       void *cb_ctx)
10603 {
10604         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10605         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10606         struct mlx5_flow_tbl_data_entry *tbl_data;
10607         struct rte_flow_error *error = ctx->error;
10608         uint32_t idx = 0;
10609
10610         tbl_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
10611         if (!tbl_data) {
10612                 rte_flow_error_set(error, ENOMEM,
10613                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10614                                    NULL,
10615                                    "cannot allocate flow table data entry");
10616                 return NULL;
10617         }
10618         memcpy(tbl_data, oentry, sizeof(*tbl_data));
10619         tbl_data->idx = idx;
10620         return &tbl_data->entry;
10621 }
10622
10623 void
10624 flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10625 {
10626         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10627         struct mlx5_flow_tbl_data_entry *tbl_data =
10628                     container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10629
10630         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
10631 }
10632
10633 /**
10634  * Get a flow table.
10635  *
10636  * @param[in, out] dev
10637  *   Pointer to rte_eth_dev structure.
10638  * @param[in] table_level
10639  *   Table level to use.
10640  * @param[in] egress
10641  *   Direction of the table.
10642  * @param[in] transfer
10643  *   E-Switch or NIC flow.
10644  * @param[in] dummy
10645  *   Dummy entry for dv API.
10646  * @param[in] table_id
10647  *   Table id to use.
10648  * @param[out] error
10649  *   pointer to error structure.
10650  *
10651  * @return
10652  *   Returns tables resource based on the index, NULL in case of failed.
10653  */
10654 struct mlx5_flow_tbl_resource *
10655 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
10656                          uint32_t table_level, uint8_t egress,
10657                          uint8_t transfer,
10658                          bool external,
10659                          const struct mlx5_flow_tunnel *tunnel,
10660                          uint32_t group_id, uint8_t dummy,
10661                          uint32_t table_id,
10662                          struct rte_flow_error *error)
10663 {
10664         struct mlx5_priv *priv = dev->data->dev_private;
10665         union mlx5_flow_tbl_key table_key = {
10666                 {
10667                         .level = table_level,
10668                         .id = table_id,
10669                         .reserved = 0,
10670                         .dummy = !!dummy,
10671                         .is_fdb = !!transfer,
10672                         .is_egress = !!egress,
10673                 }
10674         };
10675         struct mlx5_flow_tbl_tunnel_prm tt_prm = {
10676                 .tunnel = tunnel,
10677                 .group_id = group_id,
10678                 .external = external,
10679         };
10680         struct mlx5_flow_cb_ctx ctx = {
10681                 .dev = dev,
10682                 .error = error,
10683                 .data = &table_key.v64,
10684                 .data2 = &tt_prm,
10685         };
10686         struct mlx5_list_entry *entry;
10687         struct mlx5_flow_tbl_data_entry *tbl_data;
10688
10689         entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx);
10690         if (!entry) {
10691                 rte_flow_error_set(error, ENOMEM,
10692                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10693                                    "cannot get table");
10694                 return NULL;
10695         }
10696         DRV_LOG(DEBUG, "table_level %u table_id %u "
10697                 "tunnel %u group %u registered.",
10698                 table_level, table_id,
10699                 tunnel ? tunnel->tunnel_id : 0, group_id);
10700         tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10701         return &tbl_data->tbl;
10702 }
10703
10704 void
10705 flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10706 {
10707         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10708         struct mlx5_flow_tbl_data_entry *tbl_data =
10709                     container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10710
10711         MLX5_ASSERT(entry && sh);
10712         if (tbl_data->jump.action)
10713                 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
10714         if (tbl_data->tbl.obj)
10715                 mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj);
10716         if (tbl_data->tunnel_offload && tbl_data->external) {
10717                 struct mlx5_list_entry *he;
10718                 struct mlx5_hlist *tunnel_grp_hash;
10719                 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
10720                 union tunnel_tbl_key tunnel_key = {
10721                         .tunnel_id = tbl_data->tunnel ?
10722                                         tbl_data->tunnel->tunnel_id : 0,
10723                         .group = tbl_data->group_id
10724                 };
10725                 uint32_t table_level = tbl_data->level;
10726                 struct mlx5_flow_cb_ctx ctx = {
10727                         .data = (void *)&tunnel_key.val,
10728                 };
10729
10730                 tunnel_grp_hash = tbl_data->tunnel ?
10731                                         tbl_data->tunnel->groups :
10732                                         thub->groups;
10733                 he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, &ctx);
10734                 if (he)
10735                         mlx5_hlist_unregister(tunnel_grp_hash, he);
10736                 DRV_LOG(DEBUG,
10737                         "table_level %u id %u tunnel %u group %u released.",
10738                         table_level,
10739                         tbl_data->id,
10740                         tbl_data->tunnel ?
10741                         tbl_data->tunnel->tunnel_id : 0,
10742                         tbl_data->group_id);
10743         }
10744         if (tbl_data->matchers)
10745                 mlx5_list_destroy(tbl_data->matchers);
10746         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
10747 }
10748
10749 /**
10750  * Release a flow table.
10751  *
10752  * @param[in] sh
10753  *   Pointer to device shared structure.
10754  * @param[in] tbl
10755  *   Table resource to be released.
10756  *
10757  * @return
10758  *   Returns 0 if table was released, else return 1;
10759  */
10760 static int
10761 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
10762                              struct mlx5_flow_tbl_resource *tbl)
10763 {
10764         struct mlx5_flow_tbl_data_entry *tbl_data =
10765                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10766
10767         if (!tbl)
10768                 return 0;
10769         return mlx5_hlist_unregister(sh->flow_tbls, &tbl_data->entry);
10770 }
10771
10772 int
10773 flow_dv_matcher_match_cb(void *tool_ctx __rte_unused,
10774                          struct mlx5_list_entry *entry, void *cb_ctx)
10775 {
10776         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10777         struct mlx5_flow_dv_matcher *ref = ctx->data;
10778         struct mlx5_flow_dv_matcher *cur = container_of(entry, typeof(*cur),
10779                                                         entry);
10780
10781         return cur->crc != ref->crc ||
10782                cur->priority != ref->priority ||
10783                memcmp((const void *)cur->mask.buf,
10784                       (const void *)ref->mask.buf, ref->mask.size);
10785 }
10786
10787 struct mlx5_list_entry *
10788 flow_dv_matcher_create_cb(void *tool_ctx, void *cb_ctx)
10789 {
10790         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10791         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10792         struct mlx5_flow_dv_matcher *ref = ctx->data;
10793         struct mlx5_flow_dv_matcher *resource;
10794         struct mlx5dv_flow_matcher_attr dv_attr = {
10795                 .type = IBV_FLOW_ATTR_NORMAL,
10796                 .match_mask = (void *)&ref->mask,
10797         };
10798         struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
10799                                                             typeof(*tbl), tbl);
10800         int ret;
10801
10802         resource = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*resource), 0,
10803                                SOCKET_ID_ANY);
10804         if (!resource) {
10805                 rte_flow_error_set(ctx->error, ENOMEM,
10806                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10807                                    "cannot create matcher");
10808                 return NULL;
10809         }
10810         *resource = *ref;
10811         dv_attr.match_criteria_enable =
10812                 flow_dv_matcher_enable(resource->mask.buf);
10813         __flow_dv_adjust_buf_size(&ref->mask.size,
10814                                   dv_attr.match_criteria_enable);
10815         dv_attr.priority = ref->priority;
10816         if (tbl->is_egress)
10817                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
10818         ret = mlx5_flow_os_create_flow_matcher(sh->cdev->ctx, &dv_attr,
10819                                                tbl->tbl.obj,
10820                                                &resource->matcher_object);
10821         if (ret) {
10822                 mlx5_free(resource);
10823                 rte_flow_error_set(ctx->error, ENOMEM,
10824                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10825                                    "cannot create matcher");
10826                 return NULL;
10827         }
10828         return &resource->entry;
10829 }
10830
10831 /**
10832  * Register the flow matcher.
10833  *
10834  * @param[in, out] dev
10835  *   Pointer to rte_eth_dev structure.
10836  * @param[in, out] matcher
10837  *   Pointer to flow matcher.
10838  * @param[in, out] key
10839  *   Pointer to flow table key.
10840  * @parm[in, out] dev_flow
10841  *   Pointer to the dev_flow.
10842  * @param[out] error
10843  *   pointer to error structure.
10844  *
10845  * @return
10846  *   0 on success otherwise -errno and errno is set.
10847  */
10848 static int
10849 flow_dv_matcher_register(struct rte_eth_dev *dev,
10850                          struct mlx5_flow_dv_matcher *ref,
10851                          union mlx5_flow_tbl_key *key,
10852                          struct mlx5_flow *dev_flow,
10853                          const struct mlx5_flow_tunnel *tunnel,
10854                          uint32_t group_id,
10855                          struct rte_flow_error *error)
10856 {
10857         struct mlx5_list_entry *entry;
10858         struct mlx5_flow_dv_matcher *resource;
10859         struct mlx5_flow_tbl_resource *tbl;
10860         struct mlx5_flow_tbl_data_entry *tbl_data;
10861         struct mlx5_flow_cb_ctx ctx = {
10862                 .error = error,
10863                 .data = ref,
10864         };
10865         /**
10866          * tunnel offload API requires this registration for cases when
10867          * tunnel match rule was inserted before tunnel set rule.
10868          */
10869         tbl = flow_dv_tbl_resource_get(dev, key->level,
10870                                        key->is_egress, key->is_fdb,
10871                                        dev_flow->external, tunnel,
10872                                        group_id, 0, key->id, error);
10873         if (!tbl)
10874                 return -rte_errno;      /* No need to refill the error info */
10875         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10876         ref->tbl = tbl;
10877         entry = mlx5_list_register(tbl_data->matchers, &ctx);
10878         if (!entry) {
10879                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
10880                 return rte_flow_error_set(error, ENOMEM,
10881                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10882                                           "cannot allocate ref memory");
10883         }
10884         resource = container_of(entry, typeof(*resource), entry);
10885         dev_flow->handle->dvh.matcher = resource;
10886         return 0;
10887 }
10888
10889 struct mlx5_list_entry *
10890 flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx)
10891 {
10892         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10893         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10894         struct mlx5_flow_dv_tag_resource *entry;
10895         uint32_t idx = 0;
10896         int ret;
10897
10898         entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
10899         if (!entry) {
10900                 rte_flow_error_set(ctx->error, ENOMEM,
10901                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10902                                    "cannot allocate resource memory");
10903                 return NULL;
10904         }
10905         entry->idx = idx;
10906         entry->tag_id = *(uint32_t *)(ctx->data);
10907         ret = mlx5_flow_os_create_flow_action_tag(entry->tag_id,
10908                                                   &entry->action);
10909         if (ret) {
10910                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx);
10911                 rte_flow_error_set(ctx->error, ENOMEM,
10912                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10913                                    NULL, "cannot create action");
10914                 return NULL;
10915         }
10916         return &entry->entry;
10917 }
10918
10919 int
10920 flow_dv_tag_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
10921                      void *cb_ctx)
10922 {
10923         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10924         struct mlx5_flow_dv_tag_resource *tag =
10925                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10926
10927         return *(uint32_t *)(ctx->data) != tag->tag_id;
10928 }
10929
10930 struct mlx5_list_entry *
10931 flow_dv_tag_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
10932                      void *cb_ctx)
10933 {
10934         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10935         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10936         struct mlx5_flow_dv_tag_resource *entry;
10937         uint32_t idx = 0;
10938
10939         entry = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
10940         if (!entry) {
10941                 rte_flow_error_set(ctx->error, ENOMEM,
10942                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10943                                    "cannot allocate tag resource memory");
10944                 return NULL;
10945         }
10946         memcpy(entry, oentry, sizeof(*entry));
10947         entry->idx = idx;
10948         return &entry->entry;
10949 }
10950
10951 void
10952 flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10953 {
10954         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10955         struct mlx5_flow_dv_tag_resource *tag =
10956                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10957
10958         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
10959 }
10960
10961 /**
10962  * Find existing tag resource or create and register a new one.
10963  *
10964  * @param dev[in, out]
10965  *   Pointer to rte_eth_dev structure.
10966  * @param[in, out] tag_be24
10967  *   Tag value in big endian then R-shift 8.
10968  * @parm[in, out] dev_flow
10969  *   Pointer to the dev_flow.
10970  * @param[out] error
10971  *   pointer to error structure.
10972  *
10973  * @return
10974  *   0 on success otherwise -errno and errno is set.
10975  */
10976 static int
10977 flow_dv_tag_resource_register
10978                         (struct rte_eth_dev *dev,
10979                          uint32_t tag_be24,
10980                          struct mlx5_flow *dev_flow,
10981                          struct rte_flow_error *error)
10982 {
10983         struct mlx5_priv *priv = dev->data->dev_private;
10984         struct mlx5_flow_dv_tag_resource *resource;
10985         struct mlx5_list_entry *entry;
10986         struct mlx5_flow_cb_ctx ctx = {
10987                                         .error = error,
10988                                         .data = &tag_be24,
10989                                         };
10990         struct mlx5_hlist *tag_table;
10991
10992         tag_table = flow_dv_hlist_prepare(priv->sh, &priv->sh->tag_table,
10993                                       "tags",
10994                                       MLX5_TAGS_HLIST_ARRAY_SIZE,
10995                                       false, false, priv->sh,
10996                                       flow_dv_tag_create_cb,
10997                                       flow_dv_tag_match_cb,
10998                                       flow_dv_tag_remove_cb,
10999                                       flow_dv_tag_clone_cb,
11000                                       flow_dv_tag_clone_free_cb,
11001                                       error);
11002         if (unlikely(!tag_table))
11003                 return -rte_errno;
11004         entry = mlx5_hlist_register(tag_table, tag_be24, &ctx);
11005         if (entry) {
11006                 resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
11007                                         entry);
11008                 dev_flow->handle->dvh.rix_tag = resource->idx;
11009                 dev_flow->dv.tag_resource = resource;
11010                 return 0;
11011         }
11012         return -rte_errno;
11013 }
11014
11015 void
11016 flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
11017 {
11018         struct mlx5_dev_ctx_shared *sh = tool_ctx;
11019         struct mlx5_flow_dv_tag_resource *tag =
11020                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
11021
11022         MLX5_ASSERT(tag && sh && tag->action);
11023         claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
11024         DRV_LOG(DEBUG, "Tag %p: removed.", (void *)tag);
11025         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
11026 }
11027
11028 /**
11029  * Release the tag.
11030  *
11031  * @param dev
11032  *   Pointer to Ethernet device.
11033  * @param tag_idx
11034  *   Tag index.
11035  *
11036  * @return
11037  *   1 while a reference on it exists, 0 when freed.
11038  */
11039 static int
11040 flow_dv_tag_release(struct rte_eth_dev *dev,
11041                     uint32_t tag_idx)
11042 {
11043         struct mlx5_priv *priv = dev->data->dev_private;
11044         struct mlx5_flow_dv_tag_resource *tag;
11045
11046         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
11047         if (!tag)
11048                 return 0;
11049         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
11050                 dev->data->port_id, (void *)tag, tag->entry.ref_cnt);
11051         return mlx5_hlist_unregister(priv->sh->tag_table, &tag->entry);
11052 }
11053
11054 /**
11055  * Translate action PORT_ID / REPRESENTED_PORT to vport.
11056  *
11057  * @param[in] dev
11058  *   Pointer to rte_eth_dev structure.
11059  * @param[in] action
11060  *   Pointer to action PORT_ID / REPRESENTED_PORT.
11061  * @param[out] dst_port_id
11062  *   The target port ID.
11063  * @param[out] error
11064  *   Pointer to the error structure.
11065  *
11066  * @return
11067  *   0 on success, a negative errno value otherwise and rte_errno is set.
11068  */
11069 static int
11070 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
11071                                  const struct rte_flow_action *action,
11072                                  uint32_t *dst_port_id,
11073                                  struct rte_flow_error *error)
11074 {
11075         uint32_t port;
11076         struct mlx5_priv *priv;
11077
11078         switch (action->type) {
11079         case RTE_FLOW_ACTION_TYPE_PORT_ID: {
11080                 const struct rte_flow_action_port_id *conf;
11081
11082                 conf = (const struct rte_flow_action_port_id *)action->conf;
11083                 port = conf->original ? dev->data->port_id : conf->id;
11084                 break;
11085         }
11086         case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: {
11087                 const struct rte_flow_action_ethdev *ethdev;
11088
11089                 ethdev = (const struct rte_flow_action_ethdev *)action->conf;
11090                 port = ethdev->port_id;
11091                 break;
11092         }
11093         default:
11094                 MLX5_ASSERT(false);
11095                 return rte_flow_error_set(error, EINVAL,
11096                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
11097                                           "unknown E-Switch action");
11098         }
11099
11100         priv = mlx5_port_to_eswitch_info(port, false);
11101         if (!priv)
11102                 return rte_flow_error_set(error, -rte_errno,
11103                                           RTE_FLOW_ERROR_TYPE_ACTION,
11104                                           NULL,
11105                                           "No eswitch info was found for port");
11106 #ifdef HAVE_MLX5DV_DR_CREATE_DEST_IB_PORT
11107         /*
11108          * This parameter is transferred to
11109          * mlx5dv_dr_action_create_dest_ib_port().
11110          */
11111         *dst_port_id = priv->dev_port;
11112 #else
11113         /*
11114          * Legacy mode, no LAG configurations is supported.
11115          * This parameter is transferred to
11116          * mlx5dv_dr_action_create_dest_vport().
11117          */
11118         *dst_port_id = priv->vport_id;
11119 #endif
11120         return 0;
11121 }
11122
11123 /**
11124  * Create a counter with aging configuration.
11125  *
11126  * @param[in] dev
11127  *   Pointer to rte_eth_dev structure.
11128  * @param[in] dev_flow
11129  *   Pointer to the mlx5_flow.
11130  * @param[out] count
11131  *   Pointer to the counter action configuration.
11132  * @param[in] age
11133  *   Pointer to the aging action configuration.
11134  *
11135  * @return
11136  *   Index to flow counter on success, 0 otherwise.
11137  */
11138 static uint32_t
11139 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
11140                                 struct mlx5_flow *dev_flow,
11141                                 const struct rte_flow_action_count *count
11142                                         __rte_unused,
11143                                 const struct rte_flow_action_age *age)
11144 {
11145         uint32_t counter;
11146         struct mlx5_age_param *age_param;
11147
11148         counter = flow_dv_counter_alloc(dev, !!age);
11149         if (!counter || age == NULL)
11150                 return counter;
11151         age_param = flow_dv_counter_idx_get_age(dev, counter);
11152         age_param->context = age->context ? age->context :
11153                 (void *)(uintptr_t)(dev_flow->flow_idx);
11154         age_param->timeout = age->timeout;
11155         age_param->port_id = dev->data->port_id;
11156         __atomic_store_n(&age_param->sec_since_last_hit, 0, __ATOMIC_RELAXED);
11157         __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED);
11158         return counter;
11159 }
11160
11161 /**
11162  * Add Tx queue matcher
11163  *
11164  * @param[in] dev
11165  *   Pointer to the dev struct.
11166  * @param[in, out] matcher
11167  *   Flow matcher.
11168  * @param[in, out] key
11169  *   Flow matcher value.
11170  * @param[in] item
11171  *   Flow pattern to translate.
11172  * @param[in] inner
11173  *   Item is inner pattern.
11174  */
11175 static void
11176 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
11177                                 void *matcher, void *key,
11178                                 const struct rte_flow_item *item)
11179 {
11180         const struct mlx5_rte_flow_item_tx_queue *queue_m;
11181         const struct mlx5_rte_flow_item_tx_queue *queue_v;
11182         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
11183         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
11184         struct mlx5_txq_ctrl *txq;
11185         uint32_t queue, mask;
11186
11187         queue_m = (const void *)item->mask;
11188         queue_v = (const void *)item->spec;
11189         if (!queue_v)
11190                 return;
11191         txq = mlx5_txq_get(dev, queue_v->queue);
11192         if (!txq)
11193                 return;
11194         if (txq->is_hairpin)
11195                 queue = txq->obj->sq->id;
11196         else
11197                 queue = txq->obj->sq_obj.sq->id;
11198         mask = queue_m == NULL ? UINT32_MAX : queue_m->queue;
11199         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, mask);
11200         MLX5_SET(fte_match_set_misc, misc_v, source_sqn, queue & mask);
11201         mlx5_txq_release(dev, queue_v->queue);
11202 }
11203
11204 /**
11205  * Set the hash fields according to the @p flow information.
11206  *
11207  * @param[in] item_flags
11208  *   The match pattern item flags.
11209  * @param[in] rss_desc
11210  *   Pointer to the mlx5_flow_rss_desc.
11211  * @param[out] hash_fields
11212  *   Pointer to the RSS hash fields.
11213  */
11214 void
11215 flow_dv_hashfields_set(uint64_t item_flags,
11216                        struct mlx5_flow_rss_desc *rss_desc,
11217                        uint64_t *hash_fields)
11218 {
11219         uint64_t items = item_flags;
11220         uint64_t fields = 0;
11221         int rss_inner = 0;
11222         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
11223
11224         *hash_fields = 0;
11225 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
11226         if (rss_desc->level >= 2)
11227                 rss_inner = 1;
11228 #endif
11229         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
11230             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4)) ||
11231              !items) {
11232                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
11233                         if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY)
11234                                 fields |= IBV_RX_HASH_SRC_IPV4;
11235                         else if (rss_types & RTE_ETH_RSS_L3_DST_ONLY)
11236                                 fields |= IBV_RX_HASH_DST_IPV4;
11237                         else
11238                                 fields |= MLX5_IPV4_IBV_RX_HASH;
11239                 }
11240         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
11241                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6)) ||
11242                    !items) {
11243                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
11244                         if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY)
11245                                 fields |= IBV_RX_HASH_SRC_IPV6;
11246                         else if (rss_types & RTE_ETH_RSS_L3_DST_ONLY)
11247                                 fields |= IBV_RX_HASH_DST_IPV6;
11248                         else
11249                                 fields |= MLX5_IPV6_IBV_RX_HASH;
11250                 }
11251         }
11252         if (items & MLX5_FLOW_ITEM_ESP) {
11253                 if (rss_types & RTE_ETH_RSS_ESP)
11254                         fields |= IBV_RX_HASH_IPSEC_SPI;
11255         }
11256         if ((fields & ~IBV_RX_HASH_IPSEC_SPI) == 0) {
11257                 *hash_fields = fields;
11258                 /*
11259                  * There is no match between the RSS types and the
11260                  * L3 protocol (IPv4/IPv6) defined in the flow rule.
11261                  */
11262                 return;
11263         }
11264         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
11265             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP)) ||
11266             !items) {
11267                 if (rss_types & RTE_ETH_RSS_UDP) {
11268                         if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY)
11269                                 fields |= IBV_RX_HASH_SRC_PORT_UDP;
11270                         else if (rss_types & RTE_ETH_RSS_L4_DST_ONLY)
11271                                 fields |= IBV_RX_HASH_DST_PORT_UDP;
11272                         else
11273                                 fields |= MLX5_UDP_IBV_RX_HASH;
11274                 }
11275         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
11276                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP)) ||
11277                    !items) {
11278                 if (rss_types & RTE_ETH_RSS_TCP) {
11279                         if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY)
11280                                 fields |= IBV_RX_HASH_SRC_PORT_TCP;
11281                         else if (rss_types & RTE_ETH_RSS_L4_DST_ONLY)
11282                                 fields |= IBV_RX_HASH_DST_PORT_TCP;
11283                         else
11284                                 fields |= MLX5_TCP_IBV_RX_HASH;
11285                 }
11286         }
11287         if (rss_inner)
11288                 fields |= IBV_RX_HASH_INNER;
11289         *hash_fields = fields;
11290 }
11291
11292 /**
11293  * Prepare an Rx Hash queue.
11294  *
11295  * @param dev
11296  *   Pointer to Ethernet device.
11297  * @param[in] dev_flow
11298  *   Pointer to the mlx5_flow.
11299  * @param[in] rss_desc
11300  *   Pointer to the mlx5_flow_rss_desc.
11301  * @param[out] hrxq_idx
11302  *   Hash Rx queue index.
11303  *
11304  * @return
11305  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
11306  */
11307 static struct mlx5_hrxq *
11308 flow_dv_hrxq_prepare(struct rte_eth_dev *dev,
11309                      struct mlx5_flow *dev_flow,
11310                      struct mlx5_flow_rss_desc *rss_desc,
11311                      uint32_t *hrxq_idx)
11312 {
11313         struct mlx5_flow_handle *dh = dev_flow->handle;
11314         uint32_t shared_rss = rss_desc->shared_rss;
11315         struct mlx5_hrxq *hrxq;
11316
11317         MLX5_ASSERT(rss_desc->queue_num);
11318         rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN;
11319         rss_desc->hash_fields = dev_flow->hash_fields;
11320         rss_desc->tunnel = !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL);
11321         rss_desc->shared_rss = 0;
11322         if (rss_desc->hash_fields == 0)
11323                 rss_desc->queue_num = 1;
11324         hrxq = mlx5_hrxq_get(dev, rss_desc);
11325         *hrxq_idx = hrxq ? hrxq->idx : 0;
11326         rss_desc->shared_rss = shared_rss;
11327         return hrxq;
11328 }
11329
11330 /**
11331  * Release sample sub action resource.
11332  *
11333  * @param[in, out] dev
11334  *   Pointer to rte_eth_dev structure.
11335  * @param[in] act_res
11336  *   Pointer to sample sub action resource.
11337  */
11338 static void
11339 flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
11340                                    struct mlx5_flow_sub_actions_idx *act_res)
11341 {
11342         if (act_res->rix_hrxq) {
11343                 mlx5_hrxq_release(dev, act_res->rix_hrxq);
11344                 act_res->rix_hrxq = 0;
11345         }
11346         if (act_res->rix_encap_decap) {
11347                 flow_dv_encap_decap_resource_release(dev,
11348                                                      act_res->rix_encap_decap);
11349                 act_res->rix_encap_decap = 0;
11350         }
11351         if (act_res->rix_port_id_action) {
11352                 flow_dv_port_id_action_resource_release(dev,
11353                                                 act_res->rix_port_id_action);
11354                 act_res->rix_port_id_action = 0;
11355         }
11356         if (act_res->rix_tag) {
11357                 flow_dv_tag_release(dev, act_res->rix_tag);
11358                 act_res->rix_tag = 0;
11359         }
11360         if (act_res->rix_jump) {
11361                 flow_dv_jump_tbl_resource_release(dev, act_res->rix_jump);
11362                 act_res->rix_jump = 0;
11363         }
11364 }
11365
11366 int
11367 flow_dv_sample_match_cb(void *tool_ctx __rte_unused,
11368                         struct mlx5_list_entry *entry, void *cb_ctx)
11369 {
11370         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11371         struct rte_eth_dev *dev = ctx->dev;
11372         struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
11373         struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
11374                                                               typeof(*resource),
11375                                                               entry);
11376
11377         if (ctx_resource->ratio == resource->ratio &&
11378             ctx_resource->ft_type == resource->ft_type &&
11379             ctx_resource->ft_id == resource->ft_id &&
11380             ctx_resource->set_action == resource->set_action &&
11381             !memcmp((void *)&ctx_resource->sample_act,
11382                     (void *)&resource->sample_act,
11383                     sizeof(struct mlx5_flow_sub_actions_list))) {
11384                 /*
11385                  * Existing sample action should release the prepared
11386                  * sub-actions reference counter.
11387                  */
11388                 flow_dv_sample_sub_actions_release(dev,
11389                                                    &ctx_resource->sample_idx);
11390                 return 0;
11391         }
11392         return 1;
11393 }
11394
11395 struct mlx5_list_entry *
11396 flow_dv_sample_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
11397 {
11398         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11399         struct rte_eth_dev *dev = ctx->dev;
11400         struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
11401         void **sample_dv_actions = ctx_resource->sub_actions;
11402         struct mlx5_flow_dv_sample_resource *resource;
11403         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
11404         struct mlx5_priv *priv = dev->data->dev_private;
11405         struct mlx5_dev_ctx_shared *sh = priv->sh;
11406         struct mlx5_flow_tbl_resource *tbl;
11407         uint32_t idx = 0;
11408         const uint32_t next_ft_step = 1;
11409         uint32_t next_ft_id = ctx_resource->ft_id + next_ft_step;
11410         uint8_t is_egress = 0;
11411         uint8_t is_transfer = 0;
11412         struct rte_flow_error *error = ctx->error;
11413
11414         /* Register new sample resource. */
11415         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
11416         if (!resource) {
11417                 rte_flow_error_set(error, ENOMEM,
11418                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11419                                           NULL,
11420                                           "cannot allocate resource memory");
11421                 return NULL;
11422         }
11423         *resource = *ctx_resource;
11424         /* Create normal path table level */
11425         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
11426                 is_transfer = 1;
11427         else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
11428                 is_egress = 1;
11429         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
11430                                         is_egress, is_transfer,
11431                                         true, NULL, 0, 0, 0, error);
11432         if (!tbl) {
11433                 rte_flow_error_set(error, ENOMEM,
11434                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11435                                           NULL,
11436                                           "fail to create normal path table "
11437                                           "for sample");
11438                 goto error;
11439         }
11440         resource->normal_path_tbl = tbl;
11441         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
11442                 if (!sh->default_miss_action) {
11443                         rte_flow_error_set(error, ENOMEM,
11444                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11445                                                 NULL,
11446                                                 "default miss action was not "
11447                                                 "created");
11448                         goto error;
11449                 }
11450                 sample_dv_actions[ctx_resource->sample_act.actions_num++] =
11451                                                 sh->default_miss_action;
11452         }
11453         /* Create a DR sample action */
11454         sampler_attr.sample_ratio = resource->ratio;
11455         sampler_attr.default_next_table = tbl->obj;
11456         sampler_attr.num_sample_actions = ctx_resource->sample_act.actions_num;
11457         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
11458                                                         &sample_dv_actions[0];
11459         sampler_attr.action = resource->set_action;
11460         if (mlx5_os_flow_dr_create_flow_action_sampler
11461                         (&sampler_attr, &resource->verbs_action)) {
11462                 rte_flow_error_set(error, ENOMEM,
11463                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11464                                         NULL, "cannot create sample action");
11465                 goto error;
11466         }
11467         resource->idx = idx;
11468         resource->dev = dev;
11469         return &resource->entry;
11470 error:
11471         if (resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
11472                 flow_dv_sample_sub_actions_release(dev,
11473                                                    &resource->sample_idx);
11474         if (resource->normal_path_tbl)
11475                 flow_dv_tbl_resource_release(MLX5_SH(dev),
11476                                 resource->normal_path_tbl);
11477         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx);
11478         return NULL;
11479
11480 }
11481
11482 struct mlx5_list_entry *
11483 flow_dv_sample_clone_cb(void *tool_ctx __rte_unused,
11484                          struct mlx5_list_entry *entry __rte_unused,
11485                          void *cb_ctx)
11486 {
11487         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11488         struct rte_eth_dev *dev = ctx->dev;
11489         struct mlx5_flow_dv_sample_resource *resource;
11490         struct mlx5_priv *priv = dev->data->dev_private;
11491         struct mlx5_dev_ctx_shared *sh = priv->sh;
11492         uint32_t idx = 0;
11493
11494         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
11495         if (!resource) {
11496                 rte_flow_error_set(ctx->error, ENOMEM,
11497                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11498                                           NULL,
11499                                           "cannot allocate resource memory");
11500                 return NULL;
11501         }
11502         memcpy(resource, entry, sizeof(*resource));
11503         resource->idx = idx;
11504         resource->dev = dev;
11505         return &resource->entry;
11506 }
11507
11508 void
11509 flow_dv_sample_clone_free_cb(void *tool_ctx __rte_unused,
11510                              struct mlx5_list_entry *entry)
11511 {
11512         struct mlx5_flow_dv_sample_resource *resource =
11513                                   container_of(entry, typeof(*resource), entry);
11514         struct rte_eth_dev *dev = resource->dev;
11515         struct mlx5_priv *priv = dev->data->dev_private;
11516
11517         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
11518 }
11519
11520 /**
11521  * Find existing sample resource or create and register a new one.
11522  *
11523  * @param[in, out] dev
11524  *   Pointer to rte_eth_dev structure.
11525  * @param[in] ref
11526  *   Pointer to sample resource reference.
11527  * @parm[in, out] dev_flow
11528  *   Pointer to the dev_flow.
11529  * @param[out] error
11530  *   pointer to error structure.
11531  *
11532  * @return
11533  *   0 on success otherwise -errno and errno is set.
11534  */
11535 static int
11536 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
11537                          struct mlx5_flow_dv_sample_resource *ref,
11538                          struct mlx5_flow *dev_flow,
11539                          struct rte_flow_error *error)
11540 {
11541         struct mlx5_flow_dv_sample_resource *resource;
11542         struct mlx5_list_entry *entry;
11543         struct mlx5_priv *priv = dev->data->dev_private;
11544         struct mlx5_flow_cb_ctx ctx = {
11545                 .dev = dev,
11546                 .error = error,
11547                 .data = ref,
11548         };
11549
11550         entry = mlx5_list_register(priv->sh->sample_action_list, &ctx);
11551         if (!entry)
11552                 return -rte_errno;
11553         resource = container_of(entry, typeof(*resource), entry);
11554         dev_flow->handle->dvh.rix_sample = resource->idx;
11555         dev_flow->dv.sample_res = resource;
11556         return 0;
11557 }
11558
11559 int
11560 flow_dv_dest_array_match_cb(void *tool_ctx __rte_unused,
11561                             struct mlx5_list_entry *entry, void *cb_ctx)
11562 {
11563         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11564         struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
11565         struct rte_eth_dev *dev = ctx->dev;
11566         struct mlx5_flow_dv_dest_array_resource *resource =
11567                                   container_of(entry, typeof(*resource), entry);
11568         uint32_t idx = 0;
11569
11570         if (ctx_resource->num_of_dest == resource->num_of_dest &&
11571             ctx_resource->ft_type == resource->ft_type &&
11572             !memcmp((void *)resource->sample_act,
11573                     (void *)ctx_resource->sample_act,
11574                    (ctx_resource->num_of_dest *
11575                    sizeof(struct mlx5_flow_sub_actions_list)))) {
11576                 /*
11577                  * Existing sample action should release the prepared
11578                  * sub-actions reference counter.
11579                  */
11580                 for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
11581                         flow_dv_sample_sub_actions_release(dev,
11582                                         &ctx_resource->sample_idx[idx]);
11583                 return 0;
11584         }
11585         return 1;
11586 }
11587
11588 struct mlx5_list_entry *
11589 flow_dv_dest_array_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
11590 {
11591         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11592         struct rte_eth_dev *dev = ctx->dev;
11593         struct mlx5_flow_dv_dest_array_resource *resource;
11594         struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
11595         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
11596         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
11597         struct mlx5_priv *priv = dev->data->dev_private;
11598         struct mlx5_dev_ctx_shared *sh = priv->sh;
11599         struct mlx5_flow_sub_actions_list *sample_act;
11600         struct mlx5dv_dr_domain *domain;
11601         uint32_t idx = 0, res_idx = 0;
11602         struct rte_flow_error *error = ctx->error;
11603         uint64_t action_flags;
11604         int ret;
11605
11606         /* Register new destination array resource. */
11607         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
11608                                             &res_idx);
11609         if (!resource) {
11610                 rte_flow_error_set(error, ENOMEM,
11611                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11612                                           NULL,
11613                                           "cannot allocate resource memory");
11614                 return NULL;
11615         }
11616         *resource = *ctx_resource;
11617         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
11618                 domain = sh->fdb_domain;
11619         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
11620                 domain = sh->rx_domain;
11621         else
11622                 domain = sh->tx_domain;
11623         for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
11624                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
11625                                  mlx5_malloc(MLX5_MEM_ZERO,
11626                                  sizeof(struct mlx5dv_dr_action_dest_attr),
11627                                  0, SOCKET_ID_ANY);
11628                 if (!dest_attr[idx]) {
11629                         rte_flow_error_set(error, ENOMEM,
11630                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11631                                            NULL,
11632                                            "cannot allocate resource memory");
11633                         goto error;
11634                 }
11635                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
11636                 sample_act = &ctx_resource->sample_act[idx];
11637                 action_flags = sample_act->action_flags;
11638                 switch (action_flags) {
11639                 case MLX5_FLOW_ACTION_QUEUE:
11640                         dest_attr[idx]->dest = sample_act->dr_queue_action;
11641                         break;
11642                 case (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP):
11643                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
11644                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
11645                         dest_attr[idx]->dest_reformat->reformat =
11646                                         sample_act->dr_encap_action;
11647                         dest_attr[idx]->dest_reformat->dest =
11648                                         sample_act->dr_port_id_action;
11649                         break;
11650                 case MLX5_FLOW_ACTION_PORT_ID:
11651                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
11652                         break;
11653                 case MLX5_FLOW_ACTION_JUMP:
11654                         dest_attr[idx]->dest = sample_act->dr_jump_action;
11655                         break;
11656                 default:
11657                         rte_flow_error_set(error, EINVAL,
11658                                            RTE_FLOW_ERROR_TYPE_ACTION,
11659                                            NULL,
11660                                            "unsupported actions type");
11661                         goto error;
11662                 }
11663         }
11664         /* create a dest array action */
11665         ret = mlx5_os_flow_dr_create_flow_action_dest_array
11666                                                 (domain,
11667                                                  resource->num_of_dest,
11668                                                  dest_attr,
11669                                                  &resource->action);
11670         if (ret) {
11671                 rte_flow_error_set(error, ENOMEM,
11672                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11673                                    NULL,
11674                                    "cannot create destination array action");
11675                 goto error;
11676         }
11677         resource->idx = res_idx;
11678         resource->dev = dev;
11679         for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
11680                 mlx5_free(dest_attr[idx]);
11681         return &resource->entry;
11682 error:
11683         for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
11684                 flow_dv_sample_sub_actions_release(dev,
11685                                                    &resource->sample_idx[idx]);
11686                 if (dest_attr[idx])
11687                         mlx5_free(dest_attr[idx]);
11688         }
11689         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx);
11690         return NULL;
11691 }
11692
11693 struct mlx5_list_entry *
11694 flow_dv_dest_array_clone_cb(void *tool_ctx __rte_unused,
11695                             struct mlx5_list_entry *entry __rte_unused,
11696                             void *cb_ctx)
11697 {
11698         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11699         struct rte_eth_dev *dev = ctx->dev;
11700         struct mlx5_flow_dv_dest_array_resource *resource;
11701         struct mlx5_priv *priv = dev->data->dev_private;
11702         struct mlx5_dev_ctx_shared *sh = priv->sh;
11703         uint32_t res_idx = 0;
11704         struct rte_flow_error *error = ctx->error;
11705
11706         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
11707                                       &res_idx);
11708         if (!resource) {
11709                 rte_flow_error_set(error, ENOMEM,
11710                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11711                                           NULL,
11712                                           "cannot allocate dest-array memory");
11713                 return NULL;
11714         }
11715         memcpy(resource, entry, sizeof(*resource));
11716         resource->idx = res_idx;
11717         resource->dev = dev;
11718         return &resource->entry;
11719 }
11720
11721 void
11722 flow_dv_dest_array_clone_free_cb(void *tool_ctx __rte_unused,
11723                                  struct mlx5_list_entry *entry)
11724 {
11725         struct mlx5_flow_dv_dest_array_resource *resource =
11726                         container_of(entry, typeof(*resource), entry);
11727         struct rte_eth_dev *dev = resource->dev;
11728         struct mlx5_priv *priv = dev->data->dev_private;
11729
11730         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
11731 }
11732
11733 /**
11734  * Find existing destination array resource or create and register a new one.
11735  *
11736  * @param[in, out] dev
11737  *   Pointer to rte_eth_dev structure.
11738  * @param[in] ref
11739  *   Pointer to destination array resource reference.
11740  * @parm[in, out] dev_flow
11741  *   Pointer to the dev_flow.
11742  * @param[out] error
11743  *   pointer to error structure.
11744  *
11745  * @return
11746  *   0 on success otherwise -errno and errno is set.
11747  */
11748 static int
11749 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
11750                          struct mlx5_flow_dv_dest_array_resource *ref,
11751                          struct mlx5_flow *dev_flow,
11752                          struct rte_flow_error *error)
11753 {
11754         struct mlx5_flow_dv_dest_array_resource *resource;
11755         struct mlx5_priv *priv = dev->data->dev_private;
11756         struct mlx5_list_entry *entry;
11757         struct mlx5_flow_cb_ctx ctx = {
11758                 .dev = dev,
11759                 .error = error,
11760                 .data = ref,
11761         };
11762
11763         entry = mlx5_list_register(priv->sh->dest_array_list, &ctx);
11764         if (!entry)
11765                 return -rte_errno;
11766         resource = container_of(entry, typeof(*resource), entry);
11767         dev_flow->handle->dvh.rix_dest_array = resource->idx;
11768         dev_flow->dv.dest_array_res = resource;
11769         return 0;
11770 }
11771
11772 /**
11773  * Convert Sample action to DV specification.
11774  *
11775  * @param[in] dev
11776  *   Pointer to rte_eth_dev structure.
11777  * @param[in] action
11778  *   Pointer to sample action structure.
11779  * @param[in, out] dev_flow
11780  *   Pointer to the mlx5_flow.
11781  * @param[in] attr
11782  *   Pointer to the flow attributes.
11783  * @param[in, out] num_of_dest
11784  *   Pointer to the num of destination.
11785  * @param[in, out] sample_actions
11786  *   Pointer to sample actions list.
11787  * @param[in, out] res
11788  *   Pointer to sample resource.
11789  * @param[out] error
11790  *   Pointer to the error structure.
11791  *
11792  * @return
11793  *   0 on success, a negative errno value otherwise and rte_errno is set.
11794  */
11795 static int
11796 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
11797                                 const struct rte_flow_action_sample *action,
11798                                 struct mlx5_flow *dev_flow,
11799                                 const struct rte_flow_attr *attr,
11800                                 uint32_t *num_of_dest,
11801                                 void **sample_actions,
11802                                 struct mlx5_flow_dv_sample_resource *res,
11803                                 struct rte_flow_error *error)
11804 {
11805         struct mlx5_priv *priv = dev->data->dev_private;
11806         const struct rte_flow_action *sub_actions;
11807         struct mlx5_flow_sub_actions_list *sample_act;
11808         struct mlx5_flow_sub_actions_idx *sample_idx;
11809         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
11810         struct rte_flow *flow = dev_flow->flow;
11811         struct mlx5_flow_rss_desc *rss_desc;
11812         uint64_t action_flags = 0;
11813
11814         MLX5_ASSERT(wks);
11815         rss_desc = &wks->rss_desc;
11816         sample_act = &res->sample_act;
11817         sample_idx = &res->sample_idx;
11818         res->ratio = action->ratio;
11819         sub_actions = action->actions;
11820         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
11821                 int type = sub_actions->type;
11822                 uint32_t pre_rix = 0;
11823                 void *pre_r;
11824                 switch (type) {
11825                 case RTE_FLOW_ACTION_TYPE_QUEUE:
11826                 {
11827                         const struct rte_flow_action_queue *queue;
11828                         struct mlx5_hrxq *hrxq;
11829                         uint32_t hrxq_idx;
11830
11831                         queue = sub_actions->conf;
11832                         rss_desc->queue_num = 1;
11833                         rss_desc->queue[0] = queue->index;
11834                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11835                                                     rss_desc, &hrxq_idx);
11836                         if (!hrxq)
11837                                 return rte_flow_error_set
11838                                         (error, rte_errno,
11839                                          RTE_FLOW_ERROR_TYPE_ACTION,
11840                                          NULL,
11841                                          "cannot create fate queue");
11842                         sample_act->dr_queue_action = hrxq->action;
11843                         sample_idx->rix_hrxq = hrxq_idx;
11844                         sample_actions[sample_act->actions_num++] =
11845                                                 hrxq->action;
11846                         (*num_of_dest)++;
11847                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
11848                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11849                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11850                         dev_flow->handle->fate_action =
11851                                         MLX5_FLOW_FATE_QUEUE;
11852                         break;
11853                 }
11854                 case RTE_FLOW_ACTION_TYPE_RSS:
11855                 {
11856                         struct mlx5_hrxq *hrxq;
11857                         uint32_t hrxq_idx;
11858                         const struct rte_flow_action_rss *rss;
11859                         const uint8_t *rss_key;
11860
11861                         rss = sub_actions->conf;
11862                         memcpy(rss_desc->queue, rss->queue,
11863                                rss->queue_num * sizeof(uint16_t));
11864                         rss_desc->queue_num = rss->queue_num;
11865                         /* NULL RSS key indicates default RSS key. */
11866                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
11867                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
11868                         /*
11869                          * rss->level and rss.types should be set in advance
11870                          * when expanding items for RSS.
11871                          */
11872                         flow_dv_hashfields_set(dev_flow->handle->layers,
11873                                                rss_desc,
11874                                                &dev_flow->hash_fields);
11875                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11876                                                     rss_desc, &hrxq_idx);
11877                         if (!hrxq)
11878                                 return rte_flow_error_set
11879                                         (error, rte_errno,
11880                                          RTE_FLOW_ERROR_TYPE_ACTION,
11881                                          NULL,
11882                                          "cannot create fate queue");
11883                         sample_act->dr_queue_action = hrxq->action;
11884                         sample_idx->rix_hrxq = hrxq_idx;
11885                         sample_actions[sample_act->actions_num++] =
11886                                                 hrxq->action;
11887                         (*num_of_dest)++;
11888                         action_flags |= MLX5_FLOW_ACTION_RSS;
11889                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11890                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11891                         dev_flow->handle->fate_action =
11892                                         MLX5_FLOW_FATE_QUEUE;
11893                         break;
11894                 }
11895                 case RTE_FLOW_ACTION_TYPE_MARK:
11896                 {
11897                         uint32_t tag_be = mlx5_flow_mark_set
11898                                 (((const struct rte_flow_action_mark *)
11899                                 (sub_actions->conf))->id);
11900
11901                         wks->mark = 1;
11902                         pre_rix = dev_flow->handle->dvh.rix_tag;
11903                         /* Save the mark resource before sample */
11904                         pre_r = dev_flow->dv.tag_resource;
11905                         if (flow_dv_tag_resource_register(dev, tag_be,
11906                                                   dev_flow, error))
11907                                 return -rte_errno;
11908                         MLX5_ASSERT(dev_flow->dv.tag_resource);
11909                         sample_act->dr_tag_action =
11910                                 dev_flow->dv.tag_resource->action;
11911                         sample_idx->rix_tag =
11912                                 dev_flow->handle->dvh.rix_tag;
11913                         sample_actions[sample_act->actions_num++] =
11914                                                 sample_act->dr_tag_action;
11915                         /* Recover the mark resource after sample */
11916                         dev_flow->dv.tag_resource = pre_r;
11917                         dev_flow->handle->dvh.rix_tag = pre_rix;
11918                         action_flags |= MLX5_FLOW_ACTION_MARK;
11919                         break;
11920                 }
11921                 case RTE_FLOW_ACTION_TYPE_COUNT:
11922                 {
11923                         if (!flow->counter) {
11924                                 flow->counter =
11925                                         flow_dv_translate_create_counter(dev,
11926                                                 dev_flow, sub_actions->conf,
11927                                                 0);
11928                                 if (!flow->counter)
11929                                         return rte_flow_error_set
11930                                                 (error, rte_errno,
11931                                                 RTE_FLOW_ERROR_TYPE_ACTION,
11932                                                 NULL,
11933                                                 "cannot create counter"
11934                                                 " object.");
11935                         }
11936                         sample_act->dr_cnt_action =
11937                                   (flow_dv_counter_get_by_idx(dev,
11938                                   flow->counter, NULL))->action;
11939                         sample_actions[sample_act->actions_num++] =
11940                                                 sample_act->dr_cnt_action;
11941                         action_flags |= MLX5_FLOW_ACTION_COUNT;
11942                         break;
11943                 }
11944                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
11945                 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
11946                 {
11947                         struct mlx5_flow_dv_port_id_action_resource
11948                                         port_id_resource;
11949                         uint32_t port_id = 0;
11950
11951                         memset(&port_id_resource, 0, sizeof(port_id_resource));
11952                         /* Save the port id resource before sample */
11953                         pre_rix = dev_flow->handle->rix_port_id_action;
11954                         pre_r = dev_flow->dv.port_id_action;
11955                         if (flow_dv_translate_action_port_id(dev, sub_actions,
11956                                                              &port_id, error))
11957                                 return -rte_errno;
11958                         port_id_resource.port_id = port_id;
11959                         if (flow_dv_port_id_action_resource_register
11960                             (dev, &port_id_resource, dev_flow, error))
11961                                 return -rte_errno;
11962                         sample_act->dr_port_id_action =
11963                                 dev_flow->dv.port_id_action->action;
11964                         sample_idx->rix_port_id_action =
11965                                 dev_flow->handle->rix_port_id_action;
11966                         sample_actions[sample_act->actions_num++] =
11967                                                 sample_act->dr_port_id_action;
11968                         /* Recover the port id resource after sample */
11969                         dev_flow->dv.port_id_action = pre_r;
11970                         dev_flow->handle->rix_port_id_action = pre_rix;
11971                         (*num_of_dest)++;
11972                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
11973                         break;
11974                 }
11975                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
11976                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
11977                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
11978                         /* Save the encap resource before sample */
11979                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
11980                         pre_r = dev_flow->dv.encap_decap;
11981                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
11982                                                            dev_flow,
11983                                                            attr->transfer,
11984                                                            error))
11985                                 return -rte_errno;
11986                         sample_act->dr_encap_action =
11987                                 dev_flow->dv.encap_decap->action;
11988                         sample_idx->rix_encap_decap =
11989                                 dev_flow->handle->dvh.rix_encap_decap;
11990                         sample_actions[sample_act->actions_num++] =
11991                                                 sample_act->dr_encap_action;
11992                         /* Recover the encap resource after sample */
11993                         dev_flow->dv.encap_decap = pre_r;
11994                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
11995                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
11996                         break;
11997                 default:
11998                         return rte_flow_error_set(error, EINVAL,
11999                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12000                                 NULL,
12001                                 "Not support for sampler action");
12002                 }
12003         }
12004         sample_act->action_flags = action_flags;
12005         res->ft_id = dev_flow->dv.group;
12006         if (attr->transfer) {
12007                 union {
12008                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
12009                         uint64_t set_action;
12010                 } action_ctx = { .set_action = 0 };
12011
12012                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
12013                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
12014                          MLX5_MODIFICATION_TYPE_SET);
12015                 MLX5_SET(set_action_in, action_ctx.action_in, field,
12016                          MLX5_MODI_META_REG_C_0);
12017                 MLX5_SET(set_action_in, action_ctx.action_in, data,
12018                          priv->vport_meta_tag);
12019                 res->set_action = action_ctx.set_action;
12020         } else if (attr->ingress) {
12021                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12022         } else {
12023                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX;
12024         }
12025         return 0;
12026 }
12027
12028 /**
12029  * Convert Sample action to DV specification.
12030  *
12031  * @param[in] dev
12032  *   Pointer to rte_eth_dev structure.
12033  * @param[in, out] dev_flow
12034  *   Pointer to the mlx5_flow.
12035  * @param[in] num_of_dest
12036  *   The num of destination.
12037  * @param[in, out] res
12038  *   Pointer to sample resource.
12039  * @param[in, out] mdest_res
12040  *   Pointer to destination array resource.
12041  * @param[in] sample_actions
12042  *   Pointer to sample path actions list.
12043  * @param[in] action_flags
12044  *   Holds the actions detected until now.
12045  * @param[out] error
12046  *   Pointer to the error structure.
12047  *
12048  * @return
12049  *   0 on success, a negative errno value otherwise and rte_errno is set.
12050  */
12051 static int
12052 flow_dv_create_action_sample(struct rte_eth_dev *dev,
12053                              struct mlx5_flow *dev_flow,
12054                              uint32_t num_of_dest,
12055                              struct mlx5_flow_dv_sample_resource *res,
12056                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
12057                              void **sample_actions,
12058                              uint64_t action_flags,
12059                              struct rte_flow_error *error)
12060 {
12061         /* update normal path action resource into last index of array */
12062         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
12063         struct mlx5_flow_sub_actions_list *sample_act =
12064                                         &mdest_res->sample_act[dest_index];
12065         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
12066         struct mlx5_flow_rss_desc *rss_desc;
12067         uint32_t normal_idx = 0;
12068         struct mlx5_hrxq *hrxq;
12069         uint32_t hrxq_idx;
12070
12071         MLX5_ASSERT(wks);
12072         rss_desc = &wks->rss_desc;
12073         if (num_of_dest > 1) {
12074                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
12075                         /* Handle QP action for mirroring */
12076                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
12077                                                     rss_desc, &hrxq_idx);
12078                         if (!hrxq)
12079                                 return rte_flow_error_set
12080                                      (error, rte_errno,
12081                                       RTE_FLOW_ERROR_TYPE_ACTION,
12082                                       NULL,
12083                                       "cannot create rx queue");
12084                         normal_idx++;
12085                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
12086                         sample_act->dr_queue_action = hrxq->action;
12087                         if (action_flags & MLX5_FLOW_ACTION_MARK)
12088                                 dev_flow->handle->rix_hrxq = hrxq_idx;
12089                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
12090                 }
12091                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
12092                         normal_idx++;
12093                         mdest_res->sample_idx[dest_index].rix_encap_decap =
12094                                 dev_flow->handle->dvh.rix_encap_decap;
12095                         sample_act->dr_encap_action =
12096                                 dev_flow->dv.encap_decap->action;
12097                         dev_flow->handle->dvh.rix_encap_decap = 0;
12098                 }
12099                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
12100                         normal_idx++;
12101                         mdest_res->sample_idx[dest_index].rix_port_id_action =
12102                                 dev_flow->handle->rix_port_id_action;
12103                         sample_act->dr_port_id_action =
12104                                 dev_flow->dv.port_id_action->action;
12105                         dev_flow->handle->rix_port_id_action = 0;
12106                 }
12107                 if (sample_act->action_flags & MLX5_FLOW_ACTION_JUMP) {
12108                         normal_idx++;
12109                         mdest_res->sample_idx[dest_index].rix_jump =
12110                                 dev_flow->handle->rix_jump;
12111                         sample_act->dr_jump_action =
12112                                 dev_flow->dv.jump->action;
12113                         dev_flow->handle->rix_jump = 0;
12114                 }
12115                 sample_act->actions_num = normal_idx;
12116                 /* update sample action resource into first index of array */
12117                 mdest_res->ft_type = res->ft_type;
12118                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
12119                                 sizeof(struct mlx5_flow_sub_actions_idx));
12120                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
12121                                 sizeof(struct mlx5_flow_sub_actions_list));
12122                 mdest_res->num_of_dest = num_of_dest;
12123                 if (flow_dv_dest_array_resource_register(dev, mdest_res,
12124                                                          dev_flow, error))
12125                         return rte_flow_error_set(error, EINVAL,
12126                                                   RTE_FLOW_ERROR_TYPE_ACTION,
12127                                                   NULL, "can't create sample "
12128                                                   "action");
12129         } else {
12130                 res->sub_actions = sample_actions;
12131                 if (flow_dv_sample_resource_register(dev, res, dev_flow, error))
12132                         return rte_flow_error_set(error, EINVAL,
12133                                                   RTE_FLOW_ERROR_TYPE_ACTION,
12134                                                   NULL,
12135                                                   "can't create sample action");
12136         }
12137         return 0;
12138 }
12139
12140 /**
12141  * Remove an ASO age action from age actions list.
12142  *
12143  * @param[in] dev
12144  *   Pointer to the Ethernet device structure.
12145  * @param[in] age
12146  *   Pointer to the aso age action handler.
12147  */
12148 static void
12149 flow_dv_aso_age_remove_from_age(struct rte_eth_dev *dev,
12150                                 struct mlx5_aso_age_action *age)
12151 {
12152         struct mlx5_age_info *age_info;
12153         struct mlx5_age_param *age_param = &age->age_params;
12154         struct mlx5_priv *priv = dev->data->dev_private;
12155         uint16_t expected = AGE_CANDIDATE;
12156
12157         age_info = GET_PORT_AGE_INFO(priv);
12158         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
12159                                          AGE_FREE, false, __ATOMIC_RELAXED,
12160                                          __ATOMIC_RELAXED)) {
12161                 /**
12162                  * We need the lock even it is age timeout,
12163                  * since age action may still in process.
12164                  */
12165                 rte_spinlock_lock(&age_info->aged_sl);
12166                 LIST_REMOVE(age, next);
12167                 rte_spinlock_unlock(&age_info->aged_sl);
12168                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
12169         }
12170 }
12171
12172 /**
12173  * Release an ASO age action.
12174  *
12175  * @param[in] dev
12176  *   Pointer to the Ethernet device structure.
12177  * @param[in] age_idx
12178  *   Index of ASO age action to release.
12179  * @param[in] flow
12180  *   True if the release operation is during flow destroy operation.
12181  *   False if the release operation is during action destroy operation.
12182  *
12183  * @return
12184  *   0 when age action was removed, otherwise the number of references.
12185  */
12186 static int
12187 flow_dv_aso_age_release(struct rte_eth_dev *dev, uint32_t age_idx)
12188 {
12189         struct mlx5_priv *priv = dev->data->dev_private;
12190         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
12191         struct mlx5_aso_age_action *age = flow_aso_age_get_by_idx(dev, age_idx);
12192         uint32_t ret = __atomic_sub_fetch(&age->refcnt, 1, __ATOMIC_RELAXED);
12193
12194         if (!ret) {
12195                 flow_dv_aso_age_remove_from_age(dev, age);
12196                 rte_spinlock_lock(&mng->free_sl);
12197                 LIST_INSERT_HEAD(&mng->free, age, next);
12198                 rte_spinlock_unlock(&mng->free_sl);
12199         }
12200         return ret;
12201 }
12202
12203 /**
12204  * Resize the ASO age pools array by MLX5_CNT_CONTAINER_RESIZE pools.
12205  *
12206  * @param[in] dev
12207  *   Pointer to the Ethernet device structure.
12208  *
12209  * @return
12210  *   0 on success, otherwise negative errno value and rte_errno is set.
12211  */
12212 static int
12213 flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev)
12214 {
12215         struct mlx5_priv *priv = dev->data->dev_private;
12216         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
12217         void *old_pools = mng->pools;
12218         uint32_t resize = mng->n + MLX5_CNT_CONTAINER_RESIZE;
12219         uint32_t mem_size = sizeof(struct mlx5_aso_age_pool *) * resize;
12220         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
12221
12222         if (!pools) {
12223                 rte_errno = ENOMEM;
12224                 return -ENOMEM;
12225         }
12226         if (old_pools) {
12227                 memcpy(pools, old_pools,
12228                        mng->n * sizeof(struct mlx5_flow_counter_pool *));
12229                 mlx5_free(old_pools);
12230         } else {
12231                 /* First ASO flow hit allocation - starting ASO data-path. */
12232                 int ret = mlx5_aso_flow_hit_queue_poll_start(priv->sh);
12233
12234                 if (ret) {
12235                         mlx5_free(pools);
12236                         return ret;
12237                 }
12238         }
12239         mng->n = resize;
12240         mng->pools = pools;
12241         return 0;
12242 }
12243
12244 /**
12245  * Create and initialize a new ASO aging pool.
12246  *
12247  * @param[in] dev
12248  *   Pointer to the Ethernet device structure.
12249  * @param[out] age_free
12250  *   Where to put the pointer of a new age action.
12251  *
12252  * @return
12253  *   The age actions pool pointer and @p age_free is set on success,
12254  *   NULL otherwise and rte_errno is set.
12255  */
12256 static struct mlx5_aso_age_pool *
12257 flow_dv_age_pool_create(struct rte_eth_dev *dev,
12258                         struct mlx5_aso_age_action **age_free)
12259 {
12260         struct mlx5_priv *priv = dev->data->dev_private;
12261         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
12262         struct mlx5_aso_age_pool *pool = NULL;
12263         struct mlx5_devx_obj *obj = NULL;
12264         uint32_t i;
12265
12266         obj = mlx5_devx_cmd_create_flow_hit_aso_obj(priv->sh->cdev->ctx,
12267                                                     priv->sh->cdev->pdn);
12268         if (!obj) {
12269                 rte_errno = ENODATA;
12270                 DRV_LOG(ERR, "Failed to create flow_hit_aso_obj using DevX.");
12271                 return NULL;
12272         }
12273         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
12274         if (!pool) {
12275                 claim_zero(mlx5_devx_cmd_destroy(obj));
12276                 rte_errno = ENOMEM;
12277                 return NULL;
12278         }
12279         pool->flow_hit_aso_obj = obj;
12280         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
12281         rte_rwlock_write_lock(&mng->resize_rwl);
12282         pool->index = mng->next;
12283         /* Resize pools array if there is no room for the new pool in it. */
12284         if (pool->index == mng->n && flow_dv_aso_age_pools_resize(dev)) {
12285                 claim_zero(mlx5_devx_cmd_destroy(obj));
12286                 mlx5_free(pool);
12287                 rte_rwlock_write_unlock(&mng->resize_rwl);
12288                 return NULL;
12289         }
12290         mng->pools[pool->index] = pool;
12291         mng->next++;
12292         rte_rwlock_write_unlock(&mng->resize_rwl);
12293         /* Assign the first action in the new pool, the rest go to free list. */
12294         *age_free = &pool->actions[0];
12295         for (i = 1; i < MLX5_ASO_AGE_ACTIONS_PER_POOL; i++) {
12296                 pool->actions[i].offset = i;
12297                 LIST_INSERT_HEAD(&mng->free, &pool->actions[i], next);
12298         }
12299         return pool;
12300 }
12301
12302 /**
12303  * Allocate a ASO aging bit.
12304  *
12305  * @param[in] dev
12306  *   Pointer to the Ethernet device structure.
12307  * @param[out] error
12308  *   Pointer to the error structure.
12309  *
12310  * @return
12311  *   Index to ASO age action on success, 0 otherwise and rte_errno is set.
12312  */
12313 static uint32_t
12314 flow_dv_aso_age_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
12315 {
12316         struct mlx5_priv *priv = dev->data->dev_private;
12317         const struct mlx5_aso_age_pool *pool;
12318         struct mlx5_aso_age_action *age_free = NULL;
12319         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
12320
12321         MLX5_ASSERT(mng);
12322         /* Try to get the next free age action bit. */
12323         rte_spinlock_lock(&mng->free_sl);
12324         age_free = LIST_FIRST(&mng->free);
12325         if (age_free) {
12326                 LIST_REMOVE(age_free, next);
12327         } else if (!flow_dv_age_pool_create(dev, &age_free)) {
12328                 rte_spinlock_unlock(&mng->free_sl);
12329                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
12330                                    NULL, "failed to create ASO age pool");
12331                 return 0; /* 0 is an error. */
12332         }
12333         rte_spinlock_unlock(&mng->free_sl);
12334         pool = container_of
12335           ((const struct mlx5_aso_age_action (*)[MLX5_ASO_AGE_ACTIONS_PER_POOL])
12336                   (age_free - age_free->offset), const struct mlx5_aso_age_pool,
12337                                                                        actions);
12338         if (!age_free->dr_action) {
12339                 int reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_FLOW_HIT, 0,
12340                                                  error);
12341
12342                 if (reg_c < 0) {
12343                         rte_flow_error_set(error, rte_errno,
12344                                            RTE_FLOW_ERROR_TYPE_ACTION,
12345                                            NULL, "failed to get reg_c "
12346                                            "for ASO flow hit");
12347                         return 0; /* 0 is an error. */
12348                 }
12349 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
12350                 age_free->dr_action = mlx5_glue->dv_create_flow_action_aso
12351                                 (priv->sh->rx_domain,
12352                                  pool->flow_hit_aso_obj->obj, age_free->offset,
12353                                  MLX5DV_DR_ACTION_FLAGS_ASO_FIRST_HIT_SET,
12354                                  (reg_c - REG_C_0));
12355 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
12356                 if (!age_free->dr_action) {
12357                         rte_errno = errno;
12358                         rte_spinlock_lock(&mng->free_sl);
12359                         LIST_INSERT_HEAD(&mng->free, age_free, next);
12360                         rte_spinlock_unlock(&mng->free_sl);
12361                         rte_flow_error_set(error, rte_errno,
12362                                            RTE_FLOW_ERROR_TYPE_ACTION,
12363                                            NULL, "failed to create ASO "
12364                                            "flow hit action");
12365                         return 0; /* 0 is an error. */
12366                 }
12367         }
12368         __atomic_store_n(&age_free->refcnt, 1, __ATOMIC_RELAXED);
12369         return pool->index | ((age_free->offset + 1) << 16);
12370 }
12371
12372 /**
12373  * Initialize flow ASO age parameters.
12374  *
12375  * @param[in] dev
12376  *   Pointer to rte_eth_dev structure.
12377  * @param[in] age_idx
12378  *   Index of ASO age action.
12379  * @param[in] context
12380  *   Pointer to flow counter age context.
12381  * @param[in] timeout
12382  *   Aging timeout in seconds.
12383  *
12384  */
12385 static void
12386 flow_dv_aso_age_params_init(struct rte_eth_dev *dev,
12387                             uint32_t age_idx,
12388                             void *context,
12389                             uint32_t timeout)
12390 {
12391         struct mlx5_aso_age_action *aso_age;
12392
12393         aso_age = flow_aso_age_get_by_idx(dev, age_idx);
12394         MLX5_ASSERT(aso_age);
12395         aso_age->age_params.context = context;
12396         aso_age->age_params.timeout = timeout;
12397         aso_age->age_params.port_id = dev->data->port_id;
12398         __atomic_store_n(&aso_age->age_params.sec_since_last_hit, 0,
12399                          __ATOMIC_RELAXED);
12400         __atomic_store_n(&aso_age->age_params.state, AGE_CANDIDATE,
12401                          __ATOMIC_RELAXED);
12402 }
12403
12404 static void
12405 flow_dv_translate_integrity_l4(const struct rte_flow_item_integrity *mask,
12406                                const struct rte_flow_item_integrity *value,
12407                                void *headers_m, void *headers_v)
12408 {
12409         if (mask->l4_ok) {
12410                 /* RTE l4_ok filter aggregates hardware l4_ok and
12411                  * l4_checksum_ok filters.
12412                  * Positive RTE l4_ok match requires hardware match on both L4
12413                  * hardware integrity bits.
12414                  * For negative match, check hardware l4_checksum_ok bit only,
12415                  * because hardware sets that bit to 0 for all packets
12416                  * with bad L4.
12417                  */
12418                 if (value->l4_ok) {
12419                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_ok, 1);
12420                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_ok, 1);
12421                 }
12422                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok, 1);
12423                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_checksum_ok,
12424                          !!value->l4_ok);
12425         }
12426         if (mask->l4_csum_ok) {
12427                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok, 1);
12428                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_checksum_ok,
12429                          value->l4_csum_ok);
12430         }
12431 }
12432
12433 static void
12434 flow_dv_translate_integrity_l3(const struct rte_flow_item_integrity *mask,
12435                                const struct rte_flow_item_integrity *value,
12436                                void *headers_m, void *headers_v, bool is_ipv4)
12437 {
12438         if (mask->l3_ok) {
12439                 /* RTE l3_ok filter aggregates for IPv4 hardware l3_ok and
12440                  * ipv4_csum_ok filters.
12441                  * Positive RTE l3_ok match requires hardware match on both L3
12442                  * hardware integrity bits.
12443                  * For negative match, check hardware l3_csum_ok bit only,
12444                  * because hardware sets that bit to 0 for all packets
12445                  * with bad L3.
12446                  */
12447                 if (is_ipv4) {
12448                         if (value->l3_ok) {
12449                                 MLX5_SET(fte_match_set_lyr_2_4, headers_m,
12450                                          l3_ok, 1);
12451                                 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
12452                                          l3_ok, 1);
12453                         }
12454                         MLX5_SET(fte_match_set_lyr_2_4, headers_m,
12455                                  ipv4_checksum_ok, 1);
12456                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
12457                                  ipv4_checksum_ok, !!value->l3_ok);
12458                 } else {
12459                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l3_ok, 1);
12460                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l3_ok,
12461                                  value->l3_ok);
12462                 }
12463         }
12464         if (mask->ipv4_csum_ok) {
12465                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok, 1);
12466                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_checksum_ok,
12467                          value->ipv4_csum_ok);
12468         }
12469 }
12470
12471 static void
12472 set_integrity_bits(void *headers_m, void *headers_v,
12473                    const struct rte_flow_item *integrity_item, bool is_l3_ip4)
12474 {
12475         const struct rte_flow_item_integrity *spec = integrity_item->spec;
12476         const struct rte_flow_item_integrity *mask = integrity_item->mask;
12477
12478         /* Integrity bits validation cleared spec pointer */
12479         MLX5_ASSERT(spec != NULL);
12480         if (!mask)
12481                 mask = &rte_flow_item_integrity_mask;
12482         flow_dv_translate_integrity_l3(mask, spec, headers_m, headers_v,
12483                                        is_l3_ip4);
12484         flow_dv_translate_integrity_l4(mask, spec, headers_m, headers_v);
12485 }
12486
12487 static void
12488 flow_dv_translate_item_integrity_post(void *matcher, void *key,
12489                                       const
12490                                       struct rte_flow_item *integrity_items[2],
12491                                       uint64_t pattern_flags)
12492 {
12493         void *headers_m, *headers_v;
12494         bool is_l3_ip4;
12495
12496         if (pattern_flags & MLX5_FLOW_ITEM_INNER_INTEGRITY) {
12497                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
12498                                          inner_headers);
12499                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
12500                 is_l3_ip4 = (pattern_flags & MLX5_FLOW_LAYER_INNER_L3_IPV4) !=
12501                             0;
12502                 set_integrity_bits(headers_m, headers_v,
12503                                    integrity_items[1], is_l3_ip4);
12504         }
12505         if (pattern_flags & MLX5_FLOW_ITEM_OUTER_INTEGRITY) {
12506                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
12507                                          outer_headers);
12508                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
12509                 is_l3_ip4 = (pattern_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4) !=
12510                             0;
12511                 set_integrity_bits(headers_m, headers_v,
12512                                    integrity_items[0], is_l3_ip4);
12513         }
12514 }
12515
12516 static void
12517 flow_dv_translate_item_integrity(const struct rte_flow_item *item,
12518                                  const struct rte_flow_item *integrity_items[2],
12519                                  uint64_t *last_item)
12520 {
12521         const struct rte_flow_item_integrity *spec = (typeof(spec))item->spec;
12522
12523         /* integrity bits validation cleared spec pointer */
12524         MLX5_ASSERT(spec != NULL);
12525         if (spec->level > 1) {
12526                 integrity_items[1] = item;
12527                 *last_item |= MLX5_FLOW_ITEM_INNER_INTEGRITY;
12528         } else {
12529                 integrity_items[0] = item;
12530                 *last_item |= MLX5_FLOW_ITEM_OUTER_INTEGRITY;
12531         }
12532 }
12533
12534 /**
12535  * Prepares DV flow counter with aging configuration.
12536  * Gets it by index when exists, creates a new one when doesn't.
12537  *
12538  * @param[in] dev
12539  *   Pointer to rte_eth_dev structure.
12540  * @param[in] dev_flow
12541  *   Pointer to the mlx5_flow.
12542  * @param[in, out] flow
12543  *   Pointer to the sub flow.
12544  * @param[in] count
12545  *   Pointer to the counter action configuration.
12546  * @param[in] age
12547  *   Pointer to the aging action configuration.
12548  * @param[out] error
12549  *   Pointer to the error structure.
12550  *
12551  * @return
12552  *   Pointer to the counter, NULL otherwise.
12553  */
12554 static struct mlx5_flow_counter *
12555 flow_dv_prepare_counter(struct rte_eth_dev *dev,
12556                         struct mlx5_flow *dev_flow,
12557                         struct rte_flow *flow,
12558                         const struct rte_flow_action_count *count,
12559                         const struct rte_flow_action_age *age,
12560                         struct rte_flow_error *error)
12561 {
12562         if (!flow->counter) {
12563                 flow->counter = flow_dv_translate_create_counter(dev, dev_flow,
12564                                                                  count, age);
12565                 if (!flow->counter) {
12566                         rte_flow_error_set(error, rte_errno,
12567                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12568                                            "cannot create counter object.");
12569                         return NULL;
12570                 }
12571         }
12572         return flow_dv_counter_get_by_idx(dev, flow->counter, NULL);
12573 }
12574
12575 /*
12576  * Release an ASO CT action by its own device.
12577  *
12578  * @param[in] dev
12579  *   Pointer to the Ethernet device structure.
12580  * @param[in] idx
12581  *   Index of ASO CT action to release.
12582  *
12583  * @return
12584  *   0 when CT action was removed, otherwise the number of references.
12585  */
12586 static inline int
12587 flow_dv_aso_ct_dev_release(struct rte_eth_dev *dev, uint32_t idx)
12588 {
12589         struct mlx5_priv *priv = dev->data->dev_private;
12590         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12591         uint32_t ret;
12592         struct mlx5_aso_ct_action *ct = flow_aso_ct_get_by_dev_idx(dev, idx);
12593         enum mlx5_aso_ct_state state =
12594                         __atomic_load_n(&ct->state, __ATOMIC_RELAXED);
12595
12596         /* Cannot release when CT is in the ASO SQ. */
12597         if (state == ASO_CONNTRACK_WAIT || state == ASO_CONNTRACK_QUERY)
12598                 return -1;
12599         ret = __atomic_sub_fetch(&ct->refcnt, 1, __ATOMIC_RELAXED);
12600         if (!ret) {
12601                 if (ct->dr_action_orig) {
12602 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12603                         claim_zero(mlx5_glue->destroy_flow_action
12604                                         (ct->dr_action_orig));
12605 #endif
12606                         ct->dr_action_orig = NULL;
12607                 }
12608                 if (ct->dr_action_rply) {
12609 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12610                         claim_zero(mlx5_glue->destroy_flow_action
12611                                         (ct->dr_action_rply));
12612 #endif
12613                         ct->dr_action_rply = NULL;
12614                 }
12615                 /* Clear the state to free, no need in 1st allocation. */
12616                 MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_FREE);
12617                 rte_spinlock_lock(&mng->ct_sl);
12618                 LIST_INSERT_HEAD(&mng->free_cts, ct, next);
12619                 rte_spinlock_unlock(&mng->ct_sl);
12620         }
12621         return (int)ret;
12622 }
12623
12624 static inline int
12625 flow_dv_aso_ct_release(struct rte_eth_dev *dev, uint32_t own_idx,
12626                        struct rte_flow_error *error)
12627 {
12628         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(own_idx);
12629         uint32_t idx = MLX5_INDIRECT_ACT_CT_GET_IDX(own_idx);
12630         struct rte_eth_dev *owndev = &rte_eth_devices[owner];
12631         int ret;
12632
12633         MLX5_ASSERT(owner < RTE_MAX_ETHPORTS);
12634         if (dev->data->dev_started != 1)
12635                 return rte_flow_error_set(error, EAGAIN,
12636                                           RTE_FLOW_ERROR_TYPE_ACTION,
12637                                           NULL,
12638                                           "Indirect CT action cannot be destroyed when the port is stopped");
12639         ret = flow_dv_aso_ct_dev_release(owndev, idx);
12640         if (ret < 0)
12641                 return rte_flow_error_set(error, EAGAIN,
12642                                           RTE_FLOW_ERROR_TYPE_ACTION,
12643                                           NULL,
12644                                           "Current state prevents indirect CT action from being destroyed");
12645         return ret;
12646 }
12647
12648 /*
12649  * Resize the ASO CT pools array by 64 pools.
12650  *
12651  * @param[in] dev
12652  *   Pointer to the Ethernet device structure.
12653  *
12654  * @return
12655  *   0 on success, otherwise negative errno value and rte_errno is set.
12656  */
12657 static int
12658 flow_dv_aso_ct_pools_resize(struct rte_eth_dev *dev)
12659 {
12660         struct mlx5_priv *priv = dev->data->dev_private;
12661         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12662         void *old_pools = mng->pools;
12663         /* Magic number now, need a macro. */
12664         uint32_t resize = mng->n + 64;
12665         uint32_t mem_size = sizeof(struct mlx5_aso_ct_pool *) * resize;
12666         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
12667
12668         if (!pools) {
12669                 rte_errno = ENOMEM;
12670                 return -rte_errno;
12671         }
12672         rte_rwlock_write_lock(&mng->resize_rwl);
12673         /* ASO SQ/QP was already initialized in the startup. */
12674         if (old_pools) {
12675                 /* Realloc could be an alternative choice. */
12676                 rte_memcpy(pools, old_pools,
12677                            mng->n * sizeof(struct mlx5_aso_ct_pool *));
12678                 mlx5_free(old_pools);
12679         }
12680         mng->n = resize;
12681         mng->pools = pools;
12682         rte_rwlock_write_unlock(&mng->resize_rwl);
12683         return 0;
12684 }
12685
12686 /*
12687  * Create and initialize a new ASO CT pool.
12688  *
12689  * @param[in] dev
12690  *   Pointer to the Ethernet device structure.
12691  * @param[out] ct_free
12692  *   Where to put the pointer of a new CT action.
12693  *
12694  * @return
12695  *   The CT actions pool pointer and @p ct_free is set on success,
12696  *   NULL otherwise and rte_errno is set.
12697  */
12698 static struct mlx5_aso_ct_pool *
12699 flow_dv_ct_pool_create(struct rte_eth_dev *dev,
12700                        struct mlx5_aso_ct_action **ct_free)
12701 {
12702         struct mlx5_priv *priv = dev->data->dev_private;
12703         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12704         struct mlx5_aso_ct_pool *pool = NULL;
12705         struct mlx5_devx_obj *obj = NULL;
12706         uint32_t i;
12707         uint32_t log_obj_size = rte_log2_u32(MLX5_ASO_CT_ACTIONS_PER_POOL);
12708
12709         obj = mlx5_devx_cmd_create_conn_track_offload_obj(priv->sh->cdev->ctx,
12710                                                           priv->sh->cdev->pdn,
12711                                                           log_obj_size);
12712         if (!obj) {
12713                 rte_errno = ENODATA;
12714                 DRV_LOG(ERR, "Failed to create conn_track_offload_obj using DevX.");
12715                 return NULL;
12716         }
12717         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
12718         if (!pool) {
12719                 rte_errno = ENOMEM;
12720                 claim_zero(mlx5_devx_cmd_destroy(obj));
12721                 return NULL;
12722         }
12723         pool->devx_obj = obj;
12724         pool->index = mng->next;
12725         /* Resize pools array if there is no room for the new pool in it. */
12726         if (pool->index == mng->n && flow_dv_aso_ct_pools_resize(dev)) {
12727                 claim_zero(mlx5_devx_cmd_destroy(obj));
12728                 mlx5_free(pool);
12729                 return NULL;
12730         }
12731         mng->pools[pool->index] = pool;
12732         mng->next++;
12733         /* Assign the first action in the new pool, the rest go to free list. */
12734         *ct_free = &pool->actions[0];
12735         /* Lock outside, the list operation is safe here. */
12736         for (i = 1; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) {
12737                 /* refcnt is 0 when allocating the memory. */
12738                 pool->actions[i].offset = i;
12739                 LIST_INSERT_HEAD(&mng->free_cts, &pool->actions[i], next);
12740         }
12741         return pool;
12742 }
12743
12744 /*
12745  * Allocate a ASO CT action from free list.
12746  *
12747  * @param[in] dev
12748  *   Pointer to the Ethernet device structure.
12749  * @param[out] error
12750  *   Pointer to the error structure.
12751  *
12752  * @return
12753  *   Index to ASO CT action on success, 0 otherwise and rte_errno is set.
12754  */
12755 static uint32_t
12756 flow_dv_aso_ct_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
12757 {
12758         struct mlx5_priv *priv = dev->data->dev_private;
12759         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12760         struct mlx5_aso_ct_action *ct = NULL;
12761         struct mlx5_aso_ct_pool *pool;
12762         uint8_t reg_c;
12763         uint32_t ct_idx;
12764
12765         MLX5_ASSERT(mng);
12766         if (!priv->sh->cdev->config.devx) {
12767                 rte_errno = ENOTSUP;
12768                 return 0;
12769         }
12770         /* Get a free CT action, if no, a new pool will be created. */
12771         rte_spinlock_lock(&mng->ct_sl);
12772         ct = LIST_FIRST(&mng->free_cts);
12773         if (ct) {
12774                 LIST_REMOVE(ct, next);
12775         } else if (!flow_dv_ct_pool_create(dev, &ct)) {
12776                 rte_spinlock_unlock(&mng->ct_sl);
12777                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
12778                                    NULL, "failed to create ASO CT pool");
12779                 return 0;
12780         }
12781         rte_spinlock_unlock(&mng->ct_sl);
12782         pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
12783         ct_idx = MLX5_MAKE_CT_IDX(pool->index, ct->offset);
12784         /* 0: inactive, 1: created, 2+: used by flows. */
12785         __atomic_store_n(&ct->refcnt, 1, __ATOMIC_RELAXED);
12786         reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, error);
12787         if (!ct->dr_action_orig) {
12788 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12789                 ct->dr_action_orig = mlx5_glue->dv_create_flow_action_aso
12790                         (priv->sh->rx_domain, pool->devx_obj->obj,
12791                          ct->offset,
12792                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_INITIATOR,
12793                          reg_c - REG_C_0);
12794 #else
12795                 RTE_SET_USED(reg_c);
12796 #endif
12797                 if (!ct->dr_action_orig) {
12798                         flow_dv_aso_ct_dev_release(dev, ct_idx);
12799                         rte_flow_error_set(error, rte_errno,
12800                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12801                                            "failed to create ASO CT action");
12802                         return 0;
12803                 }
12804         }
12805         if (!ct->dr_action_rply) {
12806 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12807                 ct->dr_action_rply = mlx5_glue->dv_create_flow_action_aso
12808                         (priv->sh->rx_domain, pool->devx_obj->obj,
12809                          ct->offset,
12810                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_RESPONDER,
12811                          reg_c - REG_C_0);
12812 #endif
12813                 if (!ct->dr_action_rply) {
12814                         flow_dv_aso_ct_dev_release(dev, ct_idx);
12815                         rte_flow_error_set(error, rte_errno,
12816                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12817                                            "failed to create ASO CT action");
12818                         return 0;
12819                 }
12820         }
12821         return ct_idx;
12822 }
12823
12824 /*
12825  * Create a conntrack object with context and actions by using ASO mechanism.
12826  *
12827  * @param[in] dev
12828  *   Pointer to rte_eth_dev structure.
12829  * @param[in] pro
12830  *   Pointer to conntrack information profile.
12831  * @param[out] error
12832  *   Pointer to the error structure.
12833  *
12834  * @return
12835  *   Index to conntrack object on success, 0 otherwise.
12836  */
12837 static uint32_t
12838 flow_dv_translate_create_conntrack(struct rte_eth_dev *dev,
12839                                    const struct rte_flow_action_conntrack *pro,
12840                                    struct rte_flow_error *error)
12841 {
12842         struct mlx5_priv *priv = dev->data->dev_private;
12843         struct mlx5_dev_ctx_shared *sh = priv->sh;
12844         struct mlx5_aso_ct_action *ct;
12845         uint32_t idx;
12846
12847         if (!sh->ct_aso_en)
12848                 return rte_flow_error_set(error, ENOTSUP,
12849                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12850                                           "Connection is not supported");
12851         idx = flow_dv_aso_ct_alloc(dev, error);
12852         if (!idx)
12853                 return rte_flow_error_set(error, rte_errno,
12854                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12855                                           "Failed to allocate CT object");
12856         ct = flow_aso_ct_get_by_dev_idx(dev, idx);
12857         if (mlx5_aso_ct_update_by_wqe(sh, ct, pro))
12858                 return rte_flow_error_set(error, EBUSY,
12859                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12860                                           "Failed to update CT");
12861         ct->is_original = !!pro->is_original_dir;
12862         ct->peer = pro->peer_port;
12863         return idx;
12864 }
12865
12866 /**
12867  * Fill the flow with DV spec, lock free
12868  * (mutex should be acquired by caller).
12869  *
12870  * @param[in] dev
12871  *   Pointer to rte_eth_dev structure.
12872  * @param[in, out] dev_flow
12873  *   Pointer to the sub flow.
12874  * @param[in] attr
12875  *   Pointer to the flow attributes.
12876  * @param[in] items
12877  *   Pointer to the list of items.
12878  * @param[in] actions
12879  *   Pointer to the list of actions.
12880  * @param[out] error
12881  *   Pointer to the error structure.
12882  *
12883  * @return
12884  *   0 on success, a negative errno value otherwise and rte_errno is set.
12885  */
12886 static int
12887 flow_dv_translate(struct rte_eth_dev *dev,
12888                   struct mlx5_flow *dev_flow,
12889                   const struct rte_flow_attr *attr,
12890                   const struct rte_flow_item items[],
12891                   const struct rte_flow_action actions[],
12892                   struct rte_flow_error *error)
12893 {
12894         struct mlx5_priv *priv = dev->data->dev_private;
12895         struct mlx5_sh_config *dev_conf = &priv->sh->config;
12896         struct rte_flow *flow = dev_flow->flow;
12897         struct mlx5_flow_handle *handle = dev_flow->handle;
12898         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
12899         struct mlx5_flow_rss_desc *rss_desc;
12900         uint64_t item_flags = 0;
12901         uint64_t last_item = 0;
12902         uint64_t action_flags = 0;
12903         struct mlx5_flow_dv_matcher matcher = {
12904                 .mask = {
12905                         .size = sizeof(matcher.mask.buf),
12906                 },
12907         };
12908         int actions_n = 0;
12909         bool actions_end = false;
12910         union {
12911                 struct mlx5_flow_dv_modify_hdr_resource res;
12912                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
12913                             sizeof(struct mlx5_modification_cmd) *
12914                             (MLX5_MAX_MODIFY_NUM + 1)];
12915         } mhdr_dummy;
12916         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
12917         const struct rte_flow_action_count *count = NULL;
12918         const struct rte_flow_action_age *non_shared_age = NULL;
12919         union flow_dv_attr flow_attr = { .attr = 0 };
12920         uint32_t tag_be;
12921         union mlx5_flow_tbl_key tbl_key;
12922         uint32_t modify_action_position = UINT32_MAX;
12923         void *match_mask = matcher.mask.buf;
12924         void *match_value = dev_flow->dv.value.buf;
12925         uint8_t next_protocol = 0xff;
12926         struct rte_vlan_hdr vlan = { 0 };
12927         struct mlx5_flow_dv_dest_array_resource mdest_res;
12928         struct mlx5_flow_dv_sample_resource sample_res;
12929         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
12930         const struct rte_flow_action_sample *sample = NULL;
12931         struct mlx5_flow_sub_actions_list *sample_act;
12932         uint32_t sample_act_pos = UINT32_MAX;
12933         uint32_t age_act_pos = UINT32_MAX;
12934         uint32_t num_of_dest = 0;
12935         int tmp_actions_n = 0;
12936         uint32_t table;
12937         int ret = 0;
12938         const struct mlx5_flow_tunnel *tunnel = NULL;
12939         struct flow_grp_info grp_info = {
12940                 .external = !!dev_flow->external,
12941                 .transfer = !!attr->transfer,
12942                 .fdb_def_rule = !!priv->fdb_def_rule,
12943                 .skip_scale = dev_flow->skip_scale &
12944                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
12945                 .std_tbl_fix = true,
12946         };
12947         const struct rte_flow_item *integrity_items[2] = {NULL, NULL};
12948         const struct rte_flow_item *tunnel_item = NULL;
12949         const struct rte_flow_item *gre_item = NULL;
12950
12951         if (!wks)
12952                 return rte_flow_error_set(error, ENOMEM,
12953                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12954                                           NULL,
12955                                           "failed to push flow workspace");
12956         rss_desc = &wks->rss_desc;
12957         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
12958         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
12959         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12960                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12961         /* update normal path action resource into last index of array */
12962         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
12963         if (is_tunnel_offload_active(dev)) {
12964                 if (dev_flow->tunnel) {
12965                         RTE_VERIFY(dev_flow->tof_type ==
12966                                    MLX5_TUNNEL_OFFLOAD_MISS_RULE);
12967                         tunnel = dev_flow->tunnel;
12968                 } else {
12969                         tunnel = mlx5_get_tof(items, actions,
12970                                               &dev_flow->tof_type);
12971                         dev_flow->tunnel = tunnel;
12972                 }
12973                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
12974                                         (dev, attr, tunnel, dev_flow->tof_type);
12975         }
12976         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12977                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12978         ret = mlx5_flow_group_to_table(dev, tunnel, attr->group, &table,
12979                                        &grp_info, error);
12980         if (ret)
12981                 return ret;
12982         dev_flow->dv.group = table;
12983         if (attr->transfer)
12984                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
12985         /* number of actions must be set to 0 in case of dirty stack. */
12986         mhdr_res->actions_num = 0;
12987         if (is_flow_tunnel_match_rule(dev_flow->tof_type)) {
12988                 /*
12989                  * do not add decap action if match rule drops packet
12990                  * HW rejects rules with decap & drop
12991                  *
12992                  * if tunnel match rule was inserted before matching tunnel set
12993                  * rule flow table used in the match rule must be registered.
12994                  * current implementation handles that in the
12995                  * flow_dv_match_register() at the function end.
12996                  */
12997                 bool add_decap = true;
12998                 const struct rte_flow_action *ptr = actions;
12999
13000                 for (; ptr->type != RTE_FLOW_ACTION_TYPE_END; ptr++) {
13001                         if (ptr->type == RTE_FLOW_ACTION_TYPE_DROP) {
13002                                 add_decap = false;
13003                                 break;
13004                         }
13005                 }
13006                 if (add_decap) {
13007                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
13008                                                            attr->transfer,
13009                                                            error))
13010                                 return -rte_errno;
13011                         dev_flow->dv.actions[actions_n++] =
13012                                         dev_flow->dv.encap_decap->action;
13013                         action_flags |= MLX5_FLOW_ACTION_DECAP;
13014                 }
13015         }
13016         for (; !actions_end ; actions++) {
13017                 const struct rte_flow_action_queue *queue;
13018                 const struct rte_flow_action_rss *rss;
13019                 const struct rte_flow_action *action = actions;
13020                 const uint8_t *rss_key;
13021                 struct mlx5_flow_tbl_resource *tbl;
13022                 struct mlx5_aso_age_action *age_act;
13023                 struct mlx5_flow_counter *cnt_act;
13024                 uint32_t port_id = 0;
13025                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
13026                 int action_type = actions->type;
13027                 const struct rte_flow_action *found_action = NULL;
13028                 uint32_t jump_group = 0;
13029                 uint32_t owner_idx;
13030                 struct mlx5_aso_ct_action *ct;
13031
13032                 if (!mlx5_flow_os_action_supported(action_type))
13033                         return rte_flow_error_set(error, ENOTSUP,
13034                                                   RTE_FLOW_ERROR_TYPE_ACTION,
13035                                                   actions,
13036                                                   "action not supported");
13037                 switch (action_type) {
13038                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
13039                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
13040                         break;
13041                 case RTE_FLOW_ACTION_TYPE_VOID:
13042                         break;
13043                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
13044                 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
13045                         if (flow_dv_translate_action_port_id(dev, action,
13046                                                              &port_id, error))
13047                                 return -rte_errno;
13048                         port_id_resource.port_id = port_id;
13049                         MLX5_ASSERT(!handle->rix_port_id_action);
13050                         if (flow_dv_port_id_action_resource_register
13051                             (dev, &port_id_resource, dev_flow, error))
13052                                 return -rte_errno;
13053                         dev_flow->dv.actions[actions_n++] =
13054                                         dev_flow->dv.port_id_action->action;
13055                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
13056                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
13057                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
13058                         num_of_dest++;
13059                         break;
13060                 case RTE_FLOW_ACTION_TYPE_FLAG:
13061                         action_flags |= MLX5_FLOW_ACTION_FLAG;
13062                         wks->mark = 1;
13063                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
13064                                 struct rte_flow_action_mark mark = {
13065                                         .id = MLX5_FLOW_MARK_DEFAULT,
13066                                 };
13067
13068                                 if (flow_dv_convert_action_mark(dev, &mark,
13069                                                                 mhdr_res,
13070                                                                 error))
13071                                         return -rte_errno;
13072                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
13073                                 break;
13074                         }
13075                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
13076                         /*
13077                          * Only one FLAG or MARK is supported per device flow
13078                          * right now. So the pointer to the tag resource must be
13079                          * zero before the register process.
13080                          */
13081                         MLX5_ASSERT(!handle->dvh.rix_tag);
13082                         if (flow_dv_tag_resource_register(dev, tag_be,
13083                                                           dev_flow, error))
13084                                 return -rte_errno;
13085                         MLX5_ASSERT(dev_flow->dv.tag_resource);
13086                         dev_flow->dv.actions[actions_n++] =
13087                                         dev_flow->dv.tag_resource->action;
13088                         break;
13089                 case RTE_FLOW_ACTION_TYPE_MARK:
13090                         action_flags |= MLX5_FLOW_ACTION_MARK;
13091                         wks->mark = 1;
13092                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
13093                                 const struct rte_flow_action_mark *mark =
13094                                         (const struct rte_flow_action_mark *)
13095                                                 actions->conf;
13096
13097                                 if (flow_dv_convert_action_mark(dev, mark,
13098                                                                 mhdr_res,
13099                                                                 error))
13100                                         return -rte_errno;
13101                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
13102                                 break;
13103                         }
13104                         /* Fall-through */
13105                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
13106                         /* Legacy (non-extensive) MARK action. */
13107                         tag_be = mlx5_flow_mark_set
13108                               (((const struct rte_flow_action_mark *)
13109                                (actions->conf))->id);
13110                         MLX5_ASSERT(!handle->dvh.rix_tag);
13111                         if (flow_dv_tag_resource_register(dev, tag_be,
13112                                                           dev_flow, error))
13113                                 return -rte_errno;
13114                         MLX5_ASSERT(dev_flow->dv.tag_resource);
13115                         dev_flow->dv.actions[actions_n++] =
13116                                         dev_flow->dv.tag_resource->action;
13117                         break;
13118                 case RTE_FLOW_ACTION_TYPE_SET_META:
13119                         if (flow_dv_convert_action_set_meta
13120                                 (dev, mhdr_res, attr,
13121                                  (const struct rte_flow_action_set_meta *)
13122                                   actions->conf, error))
13123                                 return -rte_errno;
13124                         action_flags |= MLX5_FLOW_ACTION_SET_META;
13125                         break;
13126                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
13127                         if (flow_dv_convert_action_set_tag
13128                                 (dev, mhdr_res,
13129                                  (const struct rte_flow_action_set_tag *)
13130                                   actions->conf, error))
13131                                 return -rte_errno;
13132                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
13133                         break;
13134                 case RTE_FLOW_ACTION_TYPE_DROP:
13135                         action_flags |= MLX5_FLOW_ACTION_DROP;
13136                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
13137                         break;
13138                 case RTE_FLOW_ACTION_TYPE_QUEUE:
13139                         queue = actions->conf;
13140                         rss_desc->queue_num = 1;
13141                         rss_desc->queue[0] = queue->index;
13142                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
13143                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
13144                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
13145                         num_of_dest++;
13146                         break;
13147                 case RTE_FLOW_ACTION_TYPE_RSS:
13148                         rss = actions->conf;
13149                         memcpy(rss_desc->queue, rss->queue,
13150                                rss->queue_num * sizeof(uint16_t));
13151                         rss_desc->queue_num = rss->queue_num;
13152                         /* NULL RSS key indicates default RSS key. */
13153                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
13154                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
13155                         /*
13156                          * rss->level and rss.types should be set in advance
13157                          * when expanding items for RSS.
13158                          */
13159                         action_flags |= MLX5_FLOW_ACTION_RSS;
13160                         dev_flow->handle->fate_action = rss_desc->shared_rss ?
13161                                 MLX5_FLOW_FATE_SHARED_RSS :
13162                                 MLX5_FLOW_FATE_QUEUE;
13163                         break;
13164                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
13165                         owner_idx = (uint32_t)(uintptr_t)action->conf;
13166                         age_act = flow_aso_age_get_by_idx(dev, owner_idx);
13167                         if (flow->age == 0) {
13168                                 flow->age = owner_idx;
13169                                 __atomic_fetch_add(&age_act->refcnt, 1,
13170                                                    __ATOMIC_RELAXED);
13171                         }
13172                         age_act_pos = actions_n++;
13173                         action_flags |= MLX5_FLOW_ACTION_AGE;
13174                         break;
13175                 case RTE_FLOW_ACTION_TYPE_AGE:
13176                         non_shared_age = action->conf;
13177                         age_act_pos = actions_n++;
13178                         action_flags |= MLX5_FLOW_ACTION_AGE;
13179                         break;
13180                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
13181                         owner_idx = (uint32_t)(uintptr_t)action->conf;
13182                         cnt_act = flow_dv_counter_get_by_idx(dev, owner_idx,
13183                                                              NULL);
13184                         MLX5_ASSERT(cnt_act != NULL);
13185                         /**
13186                          * When creating meter drop flow in drop table, the
13187                          * counter should not overwrite the rte flow counter.
13188                          */
13189                         if (attr->group == MLX5_FLOW_TABLE_LEVEL_METER &&
13190                             dev_flow->dv.table_id == MLX5_MTR_TABLE_ID_DROP) {
13191                                 dev_flow->dv.actions[actions_n++] =
13192                                                         cnt_act->action;
13193                         } else {
13194                                 if (flow->counter == 0) {
13195                                         flow->counter = owner_idx;
13196                                         __atomic_fetch_add
13197                                                 (&cnt_act->shared_info.refcnt,
13198                                                  1, __ATOMIC_RELAXED);
13199                                 }
13200                                 /* Save information first, will apply later. */
13201                                 action_flags |= MLX5_FLOW_ACTION_COUNT;
13202                         }
13203                         break;
13204                 case RTE_FLOW_ACTION_TYPE_COUNT:
13205                         if (!priv->sh->cdev->config.devx) {
13206                                 return rte_flow_error_set
13207                                               (error, ENOTSUP,
13208                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
13209                                                NULL,
13210                                                "count action not supported");
13211                         }
13212                         /* Save information first, will apply later. */
13213                         count = action->conf;
13214                         action_flags |= MLX5_FLOW_ACTION_COUNT;
13215                         break;
13216                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
13217                         dev_flow->dv.actions[actions_n++] =
13218                                                 priv->sh->pop_vlan_action;
13219                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
13220                         break;
13221                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
13222                         if (!(action_flags &
13223                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
13224                                 flow_dev_get_vlan_info_from_items(items, &vlan);
13225                         vlan.eth_proto = rte_be_to_cpu_16
13226                              ((((const struct rte_flow_action_of_push_vlan *)
13227                                                    actions->conf)->ethertype));
13228                         found_action = mlx5_flow_find_action
13229                                         (actions + 1,
13230                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
13231                         if (found_action)
13232                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
13233                         found_action = mlx5_flow_find_action
13234                                         (actions + 1,
13235                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
13236                         if (found_action)
13237                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
13238                         if (flow_dv_create_action_push_vlan
13239                                             (dev, attr, &vlan, dev_flow, error))
13240                                 return -rte_errno;
13241                         dev_flow->dv.actions[actions_n++] =
13242                                         dev_flow->dv.push_vlan_res->action;
13243                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
13244                         break;
13245                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
13246                         /* of_vlan_push action handled this action */
13247                         MLX5_ASSERT(action_flags &
13248                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
13249                         break;
13250                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
13251                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
13252                                 break;
13253                         flow_dev_get_vlan_info_from_items(items, &vlan);
13254                         mlx5_update_vlan_vid_pcp(actions, &vlan);
13255                         /* If no VLAN push - this is a modify header action */
13256                         if (flow_dv_convert_action_modify_vlan_vid
13257                                                 (mhdr_res, actions, error))
13258                                 return -rte_errno;
13259                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
13260                         break;
13261                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
13262                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
13263                         if (flow_dv_create_action_l2_encap(dev, actions,
13264                                                            dev_flow,
13265                                                            attr->transfer,
13266                                                            error))
13267                                 return -rte_errno;
13268                         dev_flow->dv.actions[actions_n++] =
13269                                         dev_flow->dv.encap_decap->action;
13270                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
13271                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
13272                                 sample_act->action_flags |=
13273                                                         MLX5_FLOW_ACTION_ENCAP;
13274                         break;
13275                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
13276                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
13277                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
13278                                                            attr->transfer,
13279                                                            error))
13280                                 return -rte_errno;
13281                         dev_flow->dv.actions[actions_n++] =
13282                                         dev_flow->dv.encap_decap->action;
13283                         action_flags |= MLX5_FLOW_ACTION_DECAP;
13284                         break;
13285                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
13286                         /* Handle encap with preceding decap. */
13287                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
13288                                 if (flow_dv_create_action_raw_encap
13289                                         (dev, actions, dev_flow, attr, error))
13290                                         return -rte_errno;
13291                                 dev_flow->dv.actions[actions_n++] =
13292                                         dev_flow->dv.encap_decap->action;
13293                         } else {
13294                                 /* Handle encap without preceding decap. */
13295                                 if (flow_dv_create_action_l2_encap
13296                                     (dev, actions, dev_flow, attr->transfer,
13297                                      error))
13298                                         return -rte_errno;
13299                                 dev_flow->dv.actions[actions_n++] =
13300                                         dev_flow->dv.encap_decap->action;
13301                         }
13302                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
13303                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
13304                                 sample_act->action_flags |=
13305                                                         MLX5_FLOW_ACTION_ENCAP;
13306                         break;
13307                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
13308                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
13309                                 ;
13310                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
13311                                 if (flow_dv_create_action_l2_decap
13312                                     (dev, dev_flow, attr->transfer, error))
13313                                         return -rte_errno;
13314                                 dev_flow->dv.actions[actions_n++] =
13315                                         dev_flow->dv.encap_decap->action;
13316                         }
13317                         /* If decap is followed by encap, handle it at encap. */
13318                         action_flags |= MLX5_FLOW_ACTION_DECAP;
13319                         break;
13320                 case MLX5_RTE_FLOW_ACTION_TYPE_JUMP:
13321                         dev_flow->dv.actions[actions_n++] =
13322                                 (void *)(uintptr_t)action->conf;
13323                         action_flags |= MLX5_FLOW_ACTION_JUMP;
13324                         break;
13325                 case RTE_FLOW_ACTION_TYPE_JUMP:
13326                         jump_group = ((const struct rte_flow_action_jump *)
13327                                                         action->conf)->group;
13328                         grp_info.std_tbl_fix = 0;
13329                         if (dev_flow->skip_scale &
13330                                 (1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT))
13331                                 grp_info.skip_scale = 1;
13332                         else
13333                                 grp_info.skip_scale = 0;
13334                         ret = mlx5_flow_group_to_table(dev, tunnel,
13335                                                        jump_group,
13336                                                        &table,
13337                                                        &grp_info, error);
13338                         if (ret)
13339                                 return ret;
13340                         tbl = flow_dv_tbl_resource_get(dev, table, attr->egress,
13341                                                        attr->transfer,
13342                                                        !!dev_flow->external,
13343                                                        tunnel, jump_group, 0,
13344                                                        0, error);
13345                         if (!tbl)
13346                                 return rte_flow_error_set
13347                                                 (error, errno,
13348                                                  RTE_FLOW_ERROR_TYPE_ACTION,
13349                                                  NULL,
13350                                                  "cannot create jump action.");
13351                         if (flow_dv_jump_tbl_resource_register
13352                             (dev, tbl, dev_flow, error)) {
13353                                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
13354                                 return rte_flow_error_set
13355                                                 (error, errno,
13356                                                  RTE_FLOW_ERROR_TYPE_ACTION,
13357                                                  NULL,
13358                                                  "cannot create jump action.");
13359                         }
13360                         dev_flow->dv.actions[actions_n++] =
13361                                         dev_flow->dv.jump->action;
13362                         action_flags |= MLX5_FLOW_ACTION_JUMP;
13363                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
13364                         sample_act->action_flags |= MLX5_FLOW_ACTION_JUMP;
13365                         num_of_dest++;
13366                         break;
13367                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
13368                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
13369                         if (flow_dv_convert_action_modify_mac
13370                                         (mhdr_res, actions, error))
13371                                 return -rte_errno;
13372                         action_flags |= actions->type ==
13373                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
13374                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
13375                                         MLX5_FLOW_ACTION_SET_MAC_DST;
13376                         break;
13377                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
13378                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
13379                         if (flow_dv_convert_action_modify_ipv4
13380                                         (mhdr_res, actions, error))
13381                                 return -rte_errno;
13382                         action_flags |= actions->type ==
13383                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
13384                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
13385                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
13386                         break;
13387                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
13388                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
13389                         if (flow_dv_convert_action_modify_ipv6
13390                                         (mhdr_res, actions, error))
13391                                 return -rte_errno;
13392                         action_flags |= actions->type ==
13393                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
13394                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
13395                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
13396                         break;
13397                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
13398                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
13399                         if (flow_dv_convert_action_modify_tp
13400                                         (mhdr_res, actions, items,
13401                                          &flow_attr, dev_flow, !!(action_flags &
13402                                          MLX5_FLOW_ACTION_DECAP), error))
13403                                 return -rte_errno;
13404                         action_flags |= actions->type ==
13405                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
13406                                         MLX5_FLOW_ACTION_SET_TP_SRC :
13407                                         MLX5_FLOW_ACTION_SET_TP_DST;
13408                         break;
13409                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
13410                         if (flow_dv_convert_action_modify_dec_ttl
13411                                         (mhdr_res, items, &flow_attr, dev_flow,
13412                                          !!(action_flags &
13413                                          MLX5_FLOW_ACTION_DECAP), error))
13414                                 return -rte_errno;
13415                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
13416                         break;
13417                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
13418                         if (flow_dv_convert_action_modify_ttl
13419                                         (mhdr_res, actions, items, &flow_attr,
13420                                          dev_flow, !!(action_flags &
13421                                          MLX5_FLOW_ACTION_DECAP), error))
13422                                 return -rte_errno;
13423                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
13424                         break;
13425                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
13426                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
13427                         if (flow_dv_convert_action_modify_tcp_seq
13428                                         (mhdr_res, actions, error))
13429                                 return -rte_errno;
13430                         action_flags |= actions->type ==
13431                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
13432                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
13433                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
13434                         break;
13435
13436                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
13437                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
13438                         if (flow_dv_convert_action_modify_tcp_ack
13439                                         (mhdr_res, actions, error))
13440                                 return -rte_errno;
13441                         action_flags |= actions->type ==
13442                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
13443                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
13444                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
13445                         break;
13446                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
13447                         if (flow_dv_convert_action_set_reg
13448                                         (mhdr_res, actions, error))
13449                                 return -rte_errno;
13450                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
13451                         break;
13452                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
13453                         if (flow_dv_convert_action_copy_mreg
13454                                         (dev, mhdr_res, actions, error))
13455                                 return -rte_errno;
13456                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
13457                         break;
13458                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
13459                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
13460                         dev_flow->handle->fate_action =
13461                                         MLX5_FLOW_FATE_DEFAULT_MISS;
13462                         break;
13463                 case RTE_FLOW_ACTION_TYPE_METER:
13464                         if (!wks->fm)
13465                                 return rte_flow_error_set(error, rte_errno,
13466                                         RTE_FLOW_ERROR_TYPE_ACTION,
13467                                         NULL, "Failed to get meter in flow.");
13468                         /* Set the meter action. */
13469                         dev_flow->dv.actions[actions_n++] =
13470                                 wks->fm->meter_action_g;
13471                         action_flags |= MLX5_FLOW_ACTION_METER;
13472                         break;
13473                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
13474                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
13475                                                               actions, error))
13476                                 return -rte_errno;
13477                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
13478                         break;
13479                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
13480                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
13481                                                               actions, error))
13482                                 return -rte_errno;
13483                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
13484                         break;
13485                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
13486                         sample_act_pos = actions_n;
13487                         sample = (const struct rte_flow_action_sample *)
13488                                  action->conf;
13489                         actions_n++;
13490                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
13491                         /* put encap action into group if work with port id */
13492                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
13493                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
13494                                 sample_act->action_flags |=
13495                                                         MLX5_FLOW_ACTION_ENCAP;
13496                         break;
13497                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
13498                         if (flow_dv_convert_action_modify_field
13499                                         (dev, mhdr_res, actions, attr, error))
13500                                 return -rte_errno;
13501                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
13502                         break;
13503                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
13504                         owner_idx = (uint32_t)(uintptr_t)action->conf;
13505                         ct = flow_aso_ct_get_by_idx(dev, owner_idx);
13506                         if (!ct)
13507                                 return rte_flow_error_set(error, EINVAL,
13508                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13509                                                 NULL,
13510                                                 "Failed to get CT object.");
13511                         if (mlx5_aso_ct_available(priv->sh, ct))
13512                                 return rte_flow_error_set(error, rte_errno,
13513                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13514                                                 NULL,
13515                                                 "CT is unavailable.");
13516                         if (ct->is_original)
13517                                 dev_flow->dv.actions[actions_n] =
13518                                                         ct->dr_action_orig;
13519                         else
13520                                 dev_flow->dv.actions[actions_n] =
13521                                                         ct->dr_action_rply;
13522                         if (flow->ct == 0) {
13523                                 flow->indirect_type =
13524                                                 MLX5_INDIRECT_ACTION_TYPE_CT;
13525                                 flow->ct = owner_idx;
13526                                 __atomic_fetch_add(&ct->refcnt, 1,
13527                                                    __ATOMIC_RELAXED);
13528                         }
13529                         actions_n++;
13530                         action_flags |= MLX5_FLOW_ACTION_CT;
13531                         break;
13532                 case RTE_FLOW_ACTION_TYPE_END:
13533                         actions_end = true;
13534                         if (mhdr_res->actions_num) {
13535                                 /* create modify action if needed. */
13536                                 if (flow_dv_modify_hdr_resource_register
13537                                         (dev, mhdr_res, dev_flow, error))
13538                                         return -rte_errno;
13539                                 dev_flow->dv.actions[modify_action_position] =
13540                                         handle->dvh.modify_hdr->action;
13541                         }
13542                         /*
13543                          * Handle AGE and COUNT action by single HW counter
13544                          * when they are not shared.
13545                          */
13546                         if (action_flags & MLX5_FLOW_ACTION_AGE) {
13547                                 if ((non_shared_age && count) ||
13548                                     !flow_hit_aso_supported(priv->sh, attr)) {
13549                                         /* Creates age by counters. */
13550                                         cnt_act = flow_dv_prepare_counter
13551                                                                 (dev, dev_flow,
13552                                                                  flow, count,
13553                                                                  non_shared_age,
13554                                                                  error);
13555                                         if (!cnt_act)
13556                                                 return -rte_errno;
13557                                         dev_flow->dv.actions[age_act_pos] =
13558                                                                 cnt_act->action;
13559                                         break;
13560                                 }
13561                                 if (!flow->age && non_shared_age) {
13562                                         flow->age = flow_dv_aso_age_alloc
13563                                                                 (dev, error);
13564                                         if (!flow->age)
13565                                                 return -rte_errno;
13566                                         flow_dv_aso_age_params_init
13567                                                     (dev, flow->age,
13568                                                      non_shared_age->context ?
13569                                                      non_shared_age->context :
13570                                                      (void *)(uintptr_t)
13571                                                      (dev_flow->flow_idx),
13572                                                      non_shared_age->timeout);
13573                                 }
13574                                 age_act = flow_aso_age_get_by_idx(dev,
13575                                                                   flow->age);
13576                                 dev_flow->dv.actions[age_act_pos] =
13577                                                              age_act->dr_action;
13578                         }
13579                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
13580                                 /*
13581                                  * Create one count action, to be used
13582                                  * by all sub-flows.
13583                                  */
13584                                 cnt_act = flow_dv_prepare_counter(dev, dev_flow,
13585                                                                   flow, count,
13586                                                                   NULL, error);
13587                                 if (!cnt_act)
13588                                         return -rte_errno;
13589                                 dev_flow->dv.actions[actions_n++] =
13590                                                                 cnt_act->action;
13591                         }
13592                 default:
13593                         break;
13594                 }
13595                 if (mhdr_res->actions_num &&
13596                     modify_action_position == UINT32_MAX)
13597                         modify_action_position = actions_n++;
13598         }
13599         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
13600                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
13601                 int item_type = items->type;
13602
13603                 if (!mlx5_flow_os_item_supported(item_type))
13604                         return rte_flow_error_set(error, ENOTSUP,
13605                                                   RTE_FLOW_ERROR_TYPE_ITEM,
13606                                                   NULL, "item not supported");
13607                 switch (item_type) {
13608                 case RTE_FLOW_ITEM_TYPE_ESP:
13609                         flow_dv_translate_item_esp(match_mask, match_value,
13610                                                    items, tunnel);
13611                         last_item = MLX5_FLOW_ITEM_ESP;
13612                         break;
13613                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
13614                         flow_dv_translate_item_port_id
13615                                 (dev, match_mask, match_value, items, attr);
13616                         last_item = MLX5_FLOW_ITEM_PORT_ID;
13617                         break;
13618                 case RTE_FLOW_ITEM_TYPE_ETH:
13619                         flow_dv_translate_item_eth(match_mask, match_value,
13620                                                    items, tunnel,
13621                                                    dev_flow->dv.group);
13622                         matcher.priority = action_flags &
13623                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
13624                                         !dev_flow->external ?
13625                                         MLX5_PRIORITY_MAP_L3 :
13626                                         MLX5_PRIORITY_MAP_L2;
13627                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
13628                                              MLX5_FLOW_LAYER_OUTER_L2;
13629                         break;
13630                 case RTE_FLOW_ITEM_TYPE_VLAN:
13631                         flow_dv_translate_item_vlan(dev_flow,
13632                                                     match_mask, match_value,
13633                                                     items, tunnel,
13634                                                     dev_flow->dv.group);
13635                         matcher.priority = MLX5_PRIORITY_MAP_L2;
13636                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
13637                                               MLX5_FLOW_LAYER_INNER_VLAN) :
13638                                              (MLX5_FLOW_LAYER_OUTER_L2 |
13639                                               MLX5_FLOW_LAYER_OUTER_VLAN);
13640                         break;
13641                 case RTE_FLOW_ITEM_TYPE_IPV4:
13642                         mlx5_flow_tunnel_ip_check(items, next_protocol,
13643                                                   &item_flags, &tunnel);
13644                         flow_dv_translate_item_ipv4(match_mask, match_value,
13645                                                     items, tunnel,
13646                                                     dev_flow->dv.group);
13647                         matcher.priority = MLX5_PRIORITY_MAP_L3;
13648                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
13649                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
13650                         if (items->mask != NULL &&
13651                             ((const struct rte_flow_item_ipv4 *)
13652                              items->mask)->hdr.next_proto_id) {
13653                                 next_protocol =
13654                                         ((const struct rte_flow_item_ipv4 *)
13655                                          (items->spec))->hdr.next_proto_id;
13656                                 next_protocol &=
13657                                         ((const struct rte_flow_item_ipv4 *)
13658                                          (items->mask))->hdr.next_proto_id;
13659                         } else {
13660                                 /* Reset for inner layer. */
13661                                 next_protocol = 0xff;
13662                         }
13663                         break;
13664                 case RTE_FLOW_ITEM_TYPE_IPV6:
13665                         mlx5_flow_tunnel_ip_check(items, next_protocol,
13666                                                   &item_flags, &tunnel);
13667                         flow_dv_translate_item_ipv6(match_mask, match_value,
13668                                                     items, tunnel,
13669                                                     dev_flow->dv.group);
13670                         matcher.priority = MLX5_PRIORITY_MAP_L3;
13671                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
13672                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
13673                         if (items->mask != NULL &&
13674                             ((const struct rte_flow_item_ipv6 *)
13675                              items->mask)->hdr.proto) {
13676                                 next_protocol =
13677                                         ((const struct rte_flow_item_ipv6 *)
13678                                          items->spec)->hdr.proto;
13679                                 next_protocol &=
13680                                         ((const struct rte_flow_item_ipv6 *)
13681                                          items->mask)->hdr.proto;
13682                         } else {
13683                                 /* Reset for inner layer. */
13684                                 next_protocol = 0xff;
13685                         }
13686                         break;
13687                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
13688                         flow_dv_translate_item_ipv6_frag_ext(match_mask,
13689                                                              match_value,
13690                                                              items, tunnel);
13691                         last_item = tunnel ?
13692                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
13693                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
13694                         if (items->mask != NULL &&
13695                             ((const struct rte_flow_item_ipv6_frag_ext *)
13696                              items->mask)->hdr.next_header) {
13697                                 next_protocol =
13698                                 ((const struct rte_flow_item_ipv6_frag_ext *)
13699                                  items->spec)->hdr.next_header;
13700                                 next_protocol &=
13701                                 ((const struct rte_flow_item_ipv6_frag_ext *)
13702                                  items->mask)->hdr.next_header;
13703                         } else {
13704                                 /* Reset for inner layer. */
13705                                 next_protocol = 0xff;
13706                         }
13707                         break;
13708                 case RTE_FLOW_ITEM_TYPE_TCP:
13709                         flow_dv_translate_item_tcp(match_mask, match_value,
13710                                                    items, tunnel);
13711                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13712                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
13713                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
13714                         break;
13715                 case RTE_FLOW_ITEM_TYPE_UDP:
13716                         flow_dv_translate_item_udp(match_mask, match_value,
13717                                                    items, tunnel);
13718                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13719                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
13720                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
13721                         break;
13722                 case RTE_FLOW_ITEM_TYPE_GRE:
13723                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13724                         last_item = MLX5_FLOW_LAYER_GRE;
13725                         tunnel_item = items;
13726                         gre_item = items;
13727                         break;
13728                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
13729                         flow_dv_translate_item_gre_key(match_mask,
13730                                                        match_value, items);
13731                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
13732                         break;
13733                 case RTE_FLOW_ITEM_TYPE_GRE_OPTION:
13734                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13735                         last_item = MLX5_FLOW_LAYER_GRE;
13736                         tunnel_item = items;
13737                         break;
13738                 case RTE_FLOW_ITEM_TYPE_NVGRE:
13739                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13740                         last_item = MLX5_FLOW_LAYER_GRE;
13741                         tunnel_item = items;
13742                         break;
13743                 case RTE_FLOW_ITEM_TYPE_VXLAN:
13744                         flow_dv_translate_item_vxlan(dev, attr,
13745                                                      match_mask, match_value,
13746                                                      items, tunnel);
13747                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13748                         last_item = MLX5_FLOW_LAYER_VXLAN;
13749                         break;
13750                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
13751                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13752                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
13753                         tunnel_item = items;
13754                         break;
13755                 case RTE_FLOW_ITEM_TYPE_GENEVE:
13756                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13757                         last_item = MLX5_FLOW_LAYER_GENEVE;
13758                         tunnel_item = items;
13759                         break;
13760                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
13761                         ret = flow_dv_translate_item_geneve_opt(dev, match_mask,
13762                                                           match_value,
13763                                                           items, error);
13764                         if (ret)
13765                                 return rte_flow_error_set(error, -ret,
13766                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
13767                                         "cannot create GENEVE TLV option");
13768                         flow->geneve_tlv_option = 1;
13769                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
13770                         break;
13771                 case RTE_FLOW_ITEM_TYPE_MPLS:
13772                         flow_dv_translate_item_mpls(match_mask, match_value,
13773                                                     items, last_item, tunnel);
13774                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13775                         last_item = MLX5_FLOW_LAYER_MPLS;
13776                         break;
13777                 case RTE_FLOW_ITEM_TYPE_MARK:
13778                         flow_dv_translate_item_mark(dev, match_mask,
13779                                                     match_value, items);
13780                         last_item = MLX5_FLOW_ITEM_MARK;
13781                         break;
13782                 case RTE_FLOW_ITEM_TYPE_META:
13783                         flow_dv_translate_item_meta(dev, match_mask,
13784                                                     match_value, attr, items);
13785                         last_item = MLX5_FLOW_ITEM_METADATA;
13786                         break;
13787                 case RTE_FLOW_ITEM_TYPE_ICMP:
13788                         flow_dv_translate_item_icmp(match_mask, match_value,
13789                                                     items, tunnel);
13790                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13791                         last_item = MLX5_FLOW_LAYER_ICMP;
13792                         break;
13793                 case RTE_FLOW_ITEM_TYPE_ICMP6:
13794                         flow_dv_translate_item_icmp6(match_mask, match_value,
13795                                                       items, tunnel);
13796                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13797                         last_item = MLX5_FLOW_LAYER_ICMP6;
13798                         break;
13799                 case RTE_FLOW_ITEM_TYPE_TAG:
13800                         flow_dv_translate_item_tag(dev, match_mask,
13801                                                    match_value, items);
13802                         last_item = MLX5_FLOW_ITEM_TAG;
13803                         break;
13804                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
13805                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
13806                                                         match_value, items);
13807                         last_item = MLX5_FLOW_ITEM_TAG;
13808                         break;
13809                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
13810                         flow_dv_translate_item_tx_queue(dev, match_mask,
13811                                                         match_value,
13812                                                         items);
13813                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
13814                         break;
13815                 case RTE_FLOW_ITEM_TYPE_GTP:
13816                         flow_dv_translate_item_gtp(match_mask, match_value,
13817                                                    items, tunnel);
13818                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13819                         last_item = MLX5_FLOW_LAYER_GTP;
13820                         break;
13821                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
13822                         ret = flow_dv_translate_item_gtp_psc(match_mask,
13823                                                           match_value,
13824                                                           items);
13825                         if (ret)
13826                                 return rte_flow_error_set(error, -ret,
13827                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
13828                                         "cannot create GTP PSC item");
13829                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
13830                         break;
13831                 case RTE_FLOW_ITEM_TYPE_ECPRI:
13832                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
13833                                 /* Create it only the first time to be used. */
13834                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
13835                                 if (ret)
13836                                         return rte_flow_error_set
13837                                                 (error, -ret,
13838                                                 RTE_FLOW_ERROR_TYPE_ITEM,
13839                                                 NULL,
13840                                                 "cannot create eCPRI parser");
13841                         }
13842                         flow_dv_translate_item_ecpri(dev, match_mask,
13843                                                      match_value, items,
13844                                                      last_item);
13845                         /* No other protocol should follow eCPRI layer. */
13846                         last_item = MLX5_FLOW_LAYER_ECPRI;
13847                         break;
13848                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
13849                         flow_dv_translate_item_integrity(items, integrity_items,
13850                                                          &last_item);
13851                         break;
13852                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
13853                         flow_dv_translate_item_aso_ct(dev, match_mask,
13854                                                       match_value, items);
13855                         break;
13856                 case RTE_FLOW_ITEM_TYPE_FLEX:
13857                         flow_dv_translate_item_flex(dev, match_mask,
13858                                                     match_value, items,
13859                                                     dev_flow, tunnel != 0);
13860                         last_item = tunnel ? MLX5_FLOW_ITEM_INNER_FLEX :
13861                                     MLX5_FLOW_ITEM_OUTER_FLEX;
13862                         break;
13863                 default:
13864                         break;
13865                 }
13866                 item_flags |= last_item;
13867         }
13868         /*
13869          * When E-Switch mode is enabled, we have two cases where we need to
13870          * set the source port manually.
13871          * The first one, is in case of NIC ingress steering rule, and the
13872          * second is E-Switch rule where no port_id item was found.
13873          * In both cases the source port is set according the current port
13874          * in use.
13875          */
13876         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) && priv->sh->esw_mode &&
13877             !(attr->egress && !attr->transfer)) {
13878                 if (flow_dv_translate_item_port_id(dev, match_mask,
13879                                                    match_value, NULL, attr))
13880                         return -rte_errno;
13881         }
13882         if (item_flags & MLX5_FLOW_ITEM_INTEGRITY) {
13883                 flow_dv_translate_item_integrity_post(match_mask, match_value,
13884                                                       integrity_items,
13885                                                       item_flags);
13886         }
13887         if (item_flags & MLX5_FLOW_LAYER_VXLAN_GPE)
13888                 flow_dv_translate_item_vxlan_gpe(match_mask, match_value,
13889                                                  tunnel_item, item_flags);
13890         else if (item_flags & MLX5_FLOW_LAYER_GENEVE)
13891                 flow_dv_translate_item_geneve(match_mask, match_value,
13892                                               tunnel_item, item_flags);
13893         else if (item_flags & MLX5_FLOW_LAYER_GRE) {
13894                 if (tunnel_item->type == RTE_FLOW_ITEM_TYPE_GRE)
13895                         flow_dv_translate_item_gre(match_mask, match_value,
13896                                                    tunnel_item, item_flags);
13897                 else if (tunnel_item->type == RTE_FLOW_ITEM_TYPE_NVGRE)
13898                         flow_dv_translate_item_nvgre(match_mask, match_value,
13899                                                      tunnel_item, item_flags);
13900                 else if (tunnel_item->type == RTE_FLOW_ITEM_TYPE_GRE_OPTION)
13901                         flow_dv_translate_item_gre_option(match_mask, match_value,
13902                                         tunnel_item, gre_item, item_flags);
13903                 else
13904                         MLX5_ASSERT(false);
13905         }
13906 #ifdef RTE_LIBRTE_MLX5_DEBUG
13907         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
13908                                               dev_flow->dv.value.buf));
13909 #endif
13910         /*
13911          * Layers may be already initialized from prefix flow if this dev_flow
13912          * is the suffix flow.
13913          */
13914         handle->layers |= item_flags;
13915         if (action_flags & MLX5_FLOW_ACTION_RSS)
13916                 flow_dv_hashfields_set(dev_flow->handle->layers,
13917                                        rss_desc,
13918                                        &dev_flow->hash_fields);
13919         /* If has RSS action in the sample action, the Sample/Mirror resource
13920          * should be registered after the hash filed be update.
13921          */
13922         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
13923                 ret = flow_dv_translate_action_sample(dev,
13924                                                       sample,
13925                                                       dev_flow, attr,
13926                                                       &num_of_dest,
13927                                                       sample_actions,
13928                                                       &sample_res,
13929                                                       error);
13930                 if (ret < 0)
13931                         return ret;
13932                 ret = flow_dv_create_action_sample(dev,
13933                                                    dev_flow,
13934                                                    num_of_dest,
13935                                                    &sample_res,
13936                                                    &mdest_res,
13937                                                    sample_actions,
13938                                                    action_flags,
13939                                                    error);
13940                 if (ret < 0)
13941                         return rte_flow_error_set
13942                                                 (error, rte_errno,
13943                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13944                                                 NULL,
13945                                                 "cannot create sample action");
13946                 if (num_of_dest > 1) {
13947                         dev_flow->dv.actions[sample_act_pos] =
13948                         dev_flow->dv.dest_array_res->action;
13949                 } else {
13950                         dev_flow->dv.actions[sample_act_pos] =
13951                         dev_flow->dv.sample_res->verbs_action;
13952                 }
13953         }
13954         /*
13955          * For multiple destination (sample action with ratio=1), the encap
13956          * action and port id action will be combined into group action.
13957          * So need remove the original these actions in the flow and only
13958          * use the sample action instead of.
13959          */
13960         if (num_of_dest > 1 &&
13961             (sample_act->dr_port_id_action || sample_act->dr_jump_action)) {
13962                 int i;
13963                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
13964
13965                 for (i = 0; i < actions_n; i++) {
13966                         if ((sample_act->dr_encap_action &&
13967                                 sample_act->dr_encap_action ==
13968                                 dev_flow->dv.actions[i]) ||
13969                                 (sample_act->dr_port_id_action &&
13970                                 sample_act->dr_port_id_action ==
13971                                 dev_flow->dv.actions[i]) ||
13972                                 (sample_act->dr_jump_action &&
13973                                 sample_act->dr_jump_action ==
13974                                 dev_flow->dv.actions[i]))
13975                                 continue;
13976                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
13977                 }
13978                 memcpy((void *)dev_flow->dv.actions,
13979                                 (void *)temp_actions,
13980                                 tmp_actions_n * sizeof(void *));
13981                 actions_n = tmp_actions_n;
13982         }
13983         dev_flow->dv.actions_n = actions_n;
13984         dev_flow->act_flags = action_flags;
13985         if (wks->skip_matcher_reg)
13986                 return 0;
13987         /* Register matcher. */
13988         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
13989                                     matcher.mask.size);
13990         matcher.priority = mlx5_get_matcher_priority(dev, attr,
13991                                                      matcher.priority,
13992                                                      dev_flow->external);
13993         /**
13994          * When creating meter drop flow in drop table, using original
13995          * 5-tuple match, the matcher priority should be lower than
13996          * mtr_id matcher.
13997          */
13998         if (attr->group == MLX5_FLOW_TABLE_LEVEL_METER &&
13999             dev_flow->dv.table_id == MLX5_MTR_TABLE_ID_DROP &&
14000             matcher.priority <= MLX5_REG_BITS)
14001                 matcher.priority += MLX5_REG_BITS;
14002         /* reserved field no needs to be set to 0 here. */
14003         tbl_key.is_fdb = attr->transfer;
14004         tbl_key.is_egress = attr->egress;
14005         tbl_key.level = dev_flow->dv.group;
14006         tbl_key.id = dev_flow->dv.table_id;
14007         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow,
14008                                      tunnel, attr->group, error))
14009                 return -rte_errno;
14010         return 0;
14011 }
14012
14013 /**
14014  * Set hash RX queue by hash fields (see enum ibv_rx_hash_fields)
14015  * and tunnel.
14016  *
14017  * @param[in, out] action
14018  *   Shred RSS action holding hash RX queue objects.
14019  * @param[in] hash_fields
14020  *   Defines combination of packet fields to participate in RX hash.
14021  * @param[in] tunnel
14022  *   Tunnel type
14023  * @param[in] hrxq_idx
14024  *   Hash RX queue index to set.
14025  *
14026  * @return
14027  *   0 on success, otherwise negative errno value.
14028  */
14029 static int
14030 __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action,
14031                               const uint64_t hash_fields,
14032                               uint32_t hrxq_idx)
14033 {
14034         uint32_t *hrxqs = action->hrxq;
14035
14036         switch (hash_fields & ~IBV_RX_HASH_INNER) {
14037         case MLX5_RSS_HASH_IPV4:
14038                 /* fall-through. */
14039         case MLX5_RSS_HASH_IPV4_DST_ONLY:
14040                 /* fall-through. */
14041         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
14042                 hrxqs[0] = hrxq_idx;
14043                 return 0;
14044         case MLX5_RSS_HASH_IPV4_TCP:
14045                 /* fall-through. */
14046         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
14047                 /* fall-through. */
14048         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
14049                 hrxqs[1] = hrxq_idx;
14050                 return 0;
14051         case MLX5_RSS_HASH_IPV4_UDP:
14052                 /* fall-through. */
14053         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
14054                 /* fall-through. */
14055         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
14056                 hrxqs[2] = hrxq_idx;
14057                 return 0;
14058         case MLX5_RSS_HASH_IPV6:
14059                 /* fall-through. */
14060         case MLX5_RSS_HASH_IPV6_DST_ONLY:
14061                 /* fall-through. */
14062         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
14063                 hrxqs[3] = hrxq_idx;
14064                 return 0;
14065         case MLX5_RSS_HASH_IPV6_TCP:
14066                 /* fall-through. */
14067         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
14068                 /* fall-through. */
14069         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
14070                 hrxqs[4] = hrxq_idx;
14071                 return 0;
14072         case MLX5_RSS_HASH_IPV6_UDP:
14073                 /* fall-through. */
14074         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
14075                 /* fall-through. */
14076         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
14077                 hrxqs[5] = hrxq_idx;
14078                 return 0;
14079         case MLX5_RSS_HASH_NONE:
14080                 hrxqs[6] = hrxq_idx;
14081                 return 0;
14082         case MLX5_RSS_HASH_IPV4_ESP:
14083                 hrxqs[7] = hrxq_idx;
14084                 return 0;
14085         case MLX5_RSS_HASH_IPV6_ESP:
14086                 hrxqs[8] = hrxq_idx;
14087                 return 0;
14088         case MLX5_RSS_HASH_ESP_SPI:
14089                 hrxqs[9] = hrxq_idx;
14090                 return 0;
14091         default:
14092                 return -1;
14093         }
14094 }
14095
14096 /**
14097  * Look up for hash RX queue by hash fields (see enum ibv_rx_hash_fields)
14098  * and tunnel.
14099  *
14100  * @param[in] dev
14101  *   Pointer to the Ethernet device structure.
14102  * @param[in] idx
14103  *   Shared RSS action ID holding hash RX queue objects.
14104  * @param[in] hash_fields
14105  *   Defines combination of packet fields to participate in RX hash.
14106  * @param[in] tunnel
14107  *   Tunnel type
14108  *
14109  * @return
14110  *   Valid hash RX queue index, otherwise 0.
14111  */
14112 uint32_t
14113 flow_dv_action_rss_hrxq_lookup(struct rte_eth_dev *dev, uint32_t idx,
14114                                const uint64_t hash_fields)
14115 {
14116         struct mlx5_priv *priv = dev->data->dev_private;
14117         struct mlx5_shared_action_rss *shared_rss =
14118             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
14119         const uint32_t *hrxqs = shared_rss->hrxq;
14120
14121         switch (hash_fields & ~IBV_RX_HASH_INNER) {
14122         case MLX5_RSS_HASH_IPV4:
14123                 /* fall-through. */
14124         case MLX5_RSS_HASH_IPV4_DST_ONLY:
14125                 /* fall-through. */
14126         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
14127                 return hrxqs[0];
14128         case MLX5_RSS_HASH_IPV4_TCP:
14129                 /* fall-through. */
14130         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
14131                 /* fall-through. */
14132         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
14133                 return hrxqs[1];
14134         case MLX5_RSS_HASH_IPV4_UDP:
14135                 /* fall-through. */
14136         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
14137                 /* fall-through. */
14138         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
14139                 return hrxqs[2];
14140         case MLX5_RSS_HASH_IPV6:
14141                 /* fall-through. */
14142         case MLX5_RSS_HASH_IPV6_DST_ONLY:
14143                 /* fall-through. */
14144         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
14145                 return hrxqs[3];
14146         case MLX5_RSS_HASH_IPV6_TCP:
14147                 /* fall-through. */
14148         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
14149                 /* fall-through. */
14150         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
14151                 return hrxqs[4];
14152         case MLX5_RSS_HASH_IPV6_UDP:
14153                 /* fall-through. */
14154         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
14155                 /* fall-through. */
14156         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
14157                 return hrxqs[5];
14158         case MLX5_RSS_HASH_NONE:
14159                 return hrxqs[6];
14160         case MLX5_RSS_HASH_IPV4_ESP:
14161                 return hrxqs[7];
14162         case MLX5_RSS_HASH_IPV6_ESP:
14163                 return hrxqs[8];
14164         case MLX5_RSS_HASH_ESP_SPI:
14165                 return hrxqs[9];
14166         default:
14167                 return 0;
14168         }
14169
14170 }
14171
14172 /**
14173  * Apply the flow to the NIC, lock free,
14174  * (mutex should be acquired by caller).
14175  *
14176  * @param[in] dev
14177  *   Pointer to the Ethernet device structure.
14178  * @param[in, out] flow
14179  *   Pointer to flow structure.
14180  * @param[out] error
14181  *   Pointer to error structure.
14182  *
14183  * @return
14184  *   0 on success, a negative errno value otherwise and rte_errno is set.
14185  */
14186 static int
14187 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
14188               struct rte_flow_error *error)
14189 {
14190         struct mlx5_flow_dv_workspace *dv;
14191         struct mlx5_flow_handle *dh;
14192         struct mlx5_flow_handle_dv *dv_h;
14193         struct mlx5_flow *dev_flow;
14194         struct mlx5_priv *priv = dev->data->dev_private;
14195         uint32_t handle_idx;
14196         int n;
14197         int err;
14198         int idx;
14199         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
14200         struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc;
14201         uint8_t misc_mask;
14202
14203         MLX5_ASSERT(wks);
14204         for (idx = wks->flow_idx - 1; idx >= 0; idx--) {
14205                 dev_flow = &wks->flows[idx];
14206                 dv = &dev_flow->dv;
14207                 dh = dev_flow->handle;
14208                 dv_h = &dh->dvh;
14209                 n = dv->actions_n;
14210                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
14211                         if (dv->transfer) {
14212                                 MLX5_ASSERT(priv->sh->dr_drop_action);
14213                                 dv->actions[n++] = priv->sh->dr_drop_action;
14214                         } else {
14215 #ifdef HAVE_MLX5DV_DR
14216                                 /* DR supports drop action placeholder. */
14217                                 MLX5_ASSERT(priv->sh->dr_drop_action);
14218                                 dv->actions[n++] = dv->group ?
14219                                         priv->sh->dr_drop_action :
14220                                         priv->root_drop_action;
14221 #else
14222                                 /* For DV we use the explicit drop queue. */
14223                                 MLX5_ASSERT(priv->drop_queue.hrxq);
14224                                 dv->actions[n++] =
14225                                                 priv->drop_queue.hrxq->action;
14226 #endif
14227                         }
14228                 } else if ((dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
14229                            !dv_h->rix_sample && !dv_h->rix_dest_array)) {
14230                         struct mlx5_hrxq *hrxq;
14231                         uint32_t hrxq_idx;
14232
14233                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow, rss_desc,
14234                                                     &hrxq_idx);
14235                         if (!hrxq) {
14236                                 rte_flow_error_set
14237                                         (error, rte_errno,
14238                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14239                                          "cannot get hash queue");
14240                                 goto error;
14241                         }
14242                         dh->rix_hrxq = hrxq_idx;
14243                         dv->actions[n++] = hrxq->action;
14244                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
14245                         struct mlx5_hrxq *hrxq = NULL;
14246                         uint32_t hrxq_idx;
14247
14248                         hrxq_idx = flow_dv_action_rss_hrxq_lookup(dev,
14249                                                 rss_desc->shared_rss,
14250                                                 dev_flow->hash_fields);
14251                         if (hrxq_idx)
14252                                 hrxq = mlx5_ipool_get
14253                                         (priv->sh->ipool[MLX5_IPOOL_HRXQ],
14254                                          hrxq_idx);
14255                         if (!hrxq) {
14256                                 rte_flow_error_set
14257                                         (error, rte_errno,
14258                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14259                                          "cannot get hash queue");
14260                                 goto error;
14261                         }
14262                         dh->rix_srss = rss_desc->shared_rss;
14263                         dv->actions[n++] = hrxq->action;
14264                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
14265                         if (!priv->sh->default_miss_action) {
14266                                 rte_flow_error_set
14267                                         (error, rte_errno,
14268                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14269                                          "default miss action not be created.");
14270                                 goto error;
14271                         }
14272                         dv->actions[n++] = priv->sh->default_miss_action;
14273                 }
14274                 misc_mask = flow_dv_matcher_enable(dv->value.buf);
14275                 __flow_dv_adjust_buf_size(&dv->value.size, misc_mask);
14276                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
14277                                                (void *)&dv->value, n,
14278                                                dv->actions, &dh->drv_flow);
14279                 if (err) {
14280                         rte_flow_error_set
14281                                 (error, errno,
14282                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14283                                 NULL,
14284                                 (!priv->sh->config.allow_duplicate_pattern &&
14285                                 errno == EEXIST) ?
14286                                 "duplicating pattern is not allowed" :
14287                                 "hardware refuses to create flow");
14288                         goto error;
14289                 }
14290                 if (priv->vmwa_context &&
14291                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
14292                         /*
14293                          * The rule contains the VLAN pattern.
14294                          * For VF we are going to create VLAN
14295                          * interface to make hypervisor set correct
14296                          * e-Switch vport context.
14297                          */
14298                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
14299                 }
14300         }
14301         return 0;
14302 error:
14303         err = rte_errno; /* Save rte_errno before cleanup. */
14304         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
14305                        handle_idx, dh, next) {
14306                 /* hrxq is union, don't clear it if the flag is not set. */
14307                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && dh->rix_hrxq) {
14308                         mlx5_hrxq_release(dev, dh->rix_hrxq);
14309                         dh->rix_hrxq = 0;
14310                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
14311                         dh->rix_srss = 0;
14312                 }
14313                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
14314                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
14315         }
14316         rte_errno = err; /* Restore rte_errno. */
14317         return -rte_errno;
14318 }
14319
14320 void
14321 flow_dv_matcher_remove_cb(void *tool_ctx __rte_unused,
14322                           struct mlx5_list_entry *entry)
14323 {
14324         struct mlx5_flow_dv_matcher *resource = container_of(entry,
14325                                                              typeof(*resource),
14326                                                              entry);
14327
14328         claim_zero(mlx5_flow_os_destroy_flow_matcher(resource->matcher_object));
14329         mlx5_free(resource);
14330 }
14331
14332 /**
14333  * Release the flow matcher.
14334  *
14335  * @param dev
14336  *   Pointer to Ethernet device.
14337  * @param port_id
14338  *   Index to port ID action resource.
14339  *
14340  * @return
14341  *   1 while a reference on it exists, 0 when freed.
14342  */
14343 static int
14344 flow_dv_matcher_release(struct rte_eth_dev *dev,
14345                         struct mlx5_flow_handle *handle)
14346 {
14347         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
14348         struct mlx5_flow_tbl_data_entry *tbl = container_of(matcher->tbl,
14349                                                             typeof(*tbl), tbl);
14350         int ret;
14351
14352         MLX5_ASSERT(matcher->matcher_object);
14353         ret = mlx5_list_unregister(tbl->matchers, &matcher->entry);
14354         flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
14355         return ret;
14356 }
14357
14358 void
14359 flow_dv_encap_decap_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
14360 {
14361         struct mlx5_dev_ctx_shared *sh = tool_ctx;
14362         struct mlx5_flow_dv_encap_decap_resource *res =
14363                                        container_of(entry, typeof(*res), entry);
14364
14365         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
14366         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
14367 }
14368
14369 /**
14370  * Release an encap/decap resource.
14371  *
14372  * @param dev
14373  *   Pointer to Ethernet device.
14374  * @param encap_decap_idx
14375  *   Index of encap decap resource.
14376  *
14377  * @return
14378  *   1 while a reference on it exists, 0 when freed.
14379  */
14380 static int
14381 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
14382                                      uint32_t encap_decap_idx)
14383 {
14384         struct mlx5_priv *priv = dev->data->dev_private;
14385         struct mlx5_flow_dv_encap_decap_resource *resource;
14386
14387         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
14388                                   encap_decap_idx);
14389         if (!resource)
14390                 return 0;
14391         MLX5_ASSERT(resource->action);
14392         return mlx5_hlist_unregister(priv->sh->encaps_decaps, &resource->entry);
14393 }
14394
14395 /**
14396  * Release an jump to table action resource.
14397  *
14398  * @param dev
14399  *   Pointer to Ethernet device.
14400  * @param rix_jump
14401  *   Index to the jump action resource.
14402  *
14403  * @return
14404  *   1 while a reference on it exists, 0 when freed.
14405  */
14406 static int
14407 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
14408                                   uint32_t rix_jump)
14409 {
14410         struct mlx5_priv *priv = dev->data->dev_private;
14411         struct mlx5_flow_tbl_data_entry *tbl_data;
14412
14413         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
14414                                   rix_jump);
14415         if (!tbl_data)
14416                 return 0;
14417         return flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl_data->tbl);
14418 }
14419
14420 void
14421 flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
14422 {
14423         struct mlx5_flow_dv_modify_hdr_resource *res =
14424                 container_of(entry, typeof(*res), entry);
14425         struct mlx5_dev_ctx_shared *sh = tool_ctx;
14426
14427         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
14428         mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
14429 }
14430
14431 /**
14432  * Release a modify-header resource.
14433  *
14434  * @param dev
14435  *   Pointer to Ethernet device.
14436  * @param handle
14437  *   Pointer to mlx5_flow_handle.
14438  *
14439  * @return
14440  *   1 while a reference on it exists, 0 when freed.
14441  */
14442 static int
14443 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
14444                                     struct mlx5_flow_handle *handle)
14445 {
14446         struct mlx5_priv *priv = dev->data->dev_private;
14447         struct mlx5_flow_dv_modify_hdr_resource *entry = handle->dvh.modify_hdr;
14448
14449         MLX5_ASSERT(entry->action);
14450         return mlx5_hlist_unregister(priv->sh->modify_cmds, &entry->entry);
14451 }
14452
14453 void
14454 flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
14455 {
14456         struct mlx5_dev_ctx_shared *sh = tool_ctx;
14457         struct mlx5_flow_dv_port_id_action_resource *resource =
14458                                   container_of(entry, typeof(*resource), entry);
14459
14460         claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
14461         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
14462 }
14463
14464 /**
14465  * Release port ID action resource.
14466  *
14467  * @param dev
14468  *   Pointer to Ethernet device.
14469  * @param handle
14470  *   Pointer to mlx5_flow_handle.
14471  *
14472  * @return
14473  *   1 while a reference on it exists, 0 when freed.
14474  */
14475 static int
14476 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
14477                                         uint32_t port_id)
14478 {
14479         struct mlx5_priv *priv = dev->data->dev_private;
14480         struct mlx5_flow_dv_port_id_action_resource *resource;
14481
14482         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
14483         if (!resource)
14484                 return 0;
14485         MLX5_ASSERT(resource->action);
14486         return mlx5_list_unregister(priv->sh->port_id_action_list,
14487                                     &resource->entry);
14488 }
14489
14490 /**
14491  * Release shared RSS action resource.
14492  *
14493  * @param dev
14494  *   Pointer to Ethernet device.
14495  * @param srss
14496  *   Shared RSS action index.
14497  */
14498 static void
14499 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
14500 {
14501         struct mlx5_priv *priv = dev->data->dev_private;
14502         struct mlx5_shared_action_rss *shared_rss;
14503
14504         shared_rss = mlx5_ipool_get
14505                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], srss);
14506         __atomic_sub_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
14507 }
14508
14509 void
14510 flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
14511 {
14512         struct mlx5_dev_ctx_shared *sh = tool_ctx;
14513         struct mlx5_flow_dv_push_vlan_action_resource *resource =
14514                         container_of(entry, typeof(*resource), entry);
14515
14516         claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
14517         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
14518 }
14519
14520 /**
14521  * Release push vlan action resource.
14522  *
14523  * @param dev
14524  *   Pointer to Ethernet device.
14525  * @param handle
14526  *   Pointer to mlx5_flow_handle.
14527  *
14528  * @return
14529  *   1 while a reference on it exists, 0 when freed.
14530  */
14531 static int
14532 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
14533                                           struct mlx5_flow_handle *handle)
14534 {
14535         struct mlx5_priv *priv = dev->data->dev_private;
14536         struct mlx5_flow_dv_push_vlan_action_resource *resource;
14537         uint32_t idx = handle->dvh.rix_push_vlan;
14538
14539         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
14540         if (!resource)
14541                 return 0;
14542         MLX5_ASSERT(resource->action);
14543         return mlx5_list_unregister(priv->sh->push_vlan_action_list,
14544                                     &resource->entry);
14545 }
14546
14547 /**
14548  * Release the fate resource.
14549  *
14550  * @param dev
14551  *   Pointer to Ethernet device.
14552  * @param handle
14553  *   Pointer to mlx5_flow_handle.
14554  */
14555 static void
14556 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
14557                                struct mlx5_flow_handle *handle)
14558 {
14559         if (!handle->rix_fate)
14560                 return;
14561         switch (handle->fate_action) {
14562         case MLX5_FLOW_FATE_QUEUE:
14563                 if (!handle->dvh.rix_sample && !handle->dvh.rix_dest_array)
14564                         mlx5_hrxq_release(dev, handle->rix_hrxq);
14565                 break;
14566         case MLX5_FLOW_FATE_JUMP:
14567                 flow_dv_jump_tbl_resource_release(dev, handle->rix_jump);
14568                 break;
14569         case MLX5_FLOW_FATE_PORT_ID:
14570                 flow_dv_port_id_action_resource_release(dev,
14571                                 handle->rix_port_id_action);
14572                 break;
14573         default:
14574                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
14575                 break;
14576         }
14577         handle->rix_fate = 0;
14578 }
14579
14580 void
14581 flow_dv_sample_remove_cb(void *tool_ctx __rte_unused,
14582                          struct mlx5_list_entry *entry)
14583 {
14584         struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
14585                                                               typeof(*resource),
14586                                                               entry);
14587         struct rte_eth_dev *dev = resource->dev;
14588         struct mlx5_priv *priv = dev->data->dev_private;
14589
14590         if (resource->verbs_action)
14591                 claim_zero(mlx5_flow_os_destroy_flow_action
14592                                                       (resource->verbs_action));
14593         if (resource->normal_path_tbl)
14594                 flow_dv_tbl_resource_release(MLX5_SH(dev),
14595                                              resource->normal_path_tbl);
14596         flow_dv_sample_sub_actions_release(dev, &resource->sample_idx);
14597         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
14598         DRV_LOG(DEBUG, "sample resource %p: removed", (void *)resource);
14599 }
14600
14601 /**
14602  * Release an sample resource.
14603  *
14604  * @param dev
14605  *   Pointer to Ethernet device.
14606  * @param handle
14607  *   Pointer to mlx5_flow_handle.
14608  *
14609  * @return
14610  *   1 while a reference on it exists, 0 when freed.
14611  */
14612 static int
14613 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
14614                                      struct mlx5_flow_handle *handle)
14615 {
14616         struct mlx5_priv *priv = dev->data->dev_private;
14617         struct mlx5_flow_dv_sample_resource *resource;
14618
14619         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
14620                                   handle->dvh.rix_sample);
14621         if (!resource)
14622                 return 0;
14623         MLX5_ASSERT(resource->verbs_action);
14624         return mlx5_list_unregister(priv->sh->sample_action_list,
14625                                     &resource->entry);
14626 }
14627
14628 void
14629 flow_dv_dest_array_remove_cb(void *tool_ctx __rte_unused,
14630                              struct mlx5_list_entry *entry)
14631 {
14632         struct mlx5_flow_dv_dest_array_resource *resource =
14633                         container_of(entry, typeof(*resource), entry);
14634         struct rte_eth_dev *dev = resource->dev;
14635         struct mlx5_priv *priv = dev->data->dev_private;
14636         uint32_t i = 0;
14637
14638         MLX5_ASSERT(resource->action);
14639         if (resource->action)
14640                 claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
14641         for (; i < resource->num_of_dest; i++)
14642                 flow_dv_sample_sub_actions_release(dev,
14643                                                    &resource->sample_idx[i]);
14644         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
14645         DRV_LOG(DEBUG, "destination array resource %p: removed",
14646                 (void *)resource);
14647 }
14648
14649 /**
14650  * Release an destination array resource.
14651  *
14652  * @param dev
14653  *   Pointer to Ethernet device.
14654  * @param handle
14655  *   Pointer to mlx5_flow_handle.
14656  *
14657  * @return
14658  *   1 while a reference on it exists, 0 when freed.
14659  */
14660 static int
14661 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
14662                                     struct mlx5_flow_handle *handle)
14663 {
14664         struct mlx5_priv *priv = dev->data->dev_private;
14665         struct mlx5_flow_dv_dest_array_resource *resource;
14666
14667         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
14668                                   handle->dvh.rix_dest_array);
14669         if (!resource)
14670                 return 0;
14671         MLX5_ASSERT(resource->action);
14672         return mlx5_list_unregister(priv->sh->dest_array_list,
14673                                     &resource->entry);
14674 }
14675
14676 static void
14677 flow_dv_geneve_tlv_option_resource_release(struct rte_eth_dev *dev)
14678 {
14679         struct mlx5_priv *priv = dev->data->dev_private;
14680         struct mlx5_dev_ctx_shared *sh = priv->sh;
14681         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
14682                                 sh->geneve_tlv_option_resource;
14683         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
14684         if (geneve_opt_resource) {
14685                 if (!(__atomic_sub_fetch(&geneve_opt_resource->refcnt, 1,
14686                                          __ATOMIC_RELAXED))) {
14687                         claim_zero(mlx5_devx_cmd_destroy
14688                                         (geneve_opt_resource->obj));
14689                         mlx5_free(sh->geneve_tlv_option_resource);
14690                         sh->geneve_tlv_option_resource = NULL;
14691                 }
14692         }
14693         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
14694 }
14695
14696 /**
14697  * Remove the flow from the NIC but keeps it in memory.
14698  * Lock free, (mutex should be acquired by caller).
14699  *
14700  * @param[in] dev
14701  *   Pointer to Ethernet device.
14702  * @param[in, out] flow
14703  *   Pointer to flow structure.
14704  */
14705 static void
14706 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
14707 {
14708         struct mlx5_flow_handle *dh;
14709         uint32_t handle_idx;
14710         struct mlx5_priv *priv = dev->data->dev_private;
14711
14712         if (!flow)
14713                 return;
14714         handle_idx = flow->dev_handles;
14715         while (handle_idx) {
14716                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
14717                                     handle_idx);
14718                 if (!dh)
14719                         return;
14720                 if (dh->drv_flow) {
14721                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
14722                         dh->drv_flow = NULL;
14723                 }
14724                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE)
14725                         flow_dv_fate_resource_release(dev, dh);
14726                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
14727                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
14728                 handle_idx = dh->next.next;
14729         }
14730 }
14731
14732 /**
14733  * Remove the flow from the NIC and the memory.
14734  * Lock free, (mutex should be acquired by caller).
14735  *
14736  * @param[in] dev
14737  *   Pointer to the Ethernet device structure.
14738  * @param[in, out] flow
14739  *   Pointer to flow structure.
14740  */
14741 static void
14742 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
14743 {
14744         struct mlx5_flow_handle *dev_handle;
14745         struct mlx5_priv *priv = dev->data->dev_private;
14746         struct mlx5_flow_meter_info *fm = NULL;
14747         uint32_t srss = 0;
14748
14749         if (!flow)
14750                 return;
14751         flow_dv_remove(dev, flow);
14752         if (flow->counter) {
14753                 flow_dv_counter_free(dev, flow->counter);
14754                 flow->counter = 0;
14755         }
14756         if (flow->meter) {
14757                 fm = flow_dv_meter_find_by_idx(priv, flow->meter);
14758                 if (fm)
14759                         mlx5_flow_meter_detach(priv, fm);
14760                 flow->meter = 0;
14761         }
14762         /* Keep the current age handling by default. */
14763         if (flow->indirect_type == MLX5_INDIRECT_ACTION_TYPE_CT && flow->ct)
14764                 flow_dv_aso_ct_release(dev, flow->ct, NULL);
14765         else if (flow->age)
14766                 flow_dv_aso_age_release(dev, flow->age);
14767         if (flow->geneve_tlv_option) {
14768                 flow_dv_geneve_tlv_option_resource_release(dev);
14769                 flow->geneve_tlv_option = 0;
14770         }
14771         while (flow->dev_handles) {
14772                 uint32_t tmp_idx = flow->dev_handles;
14773
14774                 dev_handle = mlx5_ipool_get(priv->sh->ipool
14775                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
14776                 if (!dev_handle)
14777                         return;
14778                 flow->dev_handles = dev_handle->next.next;
14779                 while (dev_handle->flex_item) {
14780                         int index = rte_bsf32(dev_handle->flex_item);
14781
14782                         mlx5_flex_release_index(dev, index);
14783                         dev_handle->flex_item &= ~(uint8_t)RTE_BIT32(index);
14784                 }
14785                 if (dev_handle->dvh.matcher)
14786                         flow_dv_matcher_release(dev, dev_handle);
14787                 if (dev_handle->dvh.rix_sample)
14788                         flow_dv_sample_resource_release(dev, dev_handle);
14789                 if (dev_handle->dvh.rix_dest_array)
14790                         flow_dv_dest_array_resource_release(dev, dev_handle);
14791                 if (dev_handle->dvh.rix_encap_decap)
14792                         flow_dv_encap_decap_resource_release(dev,
14793                                 dev_handle->dvh.rix_encap_decap);
14794                 if (dev_handle->dvh.modify_hdr)
14795                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
14796                 if (dev_handle->dvh.rix_push_vlan)
14797                         flow_dv_push_vlan_action_resource_release(dev,
14798                                                                   dev_handle);
14799                 if (dev_handle->dvh.rix_tag)
14800                         flow_dv_tag_release(dev,
14801                                             dev_handle->dvh.rix_tag);
14802                 if (dev_handle->fate_action != MLX5_FLOW_FATE_SHARED_RSS)
14803                         flow_dv_fate_resource_release(dev, dev_handle);
14804                 else if (!srss)
14805                         srss = dev_handle->rix_srss;
14806                 if (fm && dev_handle->is_meter_flow_id &&
14807                     dev_handle->split_flow_id)
14808                         mlx5_ipool_free(fm->flow_ipool,
14809                                         dev_handle->split_flow_id);
14810                 else if (dev_handle->split_flow_id &&
14811                     !dev_handle->is_meter_flow_id)
14812                         mlx5_ipool_free(priv->sh->ipool
14813                                         [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
14814                                         dev_handle->split_flow_id);
14815                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
14816                            tmp_idx);
14817         }
14818         if (srss)
14819                 flow_dv_shared_rss_action_release(dev, srss);
14820 }
14821
14822 /**
14823  * Release array of hash RX queue objects.
14824  * Helper function.
14825  *
14826  * @param[in] dev
14827  *   Pointer to the Ethernet device structure.
14828  * @param[in, out] hrxqs
14829  *   Array of hash RX queue objects.
14830  *
14831  * @return
14832  *   Total number of references to hash RX queue objects in *hrxqs* array
14833  *   after this operation.
14834  */
14835 static int
14836 __flow_dv_hrxqs_release(struct rte_eth_dev *dev,
14837                         uint32_t (*hrxqs)[MLX5_RSS_HASH_FIELDS_LEN])
14838 {
14839         size_t i;
14840         int remaining = 0;
14841
14842         for (i = 0; i < RTE_DIM(*hrxqs); i++) {
14843                 int ret = mlx5_hrxq_release(dev, (*hrxqs)[i]);
14844
14845                 if (!ret)
14846                         (*hrxqs)[i] = 0;
14847                 remaining += ret;
14848         }
14849         return remaining;
14850 }
14851
14852 /**
14853  * Release all hash RX queue objects representing shared RSS action.
14854  *
14855  * @param[in] dev
14856  *   Pointer to the Ethernet device structure.
14857  * @param[in, out] action
14858  *   Shared RSS action to remove hash RX queue objects from.
14859  *
14860  * @return
14861  *   Total number of references to hash RX queue objects stored in *action*
14862  *   after this operation.
14863  *   Expected to be 0 if no external references held.
14864  */
14865 static int
14866 __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev,
14867                                  struct mlx5_shared_action_rss *shared_rss)
14868 {
14869         return __flow_dv_hrxqs_release(dev, &shared_rss->hrxq);
14870 }
14871
14872 /**
14873  * Adjust L3/L4 hash value of pre-created shared RSS hrxq according to
14874  * user input.
14875  *
14876  * Only one hash value is available for one L3+L4 combination:
14877  * for example:
14878  * MLX5_RSS_HASH_IPV4, MLX5_RSS_HASH_IPV4_SRC_ONLY, and
14879  * MLX5_RSS_HASH_IPV4_DST_ONLY are mutually exclusive so they can share
14880  * same slot in mlx5_rss_hash_fields.
14881  *
14882  * @param[in] orig_rss_types
14883  *   RSS type as provided in shared RSS action.
14884  * @param[in, out] hash_field
14885  *   hash_field variable needed to be adjusted.
14886  *
14887  * @return
14888  *   void
14889  */
14890 void
14891 flow_dv_action_rss_l34_hash_adjust(uint64_t orig_rss_types,
14892                                    uint64_t *hash_field)
14893 {
14894         uint64_t rss_types = rte_eth_rss_hf_refine(orig_rss_types);
14895
14896         switch (*hash_field & ~IBV_RX_HASH_INNER) {
14897         case MLX5_RSS_HASH_IPV4:
14898                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
14899                         *hash_field &= ~MLX5_RSS_HASH_IPV4;
14900                         if (rss_types & RTE_ETH_RSS_L3_DST_ONLY)
14901                                 *hash_field |= IBV_RX_HASH_DST_IPV4;
14902                         else if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY)
14903                                 *hash_field |= IBV_RX_HASH_SRC_IPV4;
14904                         else
14905                                 *hash_field |= MLX5_RSS_HASH_IPV4;
14906                 }
14907                 return;
14908         case MLX5_RSS_HASH_IPV6:
14909                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
14910                         *hash_field &= ~MLX5_RSS_HASH_IPV6;
14911                         if (rss_types & RTE_ETH_RSS_L3_DST_ONLY)
14912                                 *hash_field |= IBV_RX_HASH_DST_IPV6;
14913                         else if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY)
14914                                 *hash_field |= IBV_RX_HASH_SRC_IPV6;
14915                         else
14916                                 *hash_field |= MLX5_RSS_HASH_IPV6;
14917                 }
14918                 return;
14919         case MLX5_RSS_HASH_IPV4_UDP:
14920                 /* fall-through. */
14921         case MLX5_RSS_HASH_IPV6_UDP:
14922                 if (rss_types & RTE_ETH_RSS_UDP) {
14923                         *hash_field &= ~MLX5_UDP_IBV_RX_HASH;
14924                         if (rss_types & RTE_ETH_RSS_L4_DST_ONLY)
14925                                 *hash_field |= IBV_RX_HASH_DST_PORT_UDP;
14926                         else if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY)
14927                                 *hash_field |= IBV_RX_HASH_SRC_PORT_UDP;
14928                         else
14929                                 *hash_field |= MLX5_UDP_IBV_RX_HASH;
14930                 }
14931                 return;
14932         case MLX5_RSS_HASH_IPV4_TCP:
14933                 /* fall-through. */
14934         case MLX5_RSS_HASH_IPV6_TCP:
14935                 if (rss_types & RTE_ETH_RSS_TCP) {
14936                         *hash_field &= ~MLX5_TCP_IBV_RX_HASH;
14937                         if (rss_types & RTE_ETH_RSS_L4_DST_ONLY)
14938                                 *hash_field |= IBV_RX_HASH_DST_PORT_TCP;
14939                         else if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY)
14940                                 *hash_field |= IBV_RX_HASH_SRC_PORT_TCP;
14941                         else
14942                                 *hash_field |= MLX5_TCP_IBV_RX_HASH;
14943                 }
14944                 return;
14945         default:
14946                 return;
14947         }
14948 }
14949
14950 /**
14951  * Setup shared RSS action.
14952  * Prepare set of hash RX queue objects sufficient to handle all valid
14953  * hash_fields combinations (see enum ibv_rx_hash_fields).
14954  *
14955  * @param[in] dev
14956  *   Pointer to the Ethernet device structure.
14957  * @param[in] action_idx
14958  *   Shared RSS action ipool index.
14959  * @param[in, out] action
14960  *   Partially initialized shared RSS action.
14961  * @param[out] error
14962  *   Perform verbose error reporting if not NULL. Initialized in case of
14963  *   error only.
14964  *
14965  * @return
14966  *   0 on success, otherwise negative errno value.
14967  */
14968 static int
14969 __flow_dv_action_rss_setup(struct rte_eth_dev *dev,
14970                            uint32_t action_idx,
14971                            struct mlx5_shared_action_rss *shared_rss,
14972                            struct rte_flow_error *error)
14973 {
14974         struct mlx5_priv *priv = dev->data->dev_private;
14975         struct mlx5_flow_rss_desc rss_desc = { 0 };
14976         size_t i;
14977         int err;
14978
14979         shared_rss->ind_tbl = mlx5_ind_table_obj_new
14980                               (dev, shared_rss->origin.queue,
14981                                shared_rss->origin.queue_num,
14982                                true,
14983                                !!dev->data->dev_started);
14984         if (!shared_rss->ind_tbl)
14985                 return rte_flow_error_set(error, rte_errno,
14986                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14987                                           "cannot setup indirection table");
14988         memcpy(rss_desc.key, shared_rss->origin.key, MLX5_RSS_HASH_KEY_LEN);
14989         rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN;
14990         rss_desc.const_q = shared_rss->origin.queue;
14991         rss_desc.queue_num = shared_rss->origin.queue_num;
14992         /* Set non-zero value to indicate a shared RSS. */
14993         rss_desc.shared_rss = action_idx;
14994         rss_desc.ind_tbl = shared_rss->ind_tbl;
14995         if (priv->sh->config.dv_flow_en == 2)
14996                 rss_desc.hws_flags = MLX5DR_ACTION_FLAG_HWS_RX;
14997         for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) {
14998                 struct mlx5_hrxq *hrxq;
14999                 uint64_t hash_fields = mlx5_rss_hash_fields[i];
15000                 int tunnel = 0;
15001
15002                 flow_dv_action_rss_l34_hash_adjust(shared_rss->origin.types,
15003                                                    &hash_fields);
15004                 if (shared_rss->origin.level > 1) {
15005                         hash_fields |= IBV_RX_HASH_INNER;
15006                         tunnel = 1;
15007                 }
15008                 rss_desc.tunnel = tunnel;
15009                 rss_desc.hash_fields = hash_fields;
15010                 hrxq = mlx5_hrxq_get(dev, &rss_desc);
15011                 if (!hrxq) {
15012                         rte_flow_error_set
15013                                 (error, rte_errno,
15014                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
15015                                  "cannot get hash queue");
15016                         goto error_hrxq_new;
15017                 }
15018                 err = __flow_dv_action_rss_hrxq_set
15019                         (shared_rss, hash_fields, hrxq->idx);
15020                 MLX5_ASSERT(!err);
15021         }
15022         return 0;
15023 error_hrxq_new:
15024         err = rte_errno;
15025         __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
15026         if (!mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true))
15027                 shared_rss->ind_tbl = NULL;
15028         rte_errno = err;
15029         return -rte_errno;
15030 }
15031
15032 /**
15033  * Create shared RSS action.
15034  *
15035  * @param[in] dev
15036  *   Pointer to the Ethernet device structure.
15037  * @param[in] conf
15038  *   Shared action configuration.
15039  * @param[in] rss
15040  *   RSS action specification used to create shared action.
15041  * @param[out] error
15042  *   Perform verbose error reporting if not NULL. Initialized in case of
15043  *   error only.
15044  *
15045  * @return
15046  *   A valid shared action ID in case of success, 0 otherwise and
15047  *   rte_errno is set.
15048  */
15049 static uint32_t
15050 __flow_dv_action_rss_create(struct rte_eth_dev *dev,
15051                             const struct rte_flow_indir_action_conf *conf,
15052                             const struct rte_flow_action_rss *rss,
15053                             struct rte_flow_error *error)
15054 {
15055         struct mlx5_priv *priv = dev->data->dev_private;
15056         struct mlx5_shared_action_rss *shared_rss = NULL;
15057         struct rte_flow_action_rss *origin;
15058         const uint8_t *rss_key;
15059         uint32_t idx;
15060
15061         RTE_SET_USED(conf);
15062         shared_rss = mlx5_ipool_zmalloc
15063                          (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], &idx);
15064         if (!shared_rss) {
15065                 rte_flow_error_set(error, ENOMEM,
15066                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
15067                                    "cannot allocate resource memory");
15068                 goto error_rss_init;
15069         }
15070         if (idx > (1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET)) {
15071                 rte_flow_error_set(error, E2BIG,
15072                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
15073                                    "rss action number out of range");
15074                 goto error_rss_init;
15075         }
15076         origin = &shared_rss->origin;
15077         origin->func = rss->func;
15078         origin->level = rss->level;
15079         /* RSS type 0 indicates default RSS type (RTE_ETH_RSS_IP). */
15080         origin->types = !rss->types ? RTE_ETH_RSS_IP : rss->types;
15081         /* NULL RSS key indicates default RSS key. */
15082         rss_key = !rss->key ? rss_hash_default_key : rss->key;
15083         memcpy(shared_rss->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
15084         origin->key = &shared_rss->key[0];
15085         origin->key_len = MLX5_RSS_HASH_KEY_LEN;
15086         origin->queue = rss->queue;
15087         origin->queue_num = rss->queue_num;
15088         if (__flow_dv_action_rss_setup(dev, idx, shared_rss, error))
15089                 goto error_rss_init;
15090         /* Update queue with indirect table queue memoyr. */
15091         origin->queue = shared_rss->ind_tbl->queues;
15092         rte_spinlock_init(&shared_rss->action_rss_sl);
15093         __atomic_add_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
15094         rte_spinlock_lock(&priv->shared_act_sl);
15095         ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
15096                      &priv->rss_shared_actions, idx, shared_rss, next);
15097         rte_spinlock_unlock(&priv->shared_act_sl);
15098         return idx;
15099 error_rss_init:
15100         if (shared_rss) {
15101                 if (shared_rss->ind_tbl)
15102                         mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl,
15103                                                    !!dev->data->dev_started);
15104                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
15105                                 idx);
15106         }
15107         return 0;
15108 }
15109
15110 /**
15111  * Destroy the shared RSS action.
15112  * Release related hash RX queue objects.
15113  *
15114  * @param[in] dev
15115  *   Pointer to the Ethernet device structure.
15116  * @param[in] idx
15117  *   The shared RSS action object ID to be removed.
15118  * @param[out] error
15119  *   Perform verbose error reporting if not NULL. Initialized in case of
15120  *   error only.
15121  *
15122  * @return
15123  *   0 on success, otherwise negative errno value.
15124  */
15125 static int
15126 __flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx,
15127                              struct rte_flow_error *error)
15128 {
15129         struct mlx5_priv *priv = dev->data->dev_private;
15130         struct mlx5_shared_action_rss *shared_rss =
15131             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
15132         uint32_t old_refcnt = 1;
15133         int remaining;
15134
15135         if (!shared_rss)
15136                 return rte_flow_error_set(error, EINVAL,
15137                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
15138                                           "invalid shared action");
15139         if (!__atomic_compare_exchange_n(&shared_rss->refcnt, &old_refcnt,
15140                                          0, 0, __ATOMIC_ACQUIRE,
15141                                          __ATOMIC_RELAXED))
15142                 return rte_flow_error_set(error, EBUSY,
15143                                           RTE_FLOW_ERROR_TYPE_ACTION,
15144                                           NULL,
15145                                           "shared rss has references");
15146         remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
15147         if (remaining)
15148                 return rte_flow_error_set(error, EBUSY,
15149                                           RTE_FLOW_ERROR_TYPE_ACTION,
15150                                           NULL,
15151                                           "shared rss hrxq has references");
15152         remaining = mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl,
15153                                                !!dev->data->dev_started);
15154         if (remaining)
15155                 return rte_flow_error_set(error, EBUSY,
15156                                           RTE_FLOW_ERROR_TYPE_ACTION,
15157                                           NULL,
15158                                           "shared rss indirection table has"
15159                                           " references");
15160         rte_spinlock_lock(&priv->shared_act_sl);
15161         ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
15162                      &priv->rss_shared_actions, idx, shared_rss, next);
15163         rte_spinlock_unlock(&priv->shared_act_sl);
15164         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
15165                         idx);
15166         return 0;
15167 }
15168
15169 /**
15170  * Create indirect action, lock free,
15171  * (mutex should be acquired by caller).
15172  * Dispatcher for action type specific call.
15173  *
15174  * @param[in] dev
15175  *   Pointer to the Ethernet device structure.
15176  * @param[in] conf
15177  *   Shared action configuration.
15178  * @param[in] action
15179  *   Action specification used to create indirect action.
15180  * @param[out] error
15181  *   Perform verbose error reporting if not NULL. Initialized in case of
15182  *   error only.
15183  *
15184  * @return
15185  *   A valid shared action handle in case of success, NULL otherwise and
15186  *   rte_errno is set.
15187  */
15188 struct rte_flow_action_handle *
15189 flow_dv_action_create(struct rte_eth_dev *dev,
15190                       const struct rte_flow_indir_action_conf *conf,
15191                       const struct rte_flow_action *action,
15192                       struct rte_flow_error *err)
15193 {
15194         struct mlx5_priv *priv = dev->data->dev_private;
15195         uint32_t age_idx = 0;
15196         uint32_t idx = 0;
15197         uint32_t ret = 0;
15198
15199         switch (action->type) {
15200         case RTE_FLOW_ACTION_TYPE_RSS:
15201                 ret = __flow_dv_action_rss_create(dev, conf, action->conf, err);
15202                 idx = (MLX5_INDIRECT_ACTION_TYPE_RSS <<
15203                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
15204                 break;
15205         case RTE_FLOW_ACTION_TYPE_AGE:
15206                 age_idx = flow_dv_aso_age_alloc(dev, err);
15207                 if (!age_idx) {
15208                         ret = -rte_errno;
15209                         break;
15210                 }
15211                 idx = (MLX5_INDIRECT_ACTION_TYPE_AGE <<
15212                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | age_idx;
15213                 flow_dv_aso_age_params_init(dev, age_idx,
15214                                         ((const struct rte_flow_action_age *)
15215                                                 action->conf)->context ?
15216                                         ((const struct rte_flow_action_age *)
15217                                                 action->conf)->context :
15218                                         (void *)(uintptr_t)idx,
15219                                         ((const struct rte_flow_action_age *)
15220                                                 action->conf)->timeout);
15221                 ret = age_idx;
15222                 break;
15223         case RTE_FLOW_ACTION_TYPE_COUNT:
15224                 ret = flow_dv_translate_create_counter(dev, NULL, NULL, NULL);
15225                 idx = (MLX5_INDIRECT_ACTION_TYPE_COUNT <<
15226                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
15227                 break;
15228         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
15229                 ret = flow_dv_translate_create_conntrack(dev, action->conf,
15230                                                          err);
15231                 idx = MLX5_INDIRECT_ACT_CT_GEN_IDX(PORT_ID(priv), ret);
15232                 break;
15233         default:
15234                 rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
15235                                    NULL, "action type not supported");
15236                 break;
15237         }
15238         return ret ? (struct rte_flow_action_handle *)(uintptr_t)idx : NULL;
15239 }
15240
15241 /**
15242  * Destroy the indirect action.
15243  * Release action related resources on the NIC and the memory.
15244  * Lock free, (mutex should be acquired by caller).
15245  * Dispatcher for action type specific call.
15246  *
15247  * @param[in] dev
15248  *   Pointer to the Ethernet device structure.
15249  * @param[in] handle
15250  *   The indirect action object handle to be removed.
15251  * @param[out] error
15252  *   Perform verbose error reporting if not NULL. Initialized in case of
15253  *   error only.
15254  *
15255  * @return
15256  *   0 on success, otherwise negative errno value.
15257  */
15258 int
15259 flow_dv_action_destroy(struct rte_eth_dev *dev,
15260                        struct rte_flow_action_handle *handle,
15261                        struct rte_flow_error *error)
15262 {
15263         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
15264         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
15265         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
15266         struct mlx5_flow_counter *cnt;
15267         uint32_t no_flow_refcnt = 1;
15268         int ret;
15269
15270         switch (type) {
15271         case MLX5_INDIRECT_ACTION_TYPE_RSS:
15272                 return __flow_dv_action_rss_release(dev, idx, error);
15273         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
15274                 cnt = flow_dv_counter_get_by_idx(dev, idx, NULL);
15275                 if (!__atomic_compare_exchange_n(&cnt->shared_info.refcnt,
15276                                                  &no_flow_refcnt, 1, false,
15277                                                  __ATOMIC_ACQUIRE,
15278                                                  __ATOMIC_RELAXED))
15279                         return rte_flow_error_set(error, EBUSY,
15280                                                   RTE_FLOW_ERROR_TYPE_ACTION,
15281                                                   NULL,
15282                                                   "Indirect count action has references");
15283                 flow_dv_counter_free(dev, idx);
15284                 return 0;
15285         case MLX5_INDIRECT_ACTION_TYPE_AGE:
15286                 ret = flow_dv_aso_age_release(dev, idx);
15287                 if (ret)
15288                         /*
15289                          * In this case, the last flow has a reference will
15290                          * actually release the age action.
15291                          */
15292                         DRV_LOG(DEBUG, "Indirect age action %" PRIu32 " was"
15293                                 " released with references %d.", idx, ret);
15294                 return 0;
15295         case MLX5_INDIRECT_ACTION_TYPE_CT:
15296                 ret = flow_dv_aso_ct_release(dev, idx, error);
15297                 if (ret < 0)
15298                         return ret;
15299                 if (ret > 0)
15300                         DRV_LOG(DEBUG, "Connection tracking object %u still "
15301                                 "has references %d.", idx, ret);
15302                 return 0;
15303         default:
15304                 return rte_flow_error_set(error, ENOTSUP,
15305                                           RTE_FLOW_ERROR_TYPE_ACTION,
15306                                           NULL,
15307                                           "action type not supported");
15308         }
15309 }
15310
15311 /**
15312  * Updates in place shared RSS action configuration.
15313  *
15314  * @param[in] dev
15315  *   Pointer to the Ethernet device structure.
15316  * @param[in] idx
15317  *   The shared RSS action object ID to be updated.
15318  * @param[in] action_conf
15319  *   RSS action specification used to modify *shared_rss*.
15320  * @param[out] error
15321  *   Perform verbose error reporting if not NULL. Initialized in case of
15322  *   error only.
15323  *
15324  * @return
15325  *   0 on success, otherwise negative errno value.
15326  * @note: currently only support update of RSS queues.
15327  */
15328 static int
15329 __flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx,
15330                             const struct rte_flow_action_rss *action_conf,
15331                             struct rte_flow_error *error)
15332 {
15333         struct mlx5_priv *priv = dev->data->dev_private;
15334         struct mlx5_shared_action_rss *shared_rss =
15335             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
15336         int ret = 0;
15337         void *queue = NULL;
15338         void *queue_i = NULL;
15339         uint32_t queue_size = action_conf->queue_num * sizeof(uint16_t);
15340         bool dev_started = !!dev->data->dev_started;
15341
15342         if (!shared_rss)
15343                 return rte_flow_error_set(error, EINVAL,
15344                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
15345                                           "invalid shared action to update");
15346         if (priv->obj_ops.ind_table_modify == NULL)
15347                 return rte_flow_error_set(error, ENOTSUP,
15348                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
15349                                           "cannot modify indirection table");
15350         queue = mlx5_malloc(MLX5_MEM_ZERO,
15351                             RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
15352                             0, SOCKET_ID_ANY);
15353         if (!queue)
15354                 return rte_flow_error_set(error, ENOMEM,
15355                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15356                                           NULL,
15357                                           "cannot allocate resource memory");
15358         memcpy(queue, action_conf->queue, queue_size);
15359         MLX5_ASSERT(shared_rss->ind_tbl);
15360         rte_spinlock_lock(&shared_rss->action_rss_sl);
15361         queue_i = shared_rss->ind_tbl->queues;
15362         ret = mlx5_ind_table_obj_modify(dev, shared_rss->ind_tbl,
15363                                         queue, action_conf->queue_num,
15364                                         true /* standalone */,
15365                                         dev_started /* ref_new_qs */,
15366                                         dev_started /* deref_old_qs */);
15367         if (ret) {
15368                 ret = rte_flow_error_set(error, rte_errno,
15369                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
15370                                           "cannot update indirection table");
15371         } else {
15372                 /* Restore the queue to indirect table internal queue. */
15373                 memcpy(queue_i, queue, queue_size);
15374                 shared_rss->ind_tbl->queues = queue_i;
15375                 shared_rss->origin.queue_num = action_conf->queue_num;
15376         }
15377         mlx5_free(queue);
15378         rte_spinlock_unlock(&shared_rss->action_rss_sl);
15379         return ret;
15380 }
15381
15382 /*
15383  * Updates in place conntrack context or direction.
15384  * Context update should be synchronized.
15385  *
15386  * @param[in] dev
15387  *   Pointer to the Ethernet device structure.
15388  * @param[in] idx
15389  *   The conntrack object ID to be updated.
15390  * @param[in] update
15391  *   Pointer to the structure of information to update.
15392  * @param[out] error
15393  *   Perform verbose error reporting if not NULL. Initialized in case of
15394  *   error only.
15395  *
15396  * @return
15397  *   0 on success, otherwise negative errno value.
15398  */
15399 static int
15400 __flow_dv_action_ct_update(struct rte_eth_dev *dev, uint32_t idx,
15401                            const struct rte_flow_modify_conntrack *update,
15402                            struct rte_flow_error *error)
15403 {
15404         struct mlx5_priv *priv = dev->data->dev_private;
15405         struct mlx5_aso_ct_action *ct;
15406         const struct rte_flow_action_conntrack *new_prf;
15407         int ret = 0;
15408         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
15409         uint32_t dev_idx;
15410
15411         if (PORT_ID(priv) != owner)
15412                 return rte_flow_error_set(error, EACCES,
15413                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15414                                           NULL,
15415                                           "CT object owned by another port");
15416         dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
15417         ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
15418         if (!ct->refcnt)
15419                 return rte_flow_error_set(error, ENOMEM,
15420                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15421                                           NULL,
15422                                           "CT object is inactive");
15423         new_prf = &update->new_ct;
15424         if (update->direction)
15425                 ct->is_original = !!new_prf->is_original_dir;
15426         if (update->state) {
15427                 /* Only validate the profile when it needs to be updated. */
15428                 ret = mlx5_validate_action_ct(dev, new_prf, error);
15429                 if (ret)
15430                         return ret;
15431                 ret = mlx5_aso_ct_update_by_wqe(priv->sh, ct, new_prf);
15432                 if (ret)
15433                         return rte_flow_error_set(error, EIO,
15434                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15435                                         NULL,
15436                                         "Failed to send CT context update WQE");
15437                 /* Block until ready or a failure. */
15438                 ret = mlx5_aso_ct_available(priv->sh, ct);
15439                 if (ret)
15440                         rte_flow_error_set(error, rte_errno,
15441                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15442                                            NULL,
15443                                            "Timeout to get the CT update");
15444         }
15445         return ret;
15446 }
15447
15448 /**
15449  * Updates in place shared action configuration, lock free,
15450  * (mutex should be acquired by caller).
15451  *
15452  * @param[in] dev
15453  *   Pointer to the Ethernet device structure.
15454  * @param[in] handle
15455  *   The indirect action object handle to be updated.
15456  * @param[in] update
15457  *   Action specification used to modify the action pointed by *handle*.
15458  *   *update* could be of same type with the action pointed by the *handle*
15459  *   handle argument, or some other structures like a wrapper, depending on
15460  *   the indirect action type.
15461  * @param[out] error
15462  *   Perform verbose error reporting if not NULL. Initialized in case of
15463  *   error only.
15464  *
15465  * @return
15466  *   0 on success, otherwise negative errno value.
15467  */
15468 int
15469 flow_dv_action_update(struct rte_eth_dev *dev,
15470                         struct rte_flow_action_handle *handle,
15471                         const void *update,
15472                         struct rte_flow_error *err)
15473 {
15474         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
15475         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
15476         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
15477         const void *action_conf;
15478
15479         switch (type) {
15480         case MLX5_INDIRECT_ACTION_TYPE_RSS:
15481                 action_conf = ((const struct rte_flow_action *)update)->conf;
15482                 return __flow_dv_action_rss_update(dev, idx, action_conf, err);
15483         case MLX5_INDIRECT_ACTION_TYPE_CT:
15484                 return __flow_dv_action_ct_update(dev, idx, update, err);
15485         default:
15486                 return rte_flow_error_set(err, ENOTSUP,
15487                                           RTE_FLOW_ERROR_TYPE_ACTION,
15488                                           NULL,
15489                                           "action type update not supported");
15490         }
15491 }
15492
15493 /**
15494  * Destroy the meter sub policy table rules.
15495  * Lock free, (mutex should be acquired by caller).
15496  *
15497  * @param[in] dev
15498  *   Pointer to Ethernet device.
15499  * @param[in] sub_policy
15500  *   Pointer to meter sub policy table.
15501  */
15502 static void
15503 __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
15504                              struct mlx5_flow_meter_sub_policy *sub_policy)
15505 {
15506         struct mlx5_priv *priv = dev->data->dev_private;
15507         struct mlx5_flow_tbl_data_entry *tbl;
15508         struct mlx5_flow_meter_policy *policy = sub_policy->main_policy;
15509         struct mlx5_flow_meter_info *next_fm;
15510         struct mlx5_sub_policy_color_rule *color_rule;
15511         void *tmp;
15512         uint32_t i;
15513
15514         for (i = 0; i < RTE_COLORS; i++) {
15515                 next_fm = NULL;
15516                 if (i <= RTE_COLOR_YELLOW && policy &&
15517                     policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR)
15518                         next_fm = mlx5_flow_meter_find(priv,
15519                                         policy->act_cnt[i].next_mtr_id, NULL);
15520                 RTE_TAILQ_FOREACH_SAFE(color_rule, &sub_policy->color_rules[i],
15521                                    next_port, tmp) {
15522                         claim_zero(mlx5_flow_os_destroy_flow(color_rule->rule));
15523                         tbl = container_of(color_rule->matcher->tbl,
15524                                            typeof(*tbl), tbl);
15525                         mlx5_list_unregister(tbl->matchers,
15526                                              &color_rule->matcher->entry);
15527                         TAILQ_REMOVE(&sub_policy->color_rules[i],
15528                                      color_rule, next_port);
15529                         mlx5_free(color_rule);
15530                         if (next_fm)
15531                                 mlx5_flow_meter_detach(priv, next_fm);
15532                 }
15533         }
15534         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
15535                 if (sub_policy->rix_hrxq[i]) {
15536                         if (policy && !policy->is_hierarchy)
15537                                 mlx5_hrxq_release(dev, sub_policy->rix_hrxq[i]);
15538                         sub_policy->rix_hrxq[i] = 0;
15539                 }
15540                 if (sub_policy->jump_tbl[i]) {
15541                         flow_dv_tbl_resource_release(MLX5_SH(dev),
15542                                                      sub_policy->jump_tbl[i]);
15543                         sub_policy->jump_tbl[i] = NULL;
15544                 }
15545         }
15546         if (sub_policy->tbl_rsc) {
15547                 flow_dv_tbl_resource_release(MLX5_SH(dev),
15548                                              sub_policy->tbl_rsc);
15549                 sub_policy->tbl_rsc = NULL;
15550         }
15551 }
15552
15553 /**
15554  * Destroy policy rules, lock free,
15555  * (mutex should be acquired by caller).
15556  * Dispatcher for action type specific call.
15557  *
15558  * @param[in] dev
15559  *   Pointer to the Ethernet device structure.
15560  * @param[in] mtr_policy
15561  *   Meter policy struct.
15562  */
15563 static void
15564 flow_dv_destroy_policy_rules(struct rte_eth_dev *dev,
15565                              struct mlx5_flow_meter_policy *mtr_policy)
15566 {
15567         uint32_t i, j;
15568         struct mlx5_flow_meter_sub_policy *sub_policy;
15569         uint16_t sub_policy_num;
15570
15571         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15572                 sub_policy_num = (mtr_policy->sub_policy_num >>
15573                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
15574                         MLX5_MTR_SUB_POLICY_NUM_MASK;
15575                 for (j = 0; j < sub_policy_num; j++) {
15576                         sub_policy = mtr_policy->sub_policys[i][j];
15577                         if (sub_policy)
15578                                 __flow_dv_destroy_sub_policy_rules(dev,
15579                                                                    sub_policy);
15580                 }
15581         }
15582 }
15583
15584 /**
15585  * Destroy policy action, lock free,
15586  * (mutex should be acquired by caller).
15587  * Dispatcher for action type specific call.
15588  *
15589  * @param[in] dev
15590  *   Pointer to the Ethernet device structure.
15591  * @param[in] mtr_policy
15592  *   Meter policy struct.
15593  */
15594 static void
15595 flow_dv_destroy_mtr_policy_acts(struct rte_eth_dev *dev,
15596                       struct mlx5_flow_meter_policy *mtr_policy)
15597 {
15598         struct rte_flow_action *rss_action;
15599         struct mlx5_flow_handle dev_handle;
15600         uint32_t i, j;
15601
15602         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
15603                 if (mtr_policy->act_cnt[i].rix_mark) {
15604                         flow_dv_tag_release(dev,
15605                                 mtr_policy->act_cnt[i].rix_mark);
15606                         mtr_policy->act_cnt[i].rix_mark = 0;
15607                 }
15608                 if (mtr_policy->act_cnt[i].modify_hdr) {
15609                         dev_handle.dvh.modify_hdr =
15610                                 mtr_policy->act_cnt[i].modify_hdr;
15611                         flow_dv_modify_hdr_resource_release(dev, &dev_handle);
15612                 }
15613                 switch (mtr_policy->act_cnt[i].fate_action) {
15614                 case MLX5_FLOW_FATE_SHARED_RSS:
15615                         rss_action = mtr_policy->act_cnt[i].rss;
15616                         mlx5_free(rss_action);
15617                         break;
15618                 case MLX5_FLOW_FATE_PORT_ID:
15619                         if (mtr_policy->act_cnt[i].rix_port_id_action) {
15620                                 flow_dv_port_id_action_resource_release(dev,
15621                                 mtr_policy->act_cnt[i].rix_port_id_action);
15622                                 mtr_policy->act_cnt[i].rix_port_id_action = 0;
15623                         }
15624                         break;
15625                 case MLX5_FLOW_FATE_DROP:
15626                 case MLX5_FLOW_FATE_JUMP:
15627                         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
15628                                 mtr_policy->act_cnt[i].dr_jump_action[j] =
15629                                                 NULL;
15630                         break;
15631                 default:
15632                         /*Queue action do nothing*/
15633                         break;
15634                 }
15635         }
15636         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
15637                 mtr_policy->dr_drop_action[j] = NULL;
15638 }
15639
15640 /**
15641  * Create yellow action for color aware meter.
15642  *
15643  * @param[in] dev
15644  *   Pointer to the Ethernet device structure.
15645  * @param[in] fm
15646  *   Meter information table.
15647  * @param[out] error
15648  *   Perform verbose error reporting if not NULL. Initialized in case of
15649  *   error only.
15650  *
15651  * @return
15652  *   0 on success, a negative errno value otherwise and rte_errno is set.
15653  */
15654 static int
15655 __flow_dv_create_mtr_yellow_action(struct rte_eth_dev *dev,
15656                                    struct mlx5_flow_meter_info *fm,
15657                                    struct rte_mtr_error *error)
15658 {
15659 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
15660         struct mlx5_priv *priv = dev->data->dev_private;
15661         struct rte_flow_error flow_err;
15662         struct mlx5_aso_mtr *aso_mtr;
15663         struct mlx5_aso_mtr_pool *pool;
15664         uint8_t reg_id;
15665
15666         aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
15667         pool = container_of(aso_mtr, struct mlx5_aso_mtr_pool, mtrs[aso_mtr->offset]);
15668         reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &flow_err);
15669         fm->meter_action_y =
15670                 mlx5_glue->dv_create_flow_action_aso(priv->sh->rx_domain,
15671                                                      pool->devx_obj->obj,
15672                                                      aso_mtr->offset,
15673                                                      (1 << MLX5_FLOW_COLOR_YELLOW),
15674                                                      reg_id - REG_C_0);
15675 #else
15676         RTE_SET_USED(dev);
15677 #endif
15678         if (!fm->meter_action_y) {
15679                 return -rte_mtr_error_set(error, EINVAL, RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
15680                                           "Fail to create yellow meter action.");
15681         }
15682         return 0;
15683 }
15684
15685 /**
15686  * Create policy action per domain, lock free,
15687  * (mutex should be acquired by caller).
15688  * Dispatcher for action type specific call.
15689  *
15690  * @param[in] dev
15691  *   Pointer to the Ethernet device structure.
15692  * @param[in] mtr_policy
15693  *   Meter policy struct.
15694  * @param[in] action
15695  *   Action specification used to create meter actions.
15696  * @param[out] error
15697  *   Perform verbose error reporting if not NULL. Initialized in case of
15698  *   error only.
15699  *
15700  * @return
15701  *   0 on success, otherwise negative errno value.
15702  */
15703 static int
15704 __flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev,
15705                         struct mlx5_flow_meter_policy *mtr_policy,
15706                         const struct rte_flow_action *actions[RTE_COLORS],
15707                         enum mlx5_meter_domain domain,
15708                         struct rte_mtr_error *error)
15709 {
15710         struct mlx5_priv *priv = dev->data->dev_private;
15711         struct rte_flow_error flow_err;
15712         const struct rte_flow_action *act;
15713         uint64_t action_flags;
15714         struct mlx5_flow_handle dh;
15715         struct mlx5_flow dev_flow;
15716         struct mlx5_flow_dv_port_id_action_resource port_id_action;
15717         int i, ret;
15718         uint8_t egress, transfer;
15719         struct mlx5_meter_policy_action_container *act_cnt = NULL;
15720         union {
15721                 struct mlx5_flow_dv_modify_hdr_resource res;
15722                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
15723                             sizeof(struct mlx5_modification_cmd) *
15724                             (MLX5_MAX_MODIFY_NUM + 1)];
15725         } mhdr_dummy;
15726         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
15727
15728         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
15729         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
15730         memset(&dh, 0, sizeof(struct mlx5_flow_handle));
15731         memset(&dev_flow, 0, sizeof(struct mlx5_flow));
15732         memset(&port_id_action, 0,
15733                sizeof(struct mlx5_flow_dv_port_id_action_resource));
15734         memset(mhdr_res, 0, sizeof(*mhdr_res));
15735         mhdr_res->ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
15736                                        (egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
15737                                         MLX5DV_FLOW_TABLE_TYPE_NIC_RX);
15738         dev_flow.handle = &dh;
15739         dev_flow.dv.port_id_action = &port_id_action;
15740         dev_flow.external = true;
15741         for (i = 0; i < RTE_COLORS; i++) {
15742                 if (i < MLX5_MTR_RTE_COLORS)
15743                         act_cnt = &mtr_policy->act_cnt[i];
15744                 /* Skip the color policy actions creation. */
15745                 if ((i == RTE_COLOR_YELLOW && mtr_policy->skip_y) ||
15746                     (i == RTE_COLOR_GREEN && mtr_policy->skip_g))
15747                         continue;
15748                 action_flags = 0;
15749                 for (act = actions[i];
15750                      act && act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
15751                         switch (act->type) {
15752                         case RTE_FLOW_ACTION_TYPE_MARK:
15753                         {
15754                                 uint32_t tag_be = mlx5_flow_mark_set
15755                                         (((const struct rte_flow_action_mark *)
15756                                         (act->conf))->id);
15757
15758                                 if (i >= MLX5_MTR_RTE_COLORS)
15759                                         return -rte_mtr_error_set(error,
15760                                           ENOTSUP,
15761                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15762                                           NULL,
15763                                           "cannot create policy "
15764                                           "mark action for this color");
15765                                 if (flow_dv_tag_resource_register(dev, tag_be,
15766                                                   &dev_flow, &flow_err))
15767                                         return -rte_mtr_error_set(error,
15768                                         ENOTSUP,
15769                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15770                                         NULL,
15771                                         "cannot setup policy mark action");
15772                                 MLX5_ASSERT(dev_flow.dv.tag_resource);
15773                                 act_cnt->rix_mark =
15774                                         dev_flow.handle->dvh.rix_tag;
15775                                 action_flags |= MLX5_FLOW_ACTION_MARK;
15776                                 mtr_policy->mark = 1;
15777                                 break;
15778                         }
15779                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
15780                                 if (i >= MLX5_MTR_RTE_COLORS)
15781                                         return -rte_mtr_error_set(error,
15782                                           ENOTSUP,
15783                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15784                                           NULL,
15785                                           "cannot create policy "
15786                                           "set tag action for this color");
15787                                 if (flow_dv_convert_action_set_tag
15788                                 (dev, mhdr_res,
15789                                 (const struct rte_flow_action_set_tag *)
15790                                 act->conf,  &flow_err))
15791                                         return -rte_mtr_error_set(error,
15792                                         ENOTSUP,
15793                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15794                                         NULL, "cannot convert policy "
15795                                         "set tag action");
15796                                 if (!mhdr_res->actions_num)
15797                                         return -rte_mtr_error_set(error,
15798                                         ENOTSUP,
15799                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15800                                         NULL, "cannot find policy "
15801                                         "set tag action");
15802                                 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
15803                                 break;
15804                         case RTE_FLOW_ACTION_TYPE_DROP:
15805                         {
15806                                 struct mlx5_flow_mtr_mng *mtrmng =
15807                                                 priv->sh->mtrmng;
15808                                 struct mlx5_flow_tbl_data_entry *tbl_data;
15809
15810                                 /*
15811                                  * Create the drop table with
15812                                  * METER DROP level.
15813                                  */
15814                                 if (!mtrmng->drop_tbl[domain]) {
15815                                         mtrmng->drop_tbl[domain] =
15816                                         flow_dv_tbl_resource_get(dev,
15817                                         MLX5_FLOW_TABLE_LEVEL_METER,
15818                                         egress, transfer, false, NULL, 0,
15819                                         0, MLX5_MTR_TABLE_ID_DROP, &flow_err);
15820                                         if (!mtrmng->drop_tbl[domain])
15821                                                 return -rte_mtr_error_set
15822                                         (error, ENOTSUP,
15823                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15824                                         NULL,
15825                                         "Failed to create meter drop table");
15826                                 }
15827                                 tbl_data = container_of
15828                                 (mtrmng->drop_tbl[domain],
15829                                 struct mlx5_flow_tbl_data_entry, tbl);
15830                                 if (i < MLX5_MTR_RTE_COLORS) {
15831                                         act_cnt->dr_jump_action[domain] =
15832                                                 tbl_data->jump.action;
15833                                         act_cnt->fate_action =
15834                                                 MLX5_FLOW_FATE_DROP;
15835                                 }
15836                                 if (i == RTE_COLOR_RED)
15837                                         mtr_policy->dr_drop_action[domain] =
15838                                                 tbl_data->jump.action;
15839                                 action_flags |= MLX5_FLOW_ACTION_DROP;
15840                                 break;
15841                         }
15842                         case RTE_FLOW_ACTION_TYPE_QUEUE:
15843                         {
15844                                 if (i >= MLX5_MTR_RTE_COLORS)
15845                                         return -rte_mtr_error_set(error,
15846                                         ENOTSUP,
15847                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15848                                         NULL, "cannot create policy "
15849                                         "fate queue for this color");
15850                                 act_cnt->queue =
15851                                 ((const struct rte_flow_action_queue *)
15852                                         (act->conf))->index;
15853                                 act_cnt->fate_action =
15854                                         MLX5_FLOW_FATE_QUEUE;
15855                                 dev_flow.handle->fate_action =
15856                                         MLX5_FLOW_FATE_QUEUE;
15857                                 mtr_policy->is_queue = 1;
15858                                 action_flags |= MLX5_FLOW_ACTION_QUEUE;
15859                                 break;
15860                         }
15861                         case RTE_FLOW_ACTION_TYPE_RSS:
15862                         {
15863                                 int rss_size;
15864
15865                                 if (i >= MLX5_MTR_RTE_COLORS)
15866                                         return -rte_mtr_error_set(error,
15867                                           ENOTSUP,
15868                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15869                                           NULL,
15870                                           "cannot create policy "
15871                                           "rss action for this color");
15872                                 /*
15873                                  * Save RSS conf into policy struct
15874                                  * for translate stage.
15875                                  */
15876                                 rss_size = (int)rte_flow_conv
15877                                         (RTE_FLOW_CONV_OP_ACTION,
15878                                         NULL, 0, act, &flow_err);
15879                                 if (rss_size <= 0)
15880                                         return -rte_mtr_error_set(error,
15881                                           ENOTSUP,
15882                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15883                                           NULL, "Get the wrong "
15884                                           "rss action struct size");
15885                                 act_cnt->rss = mlx5_malloc(MLX5_MEM_ZERO,
15886                                                 rss_size, 0, SOCKET_ID_ANY);
15887                                 if (!act_cnt->rss)
15888                                         return -rte_mtr_error_set(error,
15889                                           ENOTSUP,
15890                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15891                                           NULL,
15892                                           "Fail to malloc rss action memory");
15893                                 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTION,
15894                                         act_cnt->rss, rss_size,
15895                                         act, &flow_err);
15896                                 if (ret < 0)
15897                                         return -rte_mtr_error_set(error,
15898                                           ENOTSUP,
15899                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15900                                           NULL, "Fail to save "
15901                                           "rss action into policy struct");
15902                                 act_cnt->fate_action =
15903                                         MLX5_FLOW_FATE_SHARED_RSS;
15904                                 action_flags |= MLX5_FLOW_ACTION_RSS;
15905                                 break;
15906                         }
15907                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
15908                         case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
15909                         {
15910                                 struct mlx5_flow_dv_port_id_action_resource
15911                                         port_id_resource;
15912                                 uint32_t port_id = 0;
15913
15914                                 if (i >= MLX5_MTR_RTE_COLORS)
15915                                         return -rte_mtr_error_set(error,
15916                                         ENOTSUP,
15917                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15918                                         NULL, "cannot create policy "
15919                                         "port action for this color");
15920                                 memset(&port_id_resource, 0,
15921                                         sizeof(port_id_resource));
15922                                 if (flow_dv_translate_action_port_id(dev, act,
15923                                                 &port_id, &flow_err))
15924                                         return -rte_mtr_error_set(error,
15925                                         ENOTSUP,
15926                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15927                                         NULL, "cannot translate "
15928                                         "policy port action");
15929                                 port_id_resource.port_id = port_id;
15930                                 if (flow_dv_port_id_action_resource_register
15931                                         (dev, &port_id_resource,
15932                                         &dev_flow, &flow_err))
15933                                         return -rte_mtr_error_set(error,
15934                                         ENOTSUP,
15935                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15936                                         NULL, "cannot setup "
15937                                         "policy port action");
15938                                 act_cnt->rix_port_id_action =
15939                                         dev_flow.handle->rix_port_id_action;
15940                                 act_cnt->fate_action =
15941                                         MLX5_FLOW_FATE_PORT_ID;
15942                                 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
15943                                 break;
15944                         }
15945                         case RTE_FLOW_ACTION_TYPE_JUMP:
15946                         {
15947                                 uint32_t jump_group = 0;
15948                                 uint32_t table = 0;
15949                                 struct mlx5_flow_tbl_data_entry *tbl_data;
15950                                 struct flow_grp_info grp_info = {
15951                                         .external = !!dev_flow.external,
15952                                         .transfer = !!transfer,
15953                                         .fdb_def_rule = !!priv->fdb_def_rule,
15954                                         .std_tbl_fix = 0,
15955                                         .skip_scale = dev_flow.skip_scale &
15956                                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
15957                                 };
15958                                 struct mlx5_flow_meter_sub_policy *sub_policy =
15959                                         mtr_policy->sub_policys[domain][0];
15960
15961                                 if (i >= MLX5_MTR_RTE_COLORS)
15962                                         return -rte_mtr_error_set(error,
15963                                           ENOTSUP,
15964                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15965                                           NULL,
15966                                           "cannot create policy "
15967                                           "jump action for this color");
15968                                 jump_group =
15969                                 ((const struct rte_flow_action_jump *)
15970                                                         act->conf)->group;
15971                                 if (mlx5_flow_group_to_table(dev, NULL,
15972                                                        jump_group,
15973                                                        &table,
15974                                                        &grp_info, &flow_err))
15975                                         return -rte_mtr_error_set(error,
15976                                         ENOTSUP,
15977                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15978                                         NULL, "cannot setup "
15979                                         "policy jump action");
15980                                 sub_policy->jump_tbl[i] =
15981                                 flow_dv_tbl_resource_get(dev,
15982                                         table, egress,
15983                                         transfer,
15984                                         !!dev_flow.external,
15985                                         NULL, jump_group, 0,
15986                                         0, &flow_err);
15987                                 if
15988                                 (!sub_policy->jump_tbl[i])
15989                                         return  -rte_mtr_error_set(error,
15990                                         ENOTSUP,
15991                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15992                                         NULL, "cannot create jump action.");
15993                                 tbl_data = container_of
15994                                 (sub_policy->jump_tbl[i],
15995                                 struct mlx5_flow_tbl_data_entry, tbl);
15996                                 act_cnt->dr_jump_action[domain] =
15997                                         tbl_data->jump.action;
15998                                 act_cnt->fate_action =
15999                                         MLX5_FLOW_FATE_JUMP;
16000                                 action_flags |= MLX5_FLOW_ACTION_JUMP;
16001                                 break;
16002                         }
16003                         /*
16004                          * No need to check meter hierarchy for R colors
16005                          * here since it is done in the validation stage.
16006                          */
16007                         case RTE_FLOW_ACTION_TYPE_METER:
16008                         {
16009                                 const struct rte_flow_action_meter *mtr;
16010                                 struct mlx5_flow_meter_info *next_fm;
16011                                 struct mlx5_flow_meter_policy *next_policy;
16012                                 struct rte_flow_action tag_action;
16013                                 struct mlx5_rte_flow_action_set_tag set_tag;
16014                                 uint32_t next_mtr_idx = 0;
16015
16016                                 mtr = act->conf;
16017                                 next_fm = mlx5_flow_meter_find(priv,
16018                                                         mtr->mtr_id,
16019                                                         &next_mtr_idx);
16020                                 if (!next_fm)
16021                                         return -rte_mtr_error_set(error, EINVAL,
16022                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
16023                                                 "Fail to find next meter.");
16024                                 if (next_fm->def_policy)
16025                                         return -rte_mtr_error_set(error, EINVAL,
16026                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
16027                                 "Hierarchy only supports termination meter.");
16028                                 next_policy = mlx5_flow_meter_policy_find(dev,
16029                                                 next_fm->policy_id, NULL);
16030                                 MLX5_ASSERT(next_policy);
16031                                 if (next_fm->drop_cnt) {
16032                                         set_tag.id =
16033                                                 (enum modify_reg)
16034                                                 mlx5_flow_get_reg_id(dev,
16035                                                 MLX5_MTR_ID,
16036                                                 0,
16037                                                 (struct rte_flow_error *)error);
16038                                         set_tag.offset = (priv->mtr_reg_share ?
16039                                                 MLX5_MTR_COLOR_BITS : 0);
16040                                         set_tag.length = (priv->mtr_reg_share ?
16041                                                MLX5_MTR_IDLE_BITS_IN_COLOR_REG :
16042                                                MLX5_REG_BITS);
16043                                         set_tag.data = next_mtr_idx;
16044                                         tag_action.type =
16045                                                 (enum rte_flow_action_type)
16046                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
16047                                         tag_action.conf = &set_tag;
16048                                         if (flow_dv_convert_action_set_reg
16049                                                 (mhdr_res, &tag_action,
16050                                                 (struct rte_flow_error *)error))
16051                                                 return -rte_errno;
16052                                         action_flags |=
16053                                                 MLX5_FLOW_ACTION_SET_TAG;
16054                                 }
16055                                 if (i == RTE_COLOR_YELLOW && next_fm->color_aware &&
16056                                     !next_fm->meter_action_y)
16057                                         if (__flow_dv_create_mtr_yellow_action(dev, next_fm, error))
16058                                                 return -rte_errno;
16059                                 act_cnt->fate_action = MLX5_FLOW_FATE_MTR;
16060                                 act_cnt->next_mtr_id = next_fm->meter_id;
16061                                 act_cnt->next_sub_policy = NULL;
16062                                 mtr_policy->is_hierarchy = 1;
16063                                 mtr_policy->dev = next_policy->dev;
16064                                 if (next_policy->mark)
16065                                         mtr_policy->mark = 1;
16066                                 action_flags |=
16067                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
16068                                 break;
16069                         }
16070                         default:
16071                                 return -rte_mtr_error_set(error, ENOTSUP,
16072                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
16073                                           NULL, "action type not supported");
16074                         }
16075                         if (action_flags & MLX5_FLOW_ACTION_SET_TAG) {
16076                                 /* create modify action if needed. */
16077                                 dev_flow.dv.group = 1;
16078                                 if (flow_dv_modify_hdr_resource_register
16079                                         (dev, mhdr_res, &dev_flow, &flow_err))
16080                                         return -rte_mtr_error_set(error,
16081                                                 ENOTSUP,
16082                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
16083                                                 NULL, "cannot register policy "
16084                                                 "set tag action");
16085                                 act_cnt->modify_hdr =
16086                                         dev_flow.handle->dvh.modify_hdr;
16087                         }
16088                 }
16089         }
16090         return 0;
16091 }
16092
16093 /**
16094  * Create policy action per domain, lock free,
16095  * (mutex should be acquired by caller).
16096  * Dispatcher for action type specific call.
16097  *
16098  * @param[in] dev
16099  *   Pointer to the Ethernet device structure.
16100  * @param[in] mtr_policy
16101  *   Meter policy struct.
16102  * @param[in] action
16103  *   Action specification used to create meter actions.
16104  * @param[out] error
16105  *   Perform verbose error reporting if not NULL. Initialized in case of
16106  *   error only.
16107  *
16108  * @return
16109  *   0 on success, otherwise negative errno value.
16110  */
16111 static int
16112 flow_dv_create_mtr_policy_acts(struct rte_eth_dev *dev,
16113                       struct mlx5_flow_meter_policy *mtr_policy,
16114                       const struct rte_flow_action *actions[RTE_COLORS],
16115                       struct rte_mtr_error *error)
16116 {
16117         int ret, i;
16118         uint16_t sub_policy_num;
16119
16120         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16121                 sub_policy_num = (mtr_policy->sub_policy_num >>
16122                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
16123                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16124                 if (sub_policy_num) {
16125                         ret = __flow_dv_create_domain_policy_acts(dev,
16126                                 mtr_policy, actions,
16127                                 (enum mlx5_meter_domain)i, error);
16128                         /* Cleaning resource is done in the caller level. */
16129                         if (ret)
16130                                 return ret;
16131                 }
16132         }
16133         return 0;
16134 }
16135
16136 /**
16137  * Query a DV flow rule for its statistics via DevX.
16138  *
16139  * @param[in] dev
16140  *   Pointer to Ethernet device.
16141  * @param[in] cnt_idx
16142  *   Index to the flow counter.
16143  * @param[out] data
16144  *   Data retrieved by the query.
16145  * @param[out] error
16146  *   Perform verbose error reporting if not NULL.
16147  *
16148  * @return
16149  *   0 on success, a negative errno value otherwise and rte_errno is set.
16150  */
16151 static int
16152 flow_dv_query_count(struct rte_eth_dev *dev, uint32_t cnt_idx, void *data,
16153                     struct rte_flow_error *error)
16154 {
16155         struct mlx5_priv *priv = dev->data->dev_private;
16156         struct rte_flow_query_count *qc = data;
16157
16158         if (!priv->sh->cdev->config.devx)
16159                 return rte_flow_error_set(error, ENOTSUP,
16160                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16161                                           NULL,
16162                                           "counters are not supported");
16163         if (cnt_idx) {
16164                 uint64_t pkts, bytes;
16165                 struct mlx5_flow_counter *cnt;
16166                 int err = _flow_dv_query_count(dev, cnt_idx, &pkts, &bytes);
16167
16168                 if (err)
16169                         return rte_flow_error_set(error, -err,
16170                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16171                                         NULL, "cannot read counters");
16172                 cnt = flow_dv_counter_get_by_idx(dev, cnt_idx, NULL);
16173                 qc->hits_set = 1;
16174                 qc->bytes_set = 1;
16175                 qc->hits = pkts - cnt->hits;
16176                 qc->bytes = bytes - cnt->bytes;
16177                 if (qc->reset) {
16178                         cnt->hits = pkts;
16179                         cnt->bytes = bytes;
16180                 }
16181                 return 0;
16182         }
16183         return rte_flow_error_set(error, EINVAL,
16184                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16185                                   NULL,
16186                                   "counters are not available");
16187 }
16188
16189 int
16190 flow_dv_action_query(struct rte_eth_dev *dev,
16191                      const struct rte_flow_action_handle *handle, void *data,
16192                      struct rte_flow_error *error)
16193 {
16194         struct mlx5_age_param *age_param;
16195         struct rte_flow_query_age *resp;
16196         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
16197         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
16198         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
16199         struct mlx5_priv *priv = dev->data->dev_private;
16200         struct mlx5_aso_ct_action *ct;
16201         uint16_t owner;
16202         uint32_t dev_idx;
16203
16204         switch (type) {
16205         case MLX5_INDIRECT_ACTION_TYPE_AGE:
16206                 age_param = &flow_aso_age_get_by_idx(dev, idx)->age_params;
16207                 resp = data;
16208                 resp->aged = __atomic_load_n(&age_param->state,
16209                                               __ATOMIC_RELAXED) == AGE_TMOUT ?
16210                                                                           1 : 0;
16211                 resp->sec_since_last_hit_valid = !resp->aged;
16212                 if (resp->sec_since_last_hit_valid)
16213                         resp->sec_since_last_hit = __atomic_load_n
16214                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
16215                 return 0;
16216         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
16217                 return flow_dv_query_count(dev, idx, data, error);
16218         case MLX5_INDIRECT_ACTION_TYPE_CT:
16219                 owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
16220                 if (owner != PORT_ID(priv))
16221                         return rte_flow_error_set(error, EACCES,
16222                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16223                                         NULL,
16224                                         "CT object owned by another port");
16225                 dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
16226                 ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
16227                 MLX5_ASSERT(ct);
16228                 if (!ct->refcnt)
16229                         return rte_flow_error_set(error, EFAULT,
16230                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16231                                         NULL,
16232                                         "CT object is inactive");
16233                 ((struct rte_flow_action_conntrack *)data)->peer_port =
16234                                                         ct->peer;
16235                 ((struct rte_flow_action_conntrack *)data)->is_original_dir =
16236                                                         ct->is_original;
16237                 if (mlx5_aso_ct_query_by_wqe(priv->sh, ct, data))
16238                         return rte_flow_error_set(error, EIO,
16239                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16240                                         NULL,
16241                                         "Failed to query CT context");
16242                 return 0;
16243         default:
16244                 return rte_flow_error_set(error, ENOTSUP,
16245                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
16246                                           "action type query not supported");
16247         }
16248 }
16249
16250 /**
16251  * Query a flow rule AGE action for aging information.
16252  *
16253  * @param[in] dev
16254  *   Pointer to Ethernet device.
16255  * @param[in] flow
16256  *   Pointer to the sub flow.
16257  * @param[out] data
16258  *   data retrieved by the query.
16259  * @param[out] error
16260  *   Perform verbose error reporting if not NULL.
16261  *
16262  * @return
16263  *   0 on success, a negative errno value otherwise and rte_errno is set.
16264  */
16265 static int
16266 flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow,
16267                   void *data, struct rte_flow_error *error)
16268 {
16269         struct rte_flow_query_age *resp = data;
16270         struct mlx5_age_param *age_param;
16271
16272         if (flow->age) {
16273                 struct mlx5_aso_age_action *act =
16274                                      flow_aso_age_get_by_idx(dev, flow->age);
16275
16276                 age_param = &act->age_params;
16277         } else if (flow->counter) {
16278                 age_param = flow_dv_counter_idx_get_age(dev, flow->counter);
16279
16280                 if (!age_param || !age_param->timeout)
16281                         return rte_flow_error_set
16282                                         (error, EINVAL,
16283                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16284                                          NULL, "cannot read age data");
16285         } else {
16286                 return rte_flow_error_set(error, EINVAL,
16287                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
16288                                           NULL, "age data not available");
16289         }
16290         resp->aged = __atomic_load_n(&age_param->state, __ATOMIC_RELAXED) ==
16291                                      AGE_TMOUT ? 1 : 0;
16292         resp->sec_since_last_hit_valid = !resp->aged;
16293         if (resp->sec_since_last_hit_valid)
16294                 resp->sec_since_last_hit = __atomic_load_n
16295                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
16296         return 0;
16297 }
16298
16299 /**
16300  * Query a flow.
16301  *
16302  * @see rte_flow_query()
16303  * @see rte_flow_ops
16304  */
16305 static int
16306 flow_dv_query(struct rte_eth_dev *dev,
16307               struct rte_flow *flow __rte_unused,
16308               const struct rte_flow_action *actions __rte_unused,
16309               void *data __rte_unused,
16310               struct rte_flow_error *error __rte_unused)
16311 {
16312         int ret = -EINVAL;
16313
16314         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
16315                 switch (actions->type) {
16316                 case RTE_FLOW_ACTION_TYPE_VOID:
16317                         break;
16318                 case RTE_FLOW_ACTION_TYPE_COUNT:
16319                         ret = flow_dv_query_count(dev, flow->counter, data,
16320                                                   error);
16321                         break;
16322                 case RTE_FLOW_ACTION_TYPE_AGE:
16323                         ret = flow_dv_query_age(dev, flow, data, error);
16324                         break;
16325                 default:
16326                         return rte_flow_error_set(error, ENOTSUP,
16327                                                   RTE_FLOW_ERROR_TYPE_ACTION,
16328                                                   actions,
16329                                                   "action not supported");
16330                 }
16331         }
16332         return ret;
16333 }
16334
16335 /**
16336  * Destroy the meter table set.
16337  * Lock free, (mutex should be acquired by caller).
16338  *
16339  * @param[in] dev
16340  *   Pointer to Ethernet device.
16341  * @param[in] fm
16342  *   Meter information table.
16343  */
16344 static void
16345 flow_dv_destroy_mtr_tbls(struct rte_eth_dev *dev,
16346                         struct mlx5_flow_meter_info *fm)
16347 {
16348         struct mlx5_priv *priv = dev->data->dev_private;
16349         int i;
16350
16351         if (!fm || !priv->sh->config.dv_flow_en)
16352                 return;
16353         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16354                 if (fm->drop_rule[i]) {
16355                         claim_zero(mlx5_flow_os_destroy_flow(fm->drop_rule[i]));
16356                         fm->drop_rule[i] = NULL;
16357                 }
16358         }
16359 }
16360
16361 static void
16362 flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
16363 {
16364         struct mlx5_priv *priv = dev->data->dev_private;
16365         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
16366         struct mlx5_flow_tbl_data_entry *tbl;
16367         int i, j;
16368
16369         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16370                 if (mtrmng->def_rule[i]) {
16371                         claim_zero(mlx5_flow_os_destroy_flow
16372                                         (mtrmng->def_rule[i]));
16373                         mtrmng->def_rule[i] = NULL;
16374                 }
16375                 if (mtrmng->def_matcher[i]) {
16376                         tbl = container_of(mtrmng->def_matcher[i]->tbl,
16377                                 struct mlx5_flow_tbl_data_entry, tbl);
16378                         mlx5_list_unregister(tbl->matchers,
16379                                              &mtrmng->def_matcher[i]->entry);
16380                         mtrmng->def_matcher[i] = NULL;
16381                 }
16382                 for (j = 0; j < MLX5_REG_BITS; j++) {
16383                         if (mtrmng->drop_matcher[i][j]) {
16384                                 tbl =
16385                                 container_of(mtrmng->drop_matcher[i][j]->tbl,
16386                                              struct mlx5_flow_tbl_data_entry,
16387                                              tbl);
16388                                 mlx5_list_unregister(tbl->matchers,
16389                                             &mtrmng->drop_matcher[i][j]->entry);
16390                                 mtrmng->drop_matcher[i][j] = NULL;
16391                         }
16392                 }
16393                 if (mtrmng->drop_tbl[i]) {
16394                         flow_dv_tbl_resource_release(MLX5_SH(dev),
16395                                 mtrmng->drop_tbl[i]);
16396                         mtrmng->drop_tbl[i] = NULL;
16397                 }
16398         }
16399 }
16400
16401 /* Number of meter flow actions, count and jump or count and drop. */
16402 #define METER_ACTIONS 2
16403
16404 static void
16405 __flow_dv_destroy_domain_def_policy(struct rte_eth_dev *dev,
16406                                     enum mlx5_meter_domain domain)
16407 {
16408         struct mlx5_priv *priv = dev->data->dev_private;
16409         struct mlx5_flow_meter_def_policy *def_policy =
16410                         priv->sh->mtrmng->def_policy[domain];
16411
16412         __flow_dv_destroy_sub_policy_rules(dev, &def_policy->sub_policy);
16413         mlx5_free(def_policy);
16414         priv->sh->mtrmng->def_policy[domain] = NULL;
16415 }
16416
16417 /**
16418  * Destroy the default policy table set.
16419  *
16420  * @param[in] dev
16421  *   Pointer to Ethernet device.
16422  */
16423 static void
16424 flow_dv_destroy_def_policy(struct rte_eth_dev *dev)
16425 {
16426         struct mlx5_priv *priv = dev->data->dev_private;
16427         int i;
16428
16429         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++)
16430                 if (priv->sh->mtrmng->def_policy[i])
16431                         __flow_dv_destroy_domain_def_policy(dev,
16432                                         (enum mlx5_meter_domain)i);
16433         priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
16434 }
16435
16436 static int
16437 __flow_dv_create_policy_flow(struct rte_eth_dev *dev,
16438                         uint32_t color_reg_c_idx,
16439                         enum rte_color color, void *matcher_object,
16440                         int actions_n, void *actions,
16441                         bool match_src_port, const struct rte_flow_item *item,
16442                         void **rule, const struct rte_flow_attr *attr)
16443 {
16444         int ret;
16445         struct mlx5_flow_dv_match_params value = {
16446                 .size = sizeof(value.buf),
16447         };
16448         struct mlx5_flow_dv_match_params matcher = {
16449                 .size = sizeof(matcher.buf),
16450         };
16451         struct mlx5_priv *priv = dev->data->dev_private;
16452         uint8_t misc_mask;
16453
16454         if (match_src_port && priv->sh->esw_mode) {
16455                 if (flow_dv_translate_item_port_id(dev, matcher.buf,
16456                                                    value.buf, item, attr)) {
16457                         DRV_LOG(ERR, "Failed to create meter policy%d flow's"
16458                                 " value with port.", color);
16459                         return -1;
16460                 }
16461         }
16462         flow_dv_match_meta_reg(matcher.buf, value.buf,
16463                                (enum modify_reg)color_reg_c_idx,
16464                                rte_col_2_mlx5_col(color), UINT32_MAX);
16465         misc_mask = flow_dv_matcher_enable(value.buf);
16466         __flow_dv_adjust_buf_size(&value.size, misc_mask);
16467         ret = mlx5_flow_os_create_flow(matcher_object, (void *)&value,
16468                                        actions_n, actions, rule);
16469         if (ret) {
16470                 DRV_LOG(ERR, "Failed to create meter policy%d flow.", color);
16471                 return -1;
16472         }
16473         return 0;
16474 }
16475
16476 static int
16477 __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
16478                         uint32_t color_reg_c_idx,
16479                         uint16_t priority,
16480                         struct mlx5_flow_meter_sub_policy *sub_policy,
16481                         const struct rte_flow_attr *attr,
16482                         bool match_src_port,
16483                         const struct rte_flow_item *item,
16484                         struct mlx5_flow_dv_matcher **policy_matcher,
16485                         struct rte_flow_error *error)
16486 {
16487         struct mlx5_list_entry *entry;
16488         struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc;
16489         struct mlx5_flow_dv_matcher matcher = {
16490                 .mask = {
16491                         .size = sizeof(matcher.mask.buf),
16492                 },
16493                 .tbl = tbl_rsc,
16494         };
16495         struct mlx5_flow_dv_match_params value = {
16496                 .size = sizeof(value.buf),
16497         };
16498         struct mlx5_flow_cb_ctx ctx = {
16499                 .error = error,
16500                 .data = &matcher,
16501         };
16502         struct mlx5_flow_tbl_data_entry *tbl_data;
16503         struct mlx5_priv *priv = dev->data->dev_private;
16504         const uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1;
16505
16506         if (match_src_port && priv->sh->esw_mode) {
16507                 if (flow_dv_translate_item_port_id(dev, matcher.mask.buf,
16508                                                    value.buf, item, attr)) {
16509                         DRV_LOG(ERR, "Failed to register meter policy%d matcher"
16510                                 " with port.", priority);
16511                         return -1;
16512                 }
16513         }
16514         tbl_data = container_of(tbl_rsc, struct mlx5_flow_tbl_data_entry, tbl);
16515         if (priority < RTE_COLOR_RED)
16516                 flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16517                         (enum modify_reg)color_reg_c_idx, 0, color_mask);
16518         matcher.priority = priority;
16519         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
16520                                     matcher.mask.size);
16521         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16522         if (!entry) {
16523                 DRV_LOG(ERR, "Failed to register meter drop matcher.");
16524                 return -1;
16525         }
16526         *policy_matcher =
16527                 container_of(entry, struct mlx5_flow_dv_matcher, entry);
16528         return 0;
16529 }
16530
16531 /**
16532  * Create the policy rules per domain.
16533  *
16534  * @param[in] dev
16535  *   Pointer to Ethernet device.
16536  * @param[in] sub_policy
16537  *    Pointer to sub policy table..
16538  * @param[in] egress
16539  *   Direction of the table.
16540  * @param[in] transfer
16541  *   E-Switch or NIC flow.
16542  * @param[in] acts
16543  *   Pointer to policy action list per color.
16544  *
16545  * @return
16546  *   0 on success, -1 otherwise.
16547  */
16548 static int
16549 __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev,
16550                 struct mlx5_flow_meter_sub_policy *sub_policy,
16551                 uint8_t egress, uint8_t transfer, bool match_src_port,
16552                 struct mlx5_meter_policy_acts acts[RTE_COLORS])
16553 {
16554         struct mlx5_priv *priv = dev->data->dev_private;
16555         struct rte_flow_error flow_err;
16556         uint32_t color_reg_c_idx;
16557         struct rte_flow_attr attr = {
16558                 .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
16559                 .priority = 0,
16560                 .ingress = 0,
16561                 .egress = !!egress,
16562                 .transfer = !!transfer,
16563                 .reserved = 0,
16564         };
16565         int i;
16566         int ret = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &flow_err);
16567         struct mlx5_sub_policy_color_rule *color_rule;
16568         bool svport_match;
16569         struct mlx5_sub_policy_color_rule *tmp_rules[RTE_COLORS] = {NULL};
16570
16571         if (ret < 0)
16572                 return -1;
16573         /* Create policy table with POLICY level. */
16574         if (!sub_policy->tbl_rsc)
16575                 sub_policy->tbl_rsc = flow_dv_tbl_resource_get(dev,
16576                                 MLX5_FLOW_TABLE_LEVEL_POLICY,
16577                                 egress, transfer, false, NULL, 0, 0,
16578                                 sub_policy->idx, &flow_err);
16579         if (!sub_policy->tbl_rsc) {
16580                 DRV_LOG(ERR,
16581                         "Failed to create meter sub policy table.");
16582                 return -1;
16583         }
16584         /* Prepare matchers. */
16585         color_reg_c_idx = ret;
16586         for (i = 0; i < RTE_COLORS; i++) {
16587                 TAILQ_INIT(&sub_policy->color_rules[i]);
16588                 if (!acts[i].actions_n)
16589                         continue;
16590                 color_rule = mlx5_malloc(MLX5_MEM_ZERO,
16591                                 sizeof(struct mlx5_sub_policy_color_rule),
16592                                 0, SOCKET_ID_ANY);
16593                 if (!color_rule) {
16594                         DRV_LOG(ERR, "No memory to create color rule.");
16595                         goto err_exit;
16596                 }
16597                 tmp_rules[i] = color_rule;
16598                 TAILQ_INSERT_TAIL(&sub_policy->color_rules[i],
16599                                   color_rule, next_port);
16600                 color_rule->src_port = priv->representor_id;
16601                 /* No use. */
16602                 attr.priority = i;
16603                 /* Create matchers for colors. */
16604                 svport_match = (i != RTE_COLOR_RED) ? match_src_port : false;
16605                 if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx,
16606                                 MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy,
16607                                 &attr, svport_match, NULL,
16608                                 &color_rule->matcher, &flow_err)) {
16609                         DRV_LOG(ERR, "Failed to create color%u matcher.", i);
16610                         goto err_exit;
16611                 }
16612                 /* Create flow, matching color. */
16613                 if (__flow_dv_create_policy_flow(dev,
16614                                 color_reg_c_idx, (enum rte_color)i,
16615                                 color_rule->matcher->matcher_object,
16616                                 acts[i].actions_n, acts[i].dv_actions,
16617                                 svport_match, NULL, &color_rule->rule,
16618                                 &attr)) {
16619                         DRV_LOG(ERR, "Failed to create color%u rule.", i);
16620                         goto err_exit;
16621                 }
16622         }
16623         return 0;
16624 err_exit:
16625         /* All the policy rules will be cleared. */
16626         do {
16627                 color_rule = tmp_rules[i];
16628                 if (color_rule) {
16629                         if (color_rule->rule)
16630                                 mlx5_flow_os_destroy_flow(color_rule->rule);
16631                         if (color_rule->matcher) {
16632                                 struct mlx5_flow_tbl_data_entry *tbl =
16633                                         container_of(color_rule->matcher->tbl,
16634                                                      typeof(*tbl), tbl);
16635                                 mlx5_list_unregister(tbl->matchers,
16636                                                 &color_rule->matcher->entry);
16637                         }
16638                         TAILQ_REMOVE(&sub_policy->color_rules[i],
16639                                      color_rule, next_port);
16640                         mlx5_free(color_rule);
16641                 }
16642         } while (i--);
16643         return -1;
16644 }
16645
16646 static int
16647 __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev,
16648                         struct mlx5_flow_meter_policy *mtr_policy,
16649                         struct mlx5_flow_meter_sub_policy *sub_policy,
16650                         uint32_t domain)
16651 {
16652         struct mlx5_priv *priv = dev->data->dev_private;
16653         struct mlx5_meter_policy_acts acts[RTE_COLORS];
16654         struct mlx5_flow_dv_tag_resource *tag;
16655         struct mlx5_flow_dv_port_id_action_resource *port_action;
16656         struct mlx5_hrxq *hrxq;
16657         struct mlx5_flow_meter_info *next_fm[RTE_COLORS] = {NULL};
16658         struct mlx5_flow_meter_policy *next_policy;
16659         struct mlx5_flow_meter_sub_policy *next_sub_policy;
16660         struct mlx5_flow_tbl_data_entry *tbl_data;
16661         struct rte_flow_error error;
16662         uint8_t egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16663         uint8_t transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16664         bool mtr_first = egress || (transfer && priv->representor_id != UINT16_MAX);
16665         bool match_src_port = false;
16666         int i;
16667
16668         /* If RSS or Queue, no previous actions / rules is created. */
16669         for (i = 0; i < RTE_COLORS; i++) {
16670                 acts[i].actions_n = 0;
16671                 if (i == RTE_COLOR_RED) {
16672                         /* Only support drop on red. */
16673                         acts[i].dv_actions[0] =
16674                                 mtr_policy->dr_drop_action[domain];
16675                         acts[i].actions_n = 1;
16676                         continue;
16677                 }
16678                 if (mtr_policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) {
16679                         struct rte_flow_attr attr = {
16680                                 .transfer = transfer
16681                         };
16682
16683                         next_fm[i] = mlx5_flow_meter_find(priv,
16684                                         mtr_policy->act_cnt[i].next_mtr_id,
16685                                         NULL);
16686                         if (!next_fm[i]) {
16687                                 DRV_LOG(ERR,
16688                                         "Failed to get next hierarchy meter.");
16689                                 goto err_exit;
16690                         }
16691                         if (mlx5_flow_meter_attach(priv, next_fm[i],
16692                                                    &attr, &error)) {
16693                                 DRV_LOG(ERR, "%s", error.message);
16694                                 next_fm[i] = NULL;
16695                                 goto err_exit;
16696                         }
16697                         /* Meter action must be the first for TX. */
16698                         if (mtr_first) {
16699                                 acts[i].dv_actions[acts[i].actions_n] =
16700                                         (next_fm[i]->color_aware && i == RTE_COLOR_YELLOW) ?
16701                                                 next_fm[i]->meter_action_y :
16702                                                 next_fm[i]->meter_action_g;
16703                                 acts[i].actions_n++;
16704                         }
16705                 }
16706                 if (mtr_policy->act_cnt[i].rix_mark) {
16707                         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG],
16708                                         mtr_policy->act_cnt[i].rix_mark);
16709                         if (!tag) {
16710                                 DRV_LOG(ERR, "Failed to find "
16711                                 "mark action for policy.");
16712                                 goto err_exit;
16713                         }
16714                         acts[i].dv_actions[acts[i].actions_n] = tag->action;
16715                         acts[i].actions_n++;
16716                 }
16717                 if (mtr_policy->act_cnt[i].modify_hdr) {
16718                         acts[i].dv_actions[acts[i].actions_n] =
16719                                 mtr_policy->act_cnt[i].modify_hdr->action;
16720                         acts[i].actions_n++;
16721                 }
16722                 if (mtr_policy->act_cnt[i].fate_action) {
16723                         switch (mtr_policy->act_cnt[i].fate_action) {
16724                         case MLX5_FLOW_FATE_PORT_ID:
16725                                 port_action = mlx5_ipool_get
16726                                         (priv->sh->ipool[MLX5_IPOOL_PORT_ID],
16727                                 mtr_policy->act_cnt[i].rix_port_id_action);
16728                                 if (!port_action) {
16729                                         DRV_LOG(ERR, "Failed to find "
16730                                                 "port action for policy.");
16731                                         goto err_exit;
16732                                 }
16733                                 acts[i].dv_actions[acts[i].actions_n] =
16734                                         port_action->action;
16735                                 acts[i].actions_n++;
16736                                 mtr_policy->dev = dev;
16737                                 match_src_port = true;
16738                                 break;
16739                         case MLX5_FLOW_FATE_DROP:
16740                         case MLX5_FLOW_FATE_JUMP:
16741                                 acts[i].dv_actions[acts[i].actions_n] =
16742                                 mtr_policy->act_cnt[i].dr_jump_action[domain];
16743                                 acts[i].actions_n++;
16744                                 break;
16745                         case MLX5_FLOW_FATE_SHARED_RSS:
16746                         case MLX5_FLOW_FATE_QUEUE:
16747                                 hrxq = mlx5_ipool_get
16748                                         (priv->sh->ipool[MLX5_IPOOL_HRXQ],
16749                                          sub_policy->rix_hrxq[i]);
16750                                 if (!hrxq) {
16751                                         DRV_LOG(ERR, "Failed to find "
16752                                                 "queue action for policy.");
16753                                         goto err_exit;
16754                                 }
16755                                 acts[i].dv_actions[acts[i].actions_n] =
16756                                         hrxq->action;
16757                                 acts[i].actions_n++;
16758                                 break;
16759                         case MLX5_FLOW_FATE_MTR:
16760                                 if (!next_fm[i]) {
16761                                         DRV_LOG(ERR,
16762                                                 "No next hierarchy meter.");
16763                                         goto err_exit;
16764                                 }
16765                                 if (!mtr_first) {
16766                                         acts[i].dv_actions[acts[i].actions_n] =
16767                                                 (next_fm[i]->color_aware && i == RTE_COLOR_YELLOW) ?
16768                                                         next_fm[i]->meter_action_y :
16769                                                         next_fm[i]->meter_action_g;
16770                                         acts[i].actions_n++;
16771                                 }
16772                                 if (mtr_policy->act_cnt[i].next_sub_policy) {
16773                                         next_sub_policy =
16774                                         mtr_policy->act_cnt[i].next_sub_policy;
16775                                 } else {
16776                                         next_policy =
16777                                                 mlx5_flow_meter_policy_find(dev,
16778                                                                 next_fm[i]->policy_id, NULL);
16779                                         MLX5_ASSERT(next_policy);
16780                                         next_sub_policy =
16781                                         next_policy->sub_policys[domain][0];
16782                                 }
16783                                 tbl_data =
16784                                         container_of(next_sub_policy->tbl_rsc,
16785                                         struct mlx5_flow_tbl_data_entry, tbl);
16786                                 acts[i].dv_actions[acts[i].actions_n++] =
16787                                                         tbl_data->jump.action;
16788                                 if (mtr_policy->act_cnt[i].modify_hdr)
16789                                         match_src_port = !!transfer;
16790                                 break;
16791                         default:
16792                                 /*Queue action do nothing*/
16793                                 break;
16794                         }
16795                 }
16796         }
16797         if (__flow_dv_create_domain_policy_rules(dev, sub_policy,
16798                                 egress, transfer, match_src_port, acts)) {
16799                 DRV_LOG(ERR,
16800                         "Failed to create policy rules per domain.");
16801                 goto err_exit;
16802         }
16803         return 0;
16804 err_exit:
16805         for (i = 0; i < RTE_COLORS; i++)
16806                 if (next_fm[i])
16807                         mlx5_flow_meter_detach(priv, next_fm[i]);
16808         return -1;
16809 }
16810
16811 /**
16812  * Create the policy rules.
16813  *
16814  * @param[in] dev
16815  *   Pointer to Ethernet device.
16816  * @param[in,out] mtr_policy
16817  *   Pointer to meter policy table.
16818  *
16819  * @return
16820  *   0 on success, -1 otherwise.
16821  */
16822 static int
16823 flow_dv_create_policy_rules(struct rte_eth_dev *dev,
16824                              struct mlx5_flow_meter_policy *mtr_policy)
16825 {
16826         int i;
16827         uint16_t sub_policy_num;
16828
16829         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16830                 sub_policy_num = (mtr_policy->sub_policy_num >>
16831                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
16832                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16833                 if (!sub_policy_num)
16834                         continue;
16835                 /* Prepare actions list and create policy rules. */
16836                 if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
16837                         mtr_policy->sub_policys[i][0], i)) {
16838                         DRV_LOG(ERR, "Failed to create policy action "
16839                                 "list per domain.");
16840                         return -1;
16841                 }
16842         }
16843         return 0;
16844 }
16845
16846 static int
16847 __flow_dv_create_domain_def_policy(struct rte_eth_dev *dev, uint32_t domain)
16848 {
16849         struct mlx5_priv *priv = dev->data->dev_private;
16850         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
16851         struct mlx5_flow_meter_def_policy *def_policy;
16852         struct mlx5_flow_tbl_resource *jump_tbl;
16853         struct mlx5_flow_tbl_data_entry *tbl_data;
16854         uint8_t egress, transfer;
16855         struct rte_flow_error error;
16856         struct mlx5_meter_policy_acts acts[RTE_COLORS];
16857         int ret;
16858
16859         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16860         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16861         def_policy = mtrmng->def_policy[domain];
16862         if (!def_policy) {
16863                 def_policy = mlx5_malloc(MLX5_MEM_ZERO,
16864                         sizeof(struct mlx5_flow_meter_def_policy),
16865                         RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
16866                 if (!def_policy) {
16867                         DRV_LOG(ERR, "Failed to alloc default policy table.");
16868                         goto def_policy_error;
16869                 }
16870                 mtrmng->def_policy[domain] = def_policy;
16871                 /* Create the meter suffix table with SUFFIX level. */
16872                 jump_tbl = flow_dv_tbl_resource_get(dev,
16873                                 MLX5_FLOW_TABLE_LEVEL_METER,
16874                                 egress, transfer, false, NULL, 0,
16875                                 0, MLX5_MTR_TABLE_ID_SUFFIX, &error);
16876                 if (!jump_tbl) {
16877                         DRV_LOG(ERR,
16878                                 "Failed to create meter suffix table.");
16879                         goto def_policy_error;
16880                 }
16881                 def_policy->sub_policy.jump_tbl[RTE_COLOR_GREEN] = jump_tbl;
16882                 tbl_data = container_of(jump_tbl,
16883                                         struct mlx5_flow_tbl_data_entry, tbl);
16884                 def_policy->dr_jump_action[RTE_COLOR_GREEN] =
16885                                                 tbl_data->jump.action;
16886                 acts[RTE_COLOR_GREEN].dv_actions[0] = tbl_data->jump.action;
16887                 acts[RTE_COLOR_GREEN].actions_n = 1;
16888                 /*
16889                  * YELLOW has the same default policy as GREEN does.
16890                  * G & Y share the same table and action. The 2nd time of table
16891                  * resource getting is just to update the reference count for
16892                  * the releasing stage.
16893                  */
16894                 jump_tbl = flow_dv_tbl_resource_get(dev,
16895                                 MLX5_FLOW_TABLE_LEVEL_METER,
16896                                 egress, transfer, false, NULL, 0,
16897                                 0, MLX5_MTR_TABLE_ID_SUFFIX, &error);
16898                 if (!jump_tbl) {
16899                         DRV_LOG(ERR,
16900                                 "Failed to get meter suffix table.");
16901                         goto def_policy_error;
16902                 }
16903                 def_policy->sub_policy.jump_tbl[RTE_COLOR_YELLOW] = jump_tbl;
16904                 tbl_data = container_of(jump_tbl,
16905                                         struct mlx5_flow_tbl_data_entry, tbl);
16906                 def_policy->dr_jump_action[RTE_COLOR_YELLOW] =
16907                                                 tbl_data->jump.action;
16908                 acts[RTE_COLOR_YELLOW].dv_actions[0] = tbl_data->jump.action;
16909                 acts[RTE_COLOR_YELLOW].actions_n = 1;
16910                 /* Create jump action to the drop table. */
16911                 if (!mtrmng->drop_tbl[domain]) {
16912                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get
16913                                 (dev, MLX5_FLOW_TABLE_LEVEL_METER,
16914                                  egress, transfer, false, NULL, 0,
16915                                  0, MLX5_MTR_TABLE_ID_DROP, &error);
16916                         if (!mtrmng->drop_tbl[domain]) {
16917                                 DRV_LOG(ERR, "Failed to create meter "
16918                                         "drop table for default policy.");
16919                                 goto def_policy_error;
16920                         }
16921                 }
16922                 /* all RED: unique Drop table for jump action. */
16923                 tbl_data = container_of(mtrmng->drop_tbl[domain],
16924                                         struct mlx5_flow_tbl_data_entry, tbl);
16925                 def_policy->dr_jump_action[RTE_COLOR_RED] =
16926                                                 tbl_data->jump.action;
16927                 acts[RTE_COLOR_RED].dv_actions[0] = tbl_data->jump.action;
16928                 acts[RTE_COLOR_RED].actions_n = 1;
16929                 /* Create default policy rules. */
16930                 ret = __flow_dv_create_domain_policy_rules(dev,
16931                                         &def_policy->sub_policy,
16932                                         egress, transfer, false, acts);
16933                 if (ret) {
16934                         DRV_LOG(ERR, "Failed to create default policy rules.");
16935                         goto def_policy_error;
16936                 }
16937         }
16938         return 0;
16939 def_policy_error:
16940         __flow_dv_destroy_domain_def_policy(dev,
16941                                             (enum mlx5_meter_domain)domain);
16942         return -1;
16943 }
16944
16945 /**
16946  * Create the default policy table set.
16947  *
16948  * @param[in] dev
16949  *   Pointer to Ethernet device.
16950  * @return
16951  *   0 on success, -1 otherwise.
16952  */
16953 static int
16954 flow_dv_create_def_policy(struct rte_eth_dev *dev)
16955 {
16956         struct mlx5_priv *priv = dev->data->dev_private;
16957         int i;
16958
16959         /* Non-termination policy table. */
16960         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16961                 if (!priv->sh->config.dv_esw_en &&
16962                     i == MLX5_MTR_DOMAIN_TRANSFER)
16963                         continue;
16964                 if (__flow_dv_create_domain_def_policy(dev, i)) {
16965                         DRV_LOG(ERR, "Failed to create default policy");
16966                         /* Rollback the created default policies for others. */
16967                         flow_dv_destroy_def_policy(dev);
16968                         return -1;
16969                 }
16970         }
16971         return 0;
16972 }
16973
16974 /**
16975  * Create the needed meter tables.
16976  * Lock free, (mutex should be acquired by caller).
16977  *
16978  * @param[in] dev
16979  *   Pointer to Ethernet device.
16980  * @param[in] fm
16981  *   Meter information table.
16982  * @param[in] mtr_idx
16983  *   Meter index.
16984  * @param[in] domain_bitmap
16985  *   Domain bitmap.
16986  * @return
16987  *   0 on success, -1 otherwise.
16988  */
16989 static int
16990 flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
16991                         struct mlx5_flow_meter_info *fm,
16992                         uint32_t mtr_idx,
16993                         uint8_t domain_bitmap)
16994 {
16995         struct mlx5_priv *priv = dev->data->dev_private;
16996         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
16997         struct rte_flow_error error;
16998         struct mlx5_flow_tbl_data_entry *tbl_data;
16999         uint8_t egress, transfer;
17000         void *actions[METER_ACTIONS];
17001         int domain, ret, i;
17002         struct mlx5_flow_counter *cnt;
17003         struct mlx5_flow_dv_match_params value = {
17004                 .size = sizeof(value.buf),
17005         };
17006         struct mlx5_flow_dv_match_params matcher_para = {
17007                 .size = sizeof(matcher_para.buf),
17008         };
17009         int mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
17010                                                      0, &error);
17011         uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1;
17012         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
17013         struct mlx5_list_entry *entry;
17014         struct mlx5_flow_dv_matcher matcher = {
17015                 .mask = {
17016                         .size = sizeof(matcher.mask.buf),
17017                 },
17018         };
17019         struct mlx5_flow_dv_matcher *drop_matcher;
17020         struct mlx5_flow_cb_ctx ctx = {
17021                 .error = &error,
17022                 .data = &matcher,
17023         };
17024         uint8_t misc_mask;
17025
17026         if (!priv->mtr_en || mtr_id_reg_c < 0) {
17027                 rte_errno = ENOTSUP;
17028                 return -1;
17029         }
17030         for (domain = 0; domain < MLX5_MTR_DOMAIN_MAX; domain++) {
17031                 if (!(domain_bitmap & (1 << domain)) ||
17032                         (mtrmng->def_rule[domain] && !fm->drop_cnt))
17033                         continue;
17034                 egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
17035                 transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
17036                 /* Create the drop table with METER DROP level. */
17037                 if (!mtrmng->drop_tbl[domain]) {
17038                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get(dev,
17039                                         MLX5_FLOW_TABLE_LEVEL_METER,
17040                                         egress, transfer, false, NULL, 0,
17041                                         0, MLX5_MTR_TABLE_ID_DROP, &error);
17042                         if (!mtrmng->drop_tbl[domain]) {
17043                                 DRV_LOG(ERR, "Failed to create meter drop table.");
17044                                 goto policy_error;
17045                         }
17046                 }
17047                 /* Create default matcher in drop table. */
17048                 matcher.tbl = mtrmng->drop_tbl[domain],
17049                 tbl_data = container_of(mtrmng->drop_tbl[domain],
17050                                 struct mlx5_flow_tbl_data_entry, tbl);
17051                 if (!mtrmng->def_matcher[domain]) {
17052                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
17053                                        (enum modify_reg)mtr_id_reg_c,
17054                                        0, 0);
17055                         matcher.priority = MLX5_MTRS_DEFAULT_RULE_PRIORITY;
17056                         matcher.crc = rte_raw_cksum
17057                                         ((const void *)matcher.mask.buf,
17058                                         matcher.mask.size);
17059                         entry = mlx5_list_register(tbl_data->matchers, &ctx);
17060                         if (!entry) {
17061                                 DRV_LOG(ERR, "Failed to register meter "
17062                                 "drop default matcher.");
17063                                 goto policy_error;
17064                         }
17065                         mtrmng->def_matcher[domain] = container_of(entry,
17066                         struct mlx5_flow_dv_matcher, entry);
17067                 }
17068                 /* Create default rule in drop table. */
17069                 if (!mtrmng->def_rule[domain]) {
17070                         i = 0;
17071                         actions[i++] = priv->sh->dr_drop_action;
17072                         flow_dv_match_meta_reg(matcher_para.buf, value.buf,
17073                                 (enum modify_reg)mtr_id_reg_c, 0, 0);
17074                         misc_mask = flow_dv_matcher_enable(value.buf);
17075                         __flow_dv_adjust_buf_size(&value.size, misc_mask);
17076                         ret = mlx5_flow_os_create_flow
17077                                 (mtrmng->def_matcher[domain]->matcher_object,
17078                                 (void *)&value, i, actions,
17079                                 &mtrmng->def_rule[domain]);
17080                         if (ret) {
17081                                 DRV_LOG(ERR, "Failed to create meter "
17082                                 "default drop rule for drop table.");
17083                                 goto policy_error;
17084                         }
17085                 }
17086                 if (!fm->drop_cnt)
17087                         continue;
17088                 MLX5_ASSERT(mtrmng->max_mtr_bits);
17089                 if (!mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1]) {
17090                         /* Create matchers for Drop. */
17091                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
17092                                         (enum modify_reg)mtr_id_reg_c, 0,
17093                                         (mtr_id_mask << mtr_id_offset));
17094                         matcher.priority = MLX5_REG_BITS - mtrmng->max_mtr_bits;
17095                         matcher.crc = rte_raw_cksum
17096                                         ((const void *)matcher.mask.buf,
17097                                         matcher.mask.size);
17098                         entry = mlx5_list_register(tbl_data->matchers, &ctx);
17099                         if (!entry) {
17100                                 DRV_LOG(ERR,
17101                                 "Failed to register meter drop matcher.");
17102                                 goto policy_error;
17103                         }
17104                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1] =
17105                                 container_of(entry, struct mlx5_flow_dv_matcher,
17106                                              entry);
17107                 }
17108                 drop_matcher =
17109                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1];
17110                 /* Create drop rule, matching meter_id only. */
17111                 flow_dv_match_meta_reg(matcher_para.buf, value.buf,
17112                                 (enum modify_reg)mtr_id_reg_c,
17113                                 (mtr_idx << mtr_id_offset), UINT32_MAX);
17114                 i = 0;
17115                 cnt = flow_dv_counter_get_by_idx(dev,
17116                                         fm->drop_cnt, NULL);
17117                 actions[i++] = cnt->action;
17118                 actions[i++] = priv->sh->dr_drop_action;
17119                 misc_mask = flow_dv_matcher_enable(value.buf);
17120                 __flow_dv_adjust_buf_size(&value.size, misc_mask);
17121                 ret = mlx5_flow_os_create_flow(drop_matcher->matcher_object,
17122                                                (void *)&value, i, actions,
17123                                                &fm->drop_rule[domain]);
17124                 if (ret) {
17125                         DRV_LOG(ERR, "Failed to create meter "
17126                                 "drop rule for drop table.");
17127                                 goto policy_error;
17128                 }
17129         }
17130         return 0;
17131 policy_error:
17132         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
17133                 if (fm->drop_rule[i]) {
17134                         claim_zero(mlx5_flow_os_destroy_flow
17135                                 (fm->drop_rule[i]));
17136                         fm->drop_rule[i] = NULL;
17137                 }
17138         }
17139         return -1;
17140 }
17141
17142 static struct mlx5_flow_meter_sub_policy *
17143 __flow_dv_meter_get_rss_sub_policy(struct rte_eth_dev *dev,
17144                 struct mlx5_flow_meter_policy *mtr_policy,
17145                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS],
17146                 struct mlx5_flow_meter_sub_policy *next_sub_policy,
17147                 bool *is_reuse)
17148 {
17149         struct mlx5_priv *priv = dev->data->dev_private;
17150         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
17151         uint32_t sub_policy_idx = 0;
17152         uint32_t hrxq_idx[MLX5_MTR_RTE_COLORS] = {0};
17153         uint32_t i, j;
17154         struct mlx5_hrxq *hrxq;
17155         struct mlx5_flow_handle dh;
17156         struct mlx5_meter_policy_action_container *act_cnt;
17157         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
17158         uint16_t sub_policy_num;
17159         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
17160
17161         MLX5_ASSERT(wks);
17162         rte_spinlock_lock(&mtr_policy->sl);
17163         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
17164                 if (!rss_desc[i])
17165                         continue;
17166                 hrxq = mlx5_hrxq_get(dev, rss_desc[i]);
17167                 if (!hrxq) {
17168                         rte_spinlock_unlock(&mtr_policy->sl);
17169                         return NULL;
17170                 }
17171                 hrxq_idx[i] = hrxq->idx;
17172         }
17173         sub_policy_num = (mtr_policy->sub_policy_num >>
17174                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
17175                         MLX5_MTR_SUB_POLICY_NUM_MASK;
17176         for (j = 0; j < sub_policy_num; j++) {
17177                 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
17178                         if (rss_desc[i] &&
17179                             hrxq_idx[i] !=
17180                             mtr_policy->sub_policys[domain][j]->rix_hrxq[i])
17181                                 break;
17182                 }
17183                 if (i >= MLX5_MTR_RTE_COLORS) {
17184                         /*
17185                          * Found the sub policy table with
17186                          * the same queue per color.
17187                          */
17188                         rte_spinlock_unlock(&mtr_policy->sl);
17189                         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++)
17190                                 mlx5_hrxq_release(dev, hrxq_idx[i]);
17191                         *is_reuse = true;
17192                         return mtr_policy->sub_policys[domain][j];
17193                 }
17194         }
17195         /* Create sub policy. */
17196         if (!mtr_policy->sub_policys[domain][0]->rix_hrxq[RTE_COLOR_GREEN] &&
17197             !mtr_policy->sub_policys[domain][0]->rix_hrxq[RTE_COLOR_YELLOW]) {
17198                 /* Reuse the first pre-allocated sub_policy. */
17199                 sub_policy = mtr_policy->sub_policys[domain][0];
17200                 sub_policy_idx = sub_policy->idx;
17201         } else {
17202                 sub_policy = mlx5_ipool_zmalloc
17203                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
17204                                  &sub_policy_idx);
17205                 if (!sub_policy ||
17206                     sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) {
17207                         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++)
17208                                 mlx5_hrxq_release(dev, hrxq_idx[i]);
17209                         goto rss_sub_policy_error;
17210                 }
17211                 sub_policy->idx = sub_policy_idx;
17212                 sub_policy->main_policy = mtr_policy;
17213         }
17214         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
17215                 if (!rss_desc[i])
17216                         continue;
17217                 sub_policy->rix_hrxq[i] = hrxq_idx[i];
17218                 if (mtr_policy->is_hierarchy) {
17219                         act_cnt = &mtr_policy->act_cnt[i];
17220                         act_cnt->next_sub_policy = next_sub_policy;
17221                         mlx5_hrxq_release(dev, hrxq_idx[i]);
17222                 } else {
17223                         /*
17224                          * Overwrite the last action from
17225                          * RSS action to Queue action.
17226                          */
17227                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
17228                                               hrxq_idx[i]);
17229                         if (!hrxq) {
17230                                 DRV_LOG(ERR, "Failed to get policy hrxq");
17231                                 goto rss_sub_policy_error;
17232                         }
17233                         act_cnt = &mtr_policy->act_cnt[i];
17234                         if (act_cnt->rix_mark || act_cnt->modify_hdr) {
17235                                 memset(&dh, 0, sizeof(struct mlx5_flow_handle));
17236                                 if (act_cnt->rix_mark)
17237                                         wks->mark = 1;
17238                                 dh.fate_action = MLX5_FLOW_FATE_QUEUE;
17239                                 dh.rix_hrxq = hrxq_idx[i];
17240                                 flow_drv_rxq_flags_set(dev, &dh);
17241                         }
17242                 }
17243         }
17244         if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
17245                                                sub_policy, domain)) {
17246                 DRV_LOG(ERR, "Failed to create policy "
17247                         "rules for ingress domain.");
17248                 goto rss_sub_policy_error;
17249         }
17250         if (sub_policy != mtr_policy->sub_policys[domain][0]) {
17251                 i = (mtr_policy->sub_policy_num >>
17252                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
17253                         MLX5_MTR_SUB_POLICY_NUM_MASK;
17254                 if (i >= MLX5_MTR_RSS_MAX_SUB_POLICY) {
17255                         DRV_LOG(ERR, "No free sub-policy slot.");
17256                         goto rss_sub_policy_error;
17257                 }
17258                 mtr_policy->sub_policys[domain][i] = sub_policy;
17259                 i++;
17260                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
17261                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
17262                 mtr_policy->sub_policy_num |=
17263                         (i & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
17264                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
17265         }
17266         rte_spinlock_unlock(&mtr_policy->sl);
17267         *is_reuse = false;
17268         return sub_policy;
17269 rss_sub_policy_error:
17270         if (sub_policy) {
17271                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
17272                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
17273                         i = (mtr_policy->sub_policy_num >>
17274                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
17275                         MLX5_MTR_SUB_POLICY_NUM_MASK;
17276                         mtr_policy->sub_policys[domain][i] = NULL;
17277                         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
17278                                         sub_policy->idx);
17279                 }
17280         }
17281         rte_spinlock_unlock(&mtr_policy->sl);
17282         return NULL;
17283 }
17284
17285 /**
17286  * Find the policy table for prefix table with RSS.
17287  *
17288  * @param[in] dev
17289  *   Pointer to Ethernet device.
17290  * @param[in] mtr_policy
17291  *   Pointer to meter policy table.
17292  * @param[in] rss_desc
17293  *   Pointer to rss_desc
17294  * @return
17295  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
17296  */
17297 static struct mlx5_flow_meter_sub_policy *
17298 flow_dv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
17299                 struct mlx5_flow_meter_policy *mtr_policy,
17300                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
17301 {
17302         struct mlx5_priv *priv = dev->data->dev_private;
17303         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
17304         struct mlx5_flow_meter_info *next_fm;
17305         struct mlx5_flow_meter_policy *next_policy;
17306         struct mlx5_flow_meter_sub_policy *next_sub_policy = NULL;
17307         struct mlx5_flow_meter_policy *policies[MLX5_MTR_CHAIN_MAX_NUM];
17308         struct mlx5_flow_meter_sub_policy *sub_policies[MLX5_MTR_CHAIN_MAX_NUM];
17309         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
17310         bool reuse_sub_policy;
17311         uint32_t i = 0;
17312         uint32_t j = 0;
17313
17314         while (true) {
17315                 /* Iterate hierarchy to get all policies in this hierarchy. */
17316                 policies[i++] = mtr_policy;
17317                 if (!mtr_policy->is_hierarchy)
17318                         break;
17319                 if (i >= MLX5_MTR_CHAIN_MAX_NUM) {
17320                         DRV_LOG(ERR, "Exceed max meter number in hierarchy.");
17321                         return NULL;
17322                 }
17323                 rte_spinlock_lock(&mtr_policy->sl);
17324                 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, mtr_policy, NULL);
17325                 rte_spinlock_unlock(&mtr_policy->sl);
17326                 if (!next_fm) {
17327                         DRV_LOG(ERR, "Failed to get next meter in hierarchy.");
17328                         return NULL;
17329                 }
17330                 next_policy =
17331                         mlx5_flow_meter_policy_find(dev, next_fm->policy_id,
17332                                                     NULL);
17333                 MLX5_ASSERT(next_policy);
17334                 mtr_policy = next_policy;
17335         }
17336         while (i) {
17337                 /**
17338                  * From last policy to the first one in hierarchy,
17339                  * create / get the sub policy for each of them.
17340                  */
17341                 sub_policy = __flow_dv_meter_get_rss_sub_policy(dev,
17342                                                         policies[--i],
17343                                                         rss_desc,
17344                                                         next_sub_policy,
17345                                                         &reuse_sub_policy);
17346                 if (!sub_policy) {
17347                         DRV_LOG(ERR, "Failed to get the sub policy.");
17348                         goto err_exit;
17349                 }
17350                 if (!reuse_sub_policy)
17351                         sub_policies[j++] = sub_policy;
17352                 next_sub_policy = sub_policy;
17353         }
17354         return sub_policy;
17355 err_exit:
17356         while (j) {
17357                 uint16_t sub_policy_num;
17358
17359                 sub_policy = sub_policies[--j];
17360                 mtr_policy = sub_policy->main_policy;
17361                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
17362                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
17363                         sub_policy_num = (mtr_policy->sub_policy_num >>
17364                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
17365                                 MLX5_MTR_SUB_POLICY_NUM_MASK;
17366                         mtr_policy->sub_policys[domain][sub_policy_num - 1] =
17367                                                                         NULL;
17368                         sub_policy_num--;
17369                         mtr_policy->sub_policy_num &=
17370                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
17371                                   (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
17372                         mtr_policy->sub_policy_num |=
17373                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
17374                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
17375                         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
17376                                         sub_policy->idx);
17377                 }
17378         }
17379         return NULL;
17380 }
17381
17382 /**
17383  * Check if need to create hierarchy tag rule.
17384  *
17385  * @param[in] priv
17386  *   Pointer to mlx5_priv.
17387  * @param[in] mtr_policy
17388  *   Pointer to current meter policy.
17389  * @param[in] src_port
17390  *   The src port this extra rule should use.
17391  * @param[out] next_fm
17392  *   Pointer to next meter in hierarchy.
17393  * @param[out] skip
17394  *   Indicate if skip the tag rule creation.
17395  * @param[out] error
17396  *   Perform verbose error reporting if not NULL.
17397  * @return
17398  *   0 on success, a negative errno value otherwise and rte_errno is set.
17399  */
17400 static int
17401 mlx5_meter_hierarchy_skip_tag_rule(struct mlx5_priv *priv,
17402                                    struct mlx5_flow_meter_policy *mtr_policy,
17403                                    int32_t src_port,
17404                                    struct mlx5_flow_meter_info **next_fm,
17405                                    bool *skip,
17406                                    struct rte_flow_error *error)
17407 {
17408         struct mlx5_flow_meter_sub_policy *sub_policy;
17409         struct mlx5_sub_policy_color_rule *color_rule;
17410         uint32_t domain = MLX5_MTR_DOMAIN_TRANSFER;
17411         int ret = 0;
17412         int i;
17413
17414         *next_fm = NULL;
17415         *skip = false;
17416         rte_spinlock_lock(&mtr_policy->sl);
17417         if (!mtr_policy->is_hierarchy)
17418                 goto exit;
17419         *next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, mtr_policy, NULL);
17420         if (!*next_fm) {
17421                 ret = rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
17422                                          NULL, "Failed to find next meter in hierarchy.");
17423                 goto exit;
17424         }
17425         if (!(*next_fm)->drop_cnt) {
17426                 *skip = true;
17427                 goto exit;
17428         }
17429         sub_policy = mtr_policy->sub_policys[domain][0];
17430         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
17431                 if (mtr_policy->act_cnt[i].fate_action != MLX5_FLOW_FATE_MTR)
17432                         continue;
17433                 TAILQ_FOREACH(color_rule, &sub_policy->color_rules[i], next_port)
17434                         if (color_rule->src_port == src_port) {
17435                                 *skip = true;
17436                                 goto exit;
17437                         }
17438         }
17439 exit:
17440         rte_spinlock_unlock(&mtr_policy->sl);
17441         return ret;
17442 }
17443
17444 /**
17445  * Create the sub policy tag rule for all meters in hierarchy.
17446  *
17447  * @param[in] dev
17448  *   Pointer to Ethernet device.
17449  * @param[in] fm
17450  *   Meter information table.
17451  * @param[in] src_port
17452  *   The src port this extra rule should use.
17453  * @param[in] item
17454  *   The src port match item.
17455  * @param[out] error
17456  *   Perform verbose error reporting if not NULL.
17457  * @return
17458  *   0 on success, a negative errno value otherwise and rte_errno is set.
17459  */
17460 static int
17461 flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev,
17462                                 struct mlx5_flow_meter_info *fm,
17463                                 int32_t src_port,
17464                                 const struct rte_flow_item *item,
17465                                 struct rte_flow_error *error)
17466 {
17467         struct mlx5_priv *priv = dev->data->dev_private;
17468         struct mlx5_flow_meter_policy *mtr_policy;
17469         struct mlx5_flow_meter_sub_policy *sub_policy;
17470         struct mlx5_flow_meter_info *next_fm = NULL;
17471         struct mlx5_flow_meter_policy *next_policy;
17472         struct mlx5_flow_meter_sub_policy *next_sub_policy;
17473         struct mlx5_flow_tbl_data_entry *tbl_data;
17474         struct mlx5_sub_policy_color_rule *color_rule;
17475         struct mlx5_meter_policy_acts acts;
17476         uint32_t color_reg_c_idx;
17477         bool mtr_first = (src_port != UINT16_MAX) ? true : false;
17478         struct rte_flow_attr attr = {
17479                 .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
17480                 .priority = 0,
17481                 .ingress = 0,
17482                 .egress = 0,
17483                 .transfer = 1,
17484                 .reserved = 0,
17485         };
17486         uint32_t domain = MLX5_MTR_DOMAIN_TRANSFER;
17487         struct {
17488                 struct mlx5_flow_meter_policy *fm_policy;
17489                 struct mlx5_flow_meter_info *next_fm;
17490                 struct mlx5_sub_policy_color_rule *tag_rule[MLX5_MTR_RTE_COLORS];
17491         } fm_info[MLX5_MTR_CHAIN_MAX_NUM] = { {0} };
17492         uint32_t fm_cnt = 0;
17493         uint32_t i, j;
17494
17495         color_reg_c_idx = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, error);
17496         /* Get all fms who need to create the tag color rule. */
17497         do {
17498                 bool skip = false;
17499
17500                 mtr_policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
17501                 MLX5_ASSERT(mtr_policy);
17502                 if (mlx5_meter_hierarchy_skip_tag_rule(priv, mtr_policy, src_port,
17503                                                        &next_fm, &skip, error))
17504                         goto err_exit;
17505                 if (next_fm && !skip) {
17506                         fm_info[fm_cnt].fm_policy = mtr_policy;
17507                         fm_info[fm_cnt].next_fm = next_fm;
17508                         if (++fm_cnt >= MLX5_MTR_CHAIN_MAX_NUM) {
17509                                 rte_flow_error_set(error, errno,
17510                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
17511                                         "Exceed max meter number in hierarchy.");
17512                                 goto err_exit;
17513                         }
17514                 }
17515                 fm = next_fm;
17516         } while (fm);
17517         /* Create tag color rules for all needed fms. */
17518         for (i = 0; i < fm_cnt; i++) {
17519                 void *mtr_action;
17520
17521                 mtr_policy = fm_info[i].fm_policy;
17522                 rte_spinlock_lock(&mtr_policy->sl);
17523                 sub_policy = mtr_policy->sub_policys[domain][0];
17524                 for (j = 0; j < MLX5_MTR_RTE_COLORS; j++) {
17525                         if (mtr_policy->act_cnt[j].fate_action != MLX5_FLOW_FATE_MTR)
17526                                 continue;
17527                         color_rule = mlx5_malloc(MLX5_MEM_ZERO,
17528                                                  sizeof(struct mlx5_sub_policy_color_rule),
17529                                                  0, SOCKET_ID_ANY);
17530                         if (!color_rule) {
17531                                 rte_spinlock_unlock(&mtr_policy->sl);
17532                                 rte_flow_error_set(error, ENOMEM,
17533                                                    RTE_FLOW_ERROR_TYPE_ACTION, NULL,
17534                                                    "No memory to create tag color rule.");
17535                                 goto err_exit;
17536                         }
17537                         color_rule->src_port = src_port;
17538                         next_fm = fm_info[i].next_fm;
17539                         if (mlx5_flow_meter_attach(priv, next_fm, &attr, error)) {
17540                                 mlx5_free(color_rule);
17541                                 rte_spinlock_unlock(&mtr_policy->sl);
17542                                 goto err_exit;
17543                         }
17544                         fm_info[i].tag_rule[j] = color_rule;
17545                         TAILQ_INSERT_TAIL(&sub_policy->color_rules[j], color_rule, next_port);
17546                         /* Prepare to create color rule. */
17547                         mtr_action = (next_fm->color_aware && j == RTE_COLOR_YELLOW) ?
17548                                                                 next_fm->meter_action_y :
17549                                                                 next_fm->meter_action_g;
17550                         next_policy = mlx5_flow_meter_policy_find(dev, next_fm->policy_id, NULL);
17551                         MLX5_ASSERT(next_policy);
17552                         next_sub_policy = next_policy->sub_policys[domain][0];
17553                         tbl_data = container_of(next_sub_policy->tbl_rsc,
17554                                                 struct mlx5_flow_tbl_data_entry, tbl);
17555                         if (mtr_first) {
17556                                 acts.dv_actions[0] = mtr_action;
17557                                 acts.dv_actions[1] = mtr_policy->act_cnt[j].modify_hdr->action;
17558                         } else {
17559                                 acts.dv_actions[0] = mtr_policy->act_cnt[j].modify_hdr->action;
17560                                 acts.dv_actions[1] = mtr_action;
17561                         }
17562                         acts.dv_actions[2] = tbl_data->jump.action;
17563                         acts.actions_n = 3;
17564                         if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx,
17565                                                 MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy,
17566                                                 &attr, true, item, &color_rule->matcher, error)) {
17567                                 rte_spinlock_unlock(&mtr_policy->sl);
17568                                 rte_flow_error_set(error, errno,
17569                                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
17570                                                    "Failed to create hierarchy meter matcher.");
17571                                 goto err_exit;
17572                         }
17573                         if (__flow_dv_create_policy_flow(dev, color_reg_c_idx, (enum rte_color)j,
17574                                                 color_rule->matcher->matcher_object,
17575                                                 acts.actions_n, acts.dv_actions,
17576                                                 true, item, &color_rule->rule, &attr)) {
17577                                 rte_spinlock_unlock(&mtr_policy->sl);
17578                                 rte_flow_error_set(error, errno,
17579                                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
17580                                                    "Failed to create hierarchy meter rule.");
17581                                 goto err_exit;
17582                         }
17583                 }
17584                 rte_spinlock_unlock(&mtr_policy->sl);
17585         }
17586         return 0;
17587 err_exit:
17588         for (i = 0; i < fm_cnt; i++) {
17589                 mtr_policy = fm_info[i].fm_policy;
17590                 rte_spinlock_lock(&mtr_policy->sl);
17591                 sub_policy = mtr_policy->sub_policys[domain][0];
17592                 for (j = 0; j < MLX5_MTR_RTE_COLORS; j++) {
17593                         color_rule = fm_info[i].tag_rule[j];
17594                         if (!color_rule)
17595                                 continue;
17596                         if (color_rule->rule)
17597                                 mlx5_flow_os_destroy_flow(color_rule->rule);
17598                         if (color_rule->matcher) {
17599                                 struct mlx5_flow_tbl_data_entry *tbl =
17600                                         container_of(color_rule->matcher->tbl, typeof(*tbl), tbl);
17601                                 mlx5_list_unregister(tbl->matchers, &color_rule->matcher->entry);
17602                         }
17603                         if (fm_info[i].next_fm)
17604                                 mlx5_flow_meter_detach(priv, fm_info[i].next_fm);
17605                         TAILQ_REMOVE(&sub_policy->color_rules[j], color_rule, next_port);
17606                         mlx5_free(color_rule);
17607                 }
17608                 rte_spinlock_unlock(&mtr_policy->sl);
17609         }
17610         return -rte_errno;
17611 }
17612
17613 /**
17614  * Destroy the sub policy table with RX queue.
17615  *
17616  * @param[in] dev
17617  *   Pointer to Ethernet device.
17618  * @param[in] mtr_policy
17619  *   Pointer to meter policy table.
17620  */
17621 static void
17622 flow_dv_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
17623                                     struct mlx5_flow_meter_policy *mtr_policy)
17624 {
17625         struct mlx5_priv *priv = dev->data->dev_private;
17626         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
17627         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
17628         uint32_t i, j;
17629         uint16_t sub_policy_num, new_policy_num;
17630
17631         rte_spinlock_lock(&mtr_policy->sl);
17632         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
17633                 switch (mtr_policy->act_cnt[i].fate_action) {
17634                 case MLX5_FLOW_FATE_SHARED_RSS:
17635                         sub_policy_num = (mtr_policy->sub_policy_num >>
17636                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
17637                         MLX5_MTR_SUB_POLICY_NUM_MASK;
17638                         new_policy_num = sub_policy_num;
17639                         for (j = 0; j < sub_policy_num; j++) {
17640                                 sub_policy =
17641                                         mtr_policy->sub_policys[domain][j];
17642                                 if (sub_policy) {
17643                                         __flow_dv_destroy_sub_policy_rules(dev,
17644                                                 sub_policy);
17645                                 if (sub_policy !=
17646                                         mtr_policy->sub_policys[domain][0]) {
17647                                         mtr_policy->sub_policys[domain][j] =
17648                                                                 NULL;
17649                                         mlx5_ipool_free
17650                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
17651                                                 sub_policy->idx);
17652                                                 new_policy_num--;
17653                                         }
17654                                 }
17655                         }
17656                         if (new_policy_num != sub_policy_num) {
17657                                 mtr_policy->sub_policy_num &=
17658                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
17659                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
17660                                 mtr_policy->sub_policy_num |=
17661                                 (new_policy_num &
17662                                         MLX5_MTR_SUB_POLICY_NUM_MASK) <<
17663                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
17664                         }
17665                         break;
17666                 case MLX5_FLOW_FATE_QUEUE:
17667                         sub_policy = mtr_policy->sub_policys[domain][0];
17668                         __flow_dv_destroy_sub_policy_rules(dev,
17669                                                            sub_policy);
17670                         break;
17671                 default:
17672                         /*Other actions without queue and do nothing*/
17673                         break;
17674                 }
17675         }
17676         rte_spinlock_unlock(&mtr_policy->sl);
17677 }
17678 /**
17679  * Check whether the DR drop action is supported on the root table or not.
17680  *
17681  * Create a simple flow with DR drop action on root table to validate
17682  * if DR drop action on root table is supported or not.
17683  *
17684  * @param[in] dev
17685  *   Pointer to rte_eth_dev structure.
17686  *
17687  * @return
17688  *   0 on success, a negative errno value otherwise and rte_errno is set.
17689  */
17690 int
17691 mlx5_flow_discover_dr_action_support(struct rte_eth_dev *dev)
17692 {
17693         struct mlx5_priv *priv = dev->data->dev_private;
17694         struct mlx5_dev_ctx_shared *sh = priv->sh;
17695         struct mlx5_flow_dv_match_params mask = {
17696                 .size = sizeof(mask.buf),
17697         };
17698         struct mlx5_flow_dv_match_params value = {
17699                 .size = sizeof(value.buf),
17700         };
17701         struct mlx5dv_flow_matcher_attr dv_attr = {
17702                 .type = IBV_FLOW_ATTR_NORMAL,
17703                 .priority = 0,
17704                 .match_criteria_enable = 0,
17705                 .match_mask = (void *)&mask,
17706         };
17707         struct mlx5_flow_tbl_resource *tbl = NULL;
17708         void *matcher = NULL;
17709         void *flow = NULL;
17710         int ret = -1;
17711
17712         tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL,
17713                                         0, 0, 0, NULL);
17714         if (!tbl)
17715                 goto err;
17716         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
17717         __flow_dv_adjust_buf_size(&mask.size, dv_attr.match_criteria_enable);
17718         ret = mlx5_flow_os_create_flow_matcher(sh->cdev->ctx, &dv_attr,
17719                                                tbl->obj, &matcher);
17720         if (ret)
17721                 goto err;
17722         __flow_dv_adjust_buf_size(&value.size, dv_attr.match_criteria_enable);
17723         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 1,
17724                                        &sh->dr_drop_action, &flow);
17725 err:
17726         /*
17727          * If DR drop action is not supported on root table, flow create will
17728          * be failed with EOPNOTSUPP or EPROTONOSUPPORT.
17729          */
17730         if (!flow) {
17731                 if (matcher &&
17732                     (errno == EPROTONOSUPPORT || errno == EOPNOTSUPP))
17733                         DRV_LOG(INFO, "DR drop action is not supported in root table.");
17734                 else
17735                         DRV_LOG(ERR, "Unexpected error in DR drop action support detection");
17736                 ret = -1;
17737         } else {
17738                 claim_zero(mlx5_flow_os_destroy_flow(flow));
17739         }
17740         if (matcher)
17741                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
17742         if (tbl)
17743                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
17744         return ret;
17745 }
17746
17747 /**
17748  * Validate the batch counter support in root table.
17749  *
17750  * Create a simple flow with invalid counter and drop action on root table to
17751  * validate if batch counter with offset on root table is supported or not.
17752  *
17753  * @param[in] dev
17754  *   Pointer to rte_eth_dev structure.
17755  *
17756  * @return
17757  *   0 on success, a negative errno value otherwise and rte_errno is set.
17758  */
17759 int
17760 mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev)
17761 {
17762         struct mlx5_priv *priv = dev->data->dev_private;
17763         struct mlx5_dev_ctx_shared *sh = priv->sh;
17764         struct mlx5_flow_dv_match_params mask = {
17765                 .size = sizeof(mask.buf),
17766         };
17767         struct mlx5_flow_dv_match_params value = {
17768                 .size = sizeof(value.buf),
17769         };
17770         struct mlx5dv_flow_matcher_attr dv_attr = {
17771                 .type = IBV_FLOW_ATTR_NORMAL | IBV_FLOW_ATTR_FLAGS_EGRESS,
17772                 .priority = 0,
17773                 .match_criteria_enable = 0,
17774                 .match_mask = (void *)&mask,
17775         };
17776         void *actions[2] = { 0 };
17777         struct mlx5_flow_tbl_resource *tbl = NULL;
17778         struct mlx5_devx_obj *dcs = NULL;
17779         void *matcher = NULL;
17780         void *flow = NULL;
17781         int ret = -1;
17782
17783         tbl = flow_dv_tbl_resource_get(dev, 0, 1, 0, false, NULL,
17784                                         0, 0, 0, NULL);
17785         if (!tbl)
17786                 goto err;
17787         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->cdev->ctx, 0x4);
17788         if (!dcs)
17789                 goto err;
17790         ret = mlx5_flow_os_create_flow_action_count(dcs->obj, UINT16_MAX,
17791                                                     &actions[0]);
17792         if (ret)
17793                 goto err;
17794         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
17795         __flow_dv_adjust_buf_size(&mask.size, dv_attr.match_criteria_enable);
17796         ret = mlx5_flow_os_create_flow_matcher(sh->cdev->ctx, &dv_attr,
17797                                                tbl->obj, &matcher);
17798         if (ret)
17799                 goto err;
17800         __flow_dv_adjust_buf_size(&value.size, dv_attr.match_criteria_enable);
17801         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 1,
17802                                        actions, &flow);
17803 err:
17804         /*
17805          * If batch counter with offset is not supported, the driver will not
17806          * validate the invalid offset value, flow create should success.
17807          * In this case, it means batch counter is not supported in root table.
17808          *
17809          * Otherwise, if flow create is failed, counter offset is supported.
17810          */
17811         if (flow) {
17812                 DRV_LOG(INFO, "Batch counter is not supported in root "
17813                               "table. Switch to fallback mode.");
17814                 rte_errno = ENOTSUP;
17815                 ret = -rte_errno;
17816                 claim_zero(mlx5_flow_os_destroy_flow(flow));
17817         } else {
17818                 /* Check matcher to make sure validate fail at flow create. */
17819                 if (!matcher || (matcher && errno != EINVAL))
17820                         DRV_LOG(ERR, "Unexpected error in counter offset "
17821                                      "support detection");
17822                 ret = 0;
17823         }
17824         if (actions[0])
17825                 claim_zero(mlx5_flow_os_destroy_flow_action(actions[0]));
17826         if (matcher)
17827                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
17828         if (tbl)
17829                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
17830         if (dcs)
17831                 claim_zero(mlx5_devx_cmd_destroy(dcs));
17832         return ret;
17833 }
17834
17835 /**
17836  * Query a devx counter.
17837  *
17838  * @param[in] dev
17839  *   Pointer to the Ethernet device structure.
17840  * @param[in] cnt
17841  *   Index to the flow counter.
17842  * @param[in] clear
17843  *   Set to clear the counter statistics.
17844  * @param[out] pkts
17845  *   The statistics value of packets.
17846  * @param[out] bytes
17847  *   The statistics value of bytes.
17848  *
17849  * @return
17850  *   0 on success, otherwise return -1.
17851  */
17852 static int
17853 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
17854                       uint64_t *pkts, uint64_t *bytes, void **action)
17855 {
17856         struct mlx5_priv *priv = dev->data->dev_private;
17857         struct mlx5_flow_counter *cnt;
17858         uint64_t inn_pkts, inn_bytes;
17859         int ret;
17860
17861         if (!priv->sh->cdev->config.devx)
17862                 return -1;
17863
17864         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
17865         if (ret)
17866                 return -1;
17867         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
17868         if (cnt && action)
17869                 *action = cnt->action;
17870
17871         *pkts = inn_pkts - cnt->hits;
17872         *bytes = inn_bytes - cnt->bytes;
17873         if (clear) {
17874                 cnt->hits = inn_pkts;
17875                 cnt->bytes = inn_bytes;
17876         }
17877         return 0;
17878 }
17879
17880 /**
17881  * Get aged-out flows.
17882  *
17883  * @param[in] dev
17884  *   Pointer to the Ethernet device structure.
17885  * @param[in] context
17886  *   The address of an array of pointers to the aged-out flows contexts.
17887  * @param[in] nb_contexts
17888  *   The length of context array pointers.
17889  * @param[out] error
17890  *   Perform verbose error reporting if not NULL. Initialized in case of
17891  *   error only.
17892  *
17893  * @return
17894  *   how many contexts get in success, otherwise negative errno value.
17895  *   if nb_contexts is 0, return the amount of all aged contexts.
17896  *   if nb_contexts is not 0 , return the amount of aged flows reported
17897  *   in the context array.
17898  * @note: only stub for now
17899  */
17900 static int
17901 flow_dv_get_aged_flows(struct rte_eth_dev *dev,
17902                     void **context,
17903                     uint32_t nb_contexts,
17904                     struct rte_flow_error *error)
17905 {
17906         struct mlx5_priv *priv = dev->data->dev_private;
17907         struct mlx5_age_info *age_info;
17908         struct mlx5_age_param *age_param;
17909         struct mlx5_flow_counter *counter;
17910         struct mlx5_aso_age_action *act;
17911         int nb_flows = 0;
17912
17913         if (nb_contexts && !context)
17914                 return rte_flow_error_set(error, EINVAL,
17915                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
17916                                           NULL, "empty context");
17917         age_info = GET_PORT_AGE_INFO(priv);
17918         rte_spinlock_lock(&age_info->aged_sl);
17919         LIST_FOREACH(act, &age_info->aged_aso, next) {
17920                 nb_flows++;
17921                 if (nb_contexts) {
17922                         context[nb_flows - 1] =
17923                                                 act->age_params.context;
17924                         if (!(--nb_contexts))
17925                                 break;
17926                 }
17927         }
17928         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
17929                 nb_flows++;
17930                 if (nb_contexts) {
17931                         age_param = MLX5_CNT_TO_AGE(counter);
17932                         context[nb_flows - 1] = age_param->context;
17933                         if (!(--nb_contexts))
17934                                 break;
17935                 }
17936         }
17937         rte_spinlock_unlock(&age_info->aged_sl);
17938         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
17939         return nb_flows;
17940 }
17941
17942 /*
17943  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
17944  */
17945 static uint32_t
17946 flow_dv_counter_allocate(struct rte_eth_dev *dev)
17947 {
17948         return flow_dv_counter_alloc(dev, 0);
17949 }
17950
17951 /**
17952  * Validate indirect action.
17953  * Dispatcher for action type specific validation.
17954  *
17955  * @param[in] dev
17956  *   Pointer to the Ethernet device structure.
17957  * @param[in] conf
17958  *   Indirect action configuration.
17959  * @param[in] action
17960  *   The indirect action object to validate.
17961  * @param[out] error
17962  *   Perform verbose error reporting if not NULL. Initialized in case of
17963  *   error only.
17964  *
17965  * @return
17966  *   0 on success, otherwise negative errno value.
17967  */
17968 int
17969 flow_dv_action_validate(struct rte_eth_dev *dev,
17970                         const struct rte_flow_indir_action_conf *conf,
17971                         const struct rte_flow_action *action,
17972                         struct rte_flow_error *err)
17973 {
17974         struct mlx5_priv *priv = dev->data->dev_private;
17975
17976         RTE_SET_USED(conf);
17977         switch (action->type) {
17978         case RTE_FLOW_ACTION_TYPE_RSS:
17979                 /*
17980                  * priv->obj_ops is set according to driver capabilities.
17981                  * When DevX capabilities are
17982                  * sufficient, it is set to devx_obj_ops.
17983                  * Otherwise, it is set to ibv_obj_ops.
17984                  * ibv_obj_ops doesn't support ind_table_modify operation.
17985                  * In this case the indirect RSS action can't be used.
17986                  */
17987                 if (priv->obj_ops.ind_table_modify == NULL)
17988                         return rte_flow_error_set
17989                                         (err, ENOTSUP,
17990                                          RTE_FLOW_ERROR_TYPE_ACTION,
17991                                          NULL,
17992                                          "Indirect RSS action not supported");
17993                 return mlx5_validate_action_rss(dev, action, err);
17994         case RTE_FLOW_ACTION_TYPE_AGE:
17995                 if (!priv->sh->aso_age_mng)
17996                         return rte_flow_error_set(err, ENOTSUP,
17997                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
17998                                                 NULL,
17999                                                 "Indirect age action not supported");
18000                 return flow_dv_validate_action_age(0, action, dev, err);
18001         case RTE_FLOW_ACTION_TYPE_COUNT:
18002                 return flow_dv_validate_action_count(dev, true, 0, NULL, err);
18003         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
18004                 if (!priv->sh->ct_aso_en)
18005                         return rte_flow_error_set(err, ENOTSUP,
18006                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
18007                                         "ASO CT is not supported");
18008                 return mlx5_validate_action_ct(dev, action->conf, err);
18009         default:
18010                 return rte_flow_error_set(err, ENOTSUP,
18011                                           RTE_FLOW_ERROR_TYPE_ACTION,
18012                                           NULL,
18013                                           "action type not supported");
18014         }
18015 }
18016
18017 /*
18018  * Check if the RSS configurations for colors of a meter policy match
18019  * each other, except the queues.
18020  *
18021  * @param[in] r1
18022  *   Pointer to the first RSS flow action.
18023  * @param[in] r2
18024  *   Pointer to the second RSS flow action.
18025  *
18026  * @return
18027  *   0 on match, 1 on conflict.
18028  */
18029 static inline int
18030 flow_dv_mtr_policy_rss_compare(const struct rte_flow_action_rss *r1,
18031                                const struct rte_flow_action_rss *r2)
18032 {
18033         if (r1 == NULL || r2 == NULL)
18034                 return 0;
18035         if (!(r1->level <= 1 && r2->level <= 1) &&
18036             !(r1->level > 1 && r2->level > 1))
18037                 return 1;
18038         if (r1->types != r2->types &&
18039             !((r1->types == 0 || r1->types == RTE_ETH_RSS_IP) &&
18040               (r2->types == 0 || r2->types == RTE_ETH_RSS_IP)))
18041                 return 1;
18042         if (r1->key || r2->key) {
18043                 const void *key1 = r1->key ? r1->key : rss_hash_default_key;
18044                 const void *key2 = r2->key ? r2->key : rss_hash_default_key;
18045
18046                 if (memcmp(key1, key2, MLX5_RSS_HASH_KEY_LEN))
18047                         return 1;
18048         }
18049         return 0;
18050 }
18051
18052 /**
18053  * Validate the meter hierarchy chain for meter policy.
18054  *
18055  * @param[in] dev
18056  *   Pointer to the Ethernet device structure.
18057  * @param[in] meter_id
18058  *   Meter id.
18059  * @param[in] action_flags
18060  *   Holds the actions detected until now.
18061  * @param[out] is_rss
18062  *   Is RSS or not.
18063  * @param[out] hierarchy_domain
18064  *   The domain bitmap for hierarchy policy.
18065  * @param[out] error
18066  *   Perform verbose error reporting if not NULL. Initialized in case of
18067  *   error only.
18068  *
18069  * @return
18070  *   0 on success, otherwise negative errno value with error set.
18071  */
18072 static int
18073 flow_dv_validate_policy_mtr_hierarchy(struct rte_eth_dev *dev,
18074                                   uint32_t meter_id,
18075                                   uint64_t action_flags,
18076                                   bool *is_rss,
18077                                   uint8_t *hierarchy_domain,
18078                                   struct rte_mtr_error *error)
18079 {
18080         struct mlx5_priv *priv = dev->data->dev_private;
18081         struct mlx5_flow_meter_info *fm;
18082         struct mlx5_flow_meter_policy *policy;
18083         uint8_t cnt = 1;
18084
18085         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
18086                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
18087                 return -rte_mtr_error_set(error, EINVAL,
18088                                         RTE_MTR_ERROR_TYPE_POLICER_ACTION_GREEN,
18089                                         NULL,
18090                                         "Multiple fate actions not supported.");
18091         *hierarchy_domain = 0;
18092         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
18093         while (true) {
18094                 if (!fm)
18095                         return -rte_mtr_error_set(error, EINVAL,
18096                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
18097                                         "Meter not found in meter hierarchy.");
18098                 if (fm->def_policy)
18099                         return -rte_mtr_error_set(error, EINVAL,
18100                                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
18101                         "Non termination meter not supported in hierarchy.");
18102                 if (!fm->shared)
18103                         return -rte_mtr_error_set(error, EINVAL,
18104                                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
18105                                         "Only shared meter supported in hierarchy.");
18106                 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
18107                 MLX5_ASSERT(policy);
18108                 /**
18109                  * Only inherit the supported domains of the first meter in
18110                  * hierarchy.
18111                  * One meter supports at least one domain.
18112                  */
18113                 if (!*hierarchy_domain) {
18114                         if (policy->transfer)
18115                                 *hierarchy_domain |=
18116                                                 MLX5_MTR_DOMAIN_TRANSFER_BIT;
18117                         if (policy->ingress)
18118                                 *hierarchy_domain |=
18119                                                 MLX5_MTR_DOMAIN_INGRESS_BIT;
18120                         if (policy->egress)
18121                                 *hierarchy_domain |= MLX5_MTR_DOMAIN_EGRESS_BIT;
18122                 }
18123                 if (!policy->is_hierarchy) {
18124                         *is_rss = policy->is_rss;
18125                         break;
18126                 }
18127                 rte_spinlock_lock(&policy->sl);
18128                 fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, NULL);
18129                 rte_spinlock_unlock(&policy->sl);
18130                 if (++cnt >= MLX5_MTR_CHAIN_MAX_NUM)
18131                         return -rte_mtr_error_set(error, EINVAL,
18132                                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
18133                                         "Exceed max hierarchy meter number.");
18134         }
18135         return 0;
18136 }
18137
18138 /**
18139  * Validate meter policy actions.
18140  * Dispatcher for action type specific validation.
18141  *
18142  * @param[in] dev
18143  *   Pointer to the Ethernet device structure.
18144  * @param[in] action
18145  *   The meter policy action object to validate.
18146  * @param[in] attr
18147  *   Attributes of flow to determine steering domain.
18148  * @param[out] error
18149  *   Perform verbose error reporting if not NULL. Initialized in case of
18150  *   error only.
18151  *
18152  * @return
18153  *   0 on success, otherwise negative errno value.
18154  */
18155 static int
18156 flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev,
18157                         const struct rte_flow_action *actions[RTE_COLORS],
18158                         struct rte_flow_attr *attr,
18159                         bool *is_rss,
18160                         uint8_t *domain_bitmap,
18161                         uint8_t *policy_mode,
18162                         struct rte_mtr_error *error)
18163 {
18164         struct mlx5_priv *priv = dev->data->dev_private;
18165         struct mlx5_sh_config *dev_conf = &priv->sh->config;
18166         const struct rte_flow_action *act;
18167         uint64_t action_flags[RTE_COLORS] = {0};
18168         int actions_n;
18169         int i, ret;
18170         struct rte_flow_error flow_err;
18171         uint8_t domain_color[RTE_COLORS] = {0};
18172         uint8_t def_domain = MLX5_MTR_ALL_DOMAIN_BIT;
18173         uint8_t hierarchy_domain = 0;
18174         const struct rte_flow_action_meter *mtr;
18175         const struct rte_flow_action_meter *next_mtr = NULL;
18176         bool def_green = false;
18177         bool def_yellow = false;
18178         const struct rte_flow_action_rss *rss_color[RTE_COLORS] = {NULL};
18179
18180         if (!dev_conf->dv_esw_en)
18181                 def_domain &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
18182         *domain_bitmap = def_domain;
18183         /* Red color could only support DROP action. */
18184         if (!actions[RTE_COLOR_RED] ||
18185             actions[RTE_COLOR_RED]->type != RTE_FLOW_ACTION_TYPE_DROP)
18186                 return -rte_mtr_error_set(error, ENOTSUP,
18187                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
18188                                 NULL, "Red color only supports drop action.");
18189         /*
18190          * Check default policy actions:
18191          * Green / Yellow: no action, Red: drop action
18192          * Either G or Y will trigger default policy actions to be created.
18193          */
18194         if (!actions[RTE_COLOR_GREEN] ||
18195             actions[RTE_COLOR_GREEN]->type == RTE_FLOW_ACTION_TYPE_END)
18196                 def_green = true;
18197         if (!actions[RTE_COLOR_YELLOW] ||
18198             actions[RTE_COLOR_YELLOW]->type == RTE_FLOW_ACTION_TYPE_END)
18199                 def_yellow = true;
18200         if (def_green && def_yellow) {
18201                 *policy_mode = MLX5_MTR_POLICY_MODE_DEF;
18202                 return 0;
18203         } else if (!def_green && def_yellow) {
18204                 *policy_mode = MLX5_MTR_POLICY_MODE_OG;
18205         } else if (def_green && !def_yellow) {
18206                 *policy_mode = MLX5_MTR_POLICY_MODE_OY;
18207         } else {
18208                 *policy_mode = MLX5_MTR_POLICY_MODE_ALL;
18209         }
18210         /* Set to empty string in case of NULL pointer access by user. */
18211         flow_err.message = "";
18212         for (i = 0; i < RTE_COLORS; i++) {
18213                 act = actions[i];
18214                 for (action_flags[i] = 0, actions_n = 0;
18215                      act && act->type != RTE_FLOW_ACTION_TYPE_END;
18216                      act++) {
18217                         if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
18218                                 return -rte_mtr_error_set(error, ENOTSUP,
18219                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
18220                                           NULL, "too many actions");
18221                         switch (act->type) {
18222                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
18223                         case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
18224                                 if (!dev_conf->dv_esw_en)
18225                                         return -rte_mtr_error_set(error,
18226                                         ENOTSUP,
18227                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18228                                         NULL, "PORT action validate check"
18229                                         " fail for ESW disable");
18230                                 ret = flow_dv_validate_action_port_id(dev,
18231                                                 action_flags[i],
18232                                                 act, attr, &flow_err);
18233                                 if (ret)
18234                                         return -rte_mtr_error_set(error,
18235                                         ENOTSUP,
18236                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18237                                         NULL, flow_err.message ?
18238                                         flow_err.message :
18239                                         "PORT action validate check fail");
18240                                 ++actions_n;
18241                                 action_flags[i] |= MLX5_FLOW_ACTION_PORT_ID;
18242                                 break;
18243                         case RTE_FLOW_ACTION_TYPE_MARK:
18244                                 ret = flow_dv_validate_action_mark(dev, act,
18245                                                            action_flags[i],
18246                                                            attr, &flow_err);
18247                                 if (ret < 0)
18248                                         return -rte_mtr_error_set(error,
18249                                         ENOTSUP,
18250                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18251                                         NULL, flow_err.message ?
18252                                         flow_err.message :
18253                                         "Mark action validate check fail");
18254                                 if (dev_conf->dv_xmeta_en !=
18255                                         MLX5_XMETA_MODE_LEGACY)
18256                                         return -rte_mtr_error_set(error,
18257                                         ENOTSUP,
18258                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18259                                         NULL, "Extend MARK action is "
18260                                         "not supported. Please try use "
18261                                         "default policy for meter.");
18262                                 action_flags[i] |= MLX5_FLOW_ACTION_MARK;
18263                                 ++actions_n;
18264                                 break;
18265                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
18266                                 ret = flow_dv_validate_action_set_tag(dev,
18267                                                         act, action_flags[i],
18268                                                         attr, &flow_err);
18269                                 if (ret)
18270                                         return -rte_mtr_error_set(error,
18271                                         ENOTSUP,
18272                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18273                                         NULL, flow_err.message ?
18274                                         flow_err.message :
18275                                         "Set tag action validate check fail");
18276                                 action_flags[i] |= MLX5_FLOW_ACTION_SET_TAG;
18277                                 ++actions_n;
18278                                 break;
18279                         case RTE_FLOW_ACTION_TYPE_DROP:
18280                                 ret = mlx5_flow_validate_action_drop
18281                                         (action_flags[i], attr, &flow_err);
18282                                 if (ret < 0)
18283                                         return -rte_mtr_error_set(error,
18284                                         ENOTSUP,
18285                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18286                                         NULL, flow_err.message ?
18287                                         flow_err.message :
18288                                         "Drop action validate check fail");
18289                                 action_flags[i] |= MLX5_FLOW_ACTION_DROP;
18290                                 ++actions_n;
18291                                 break;
18292                         case RTE_FLOW_ACTION_TYPE_QUEUE:
18293                                 /*
18294                                  * Check whether extensive
18295                                  * metadata feature is engaged.
18296                                  */
18297                                 if (dev_conf->dv_flow_en &&
18298                                     (dev_conf->dv_xmeta_en !=
18299                                      MLX5_XMETA_MODE_LEGACY) &&
18300                                     mlx5_flow_ext_mreg_supported(dev))
18301                                         return -rte_mtr_error_set(error,
18302                                           ENOTSUP,
18303                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
18304                                           NULL, "Queue action with meta "
18305                                           "is not supported. Please try use "
18306                                           "default policy for meter.");
18307                                 ret = mlx5_flow_validate_action_queue(act,
18308                                                         action_flags[i], dev,
18309                                                         attr, &flow_err);
18310                                 if (ret < 0)
18311                                         return -rte_mtr_error_set(error,
18312                                           ENOTSUP,
18313                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
18314                                           NULL, flow_err.message ?
18315                                           flow_err.message :
18316                                           "Queue action validate check fail");
18317                                 action_flags[i] |= MLX5_FLOW_ACTION_QUEUE;
18318                                 ++actions_n;
18319                                 break;
18320                         case RTE_FLOW_ACTION_TYPE_RSS:
18321                                 if (dev_conf->dv_flow_en &&
18322                                     (dev_conf->dv_xmeta_en !=
18323                                      MLX5_XMETA_MODE_LEGACY) &&
18324                                     mlx5_flow_ext_mreg_supported(dev))
18325                                         return -rte_mtr_error_set(error,
18326                                           ENOTSUP,
18327                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
18328                                           NULL, "RSS action with meta "
18329                                           "is not supported. Please try use "
18330                                           "default policy for meter.");
18331                                 ret = mlx5_validate_action_rss(dev, act,
18332                                                                &flow_err);
18333                                 if (ret < 0)
18334                                         return -rte_mtr_error_set(error,
18335                                           ENOTSUP,
18336                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
18337                                           NULL, flow_err.message ?
18338                                           flow_err.message :
18339                                           "RSS action validate check fail");
18340                                 action_flags[i] |= MLX5_FLOW_ACTION_RSS;
18341                                 ++actions_n;
18342                                 /* Either G or Y will set the RSS. */
18343                                 rss_color[i] = act->conf;
18344                                 break;
18345                         case RTE_FLOW_ACTION_TYPE_JUMP:
18346                                 ret = flow_dv_validate_action_jump(dev,
18347                                         NULL, act, action_flags[i],
18348                                         attr, true, &flow_err);
18349                                 if (ret)
18350                                         return -rte_mtr_error_set(error,
18351                                           ENOTSUP,
18352                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
18353                                           NULL, flow_err.message ?
18354                                           flow_err.message :
18355                                           "Jump action validate check fail");
18356                                 ++actions_n;
18357                                 action_flags[i] |= MLX5_FLOW_ACTION_JUMP;
18358                                 break;
18359                         case RTE_FLOW_ACTION_TYPE_METER:
18360                                 mtr = act->conf;
18361                                 if (next_mtr && next_mtr->mtr_id != mtr->mtr_id)
18362                                         return -rte_mtr_error_set(error, ENOTSUP,
18363                                                 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
18364                                                 "Green and Yellow must use the same meter.");
18365                                 ret = flow_dv_validate_policy_mtr_hierarchy(dev,
18366                                                         mtr->mtr_id,
18367                                                         action_flags[i],
18368                                                         is_rss,
18369                                                         &hierarchy_domain,
18370                                                         error);
18371                                 if (ret)
18372                                         return ret;
18373                                 ++actions_n;
18374                                 action_flags[i] |=
18375                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
18376                                 next_mtr = mtr;
18377                                 break;
18378                         default:
18379                                 return -rte_mtr_error_set(error, ENOTSUP,
18380                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18381                                         NULL,
18382                                         "Doesn't support optional action");
18383                         }
18384                 }
18385                 if (action_flags[i] & MLX5_FLOW_ACTION_PORT_ID) {
18386                         domain_color[i] = MLX5_MTR_DOMAIN_TRANSFER_BIT;
18387                 } else if ((action_flags[i] &
18388                           (MLX5_FLOW_ACTION_RSS | MLX5_FLOW_ACTION_QUEUE)) ||
18389                           (action_flags[i] & MLX5_FLOW_ACTION_MARK)) {
18390                         /*
18391                          * Only support MLX5_XMETA_MODE_LEGACY
18392                          * so MARK action is only in ingress domain.
18393                          */
18394                         domain_color[i] = MLX5_MTR_DOMAIN_INGRESS_BIT;
18395                 } else {
18396                         domain_color[i] = def_domain;
18397                         if (action_flags[i] &&
18398                             !(action_flags[i] & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
18399                                 domain_color[i] &=
18400                                 ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
18401                 }
18402                 if (action_flags[i] &
18403                     MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
18404                         domain_color[i] &= hierarchy_domain;
18405                 /*
18406                  * Non-termination actions only support NIC Tx domain.
18407                  * The adjustion should be skipped when there is no
18408                  * action or only END is provided. The default domains
18409                  * bit-mask is set to find the MIN intersection.
18410                  * The action flags checking should also be skipped.
18411                  */
18412                 if ((def_green && i == RTE_COLOR_GREEN) ||
18413                     (def_yellow && i == RTE_COLOR_YELLOW))
18414                         continue;
18415                 /*
18416                  * Validate the drop action mutual exclusion
18417                  * with other actions. Drop action is mutually-exclusive
18418                  * with any other action, except for Count action.
18419                  */
18420                 if ((action_flags[i] & MLX5_FLOW_ACTION_DROP) &&
18421                     (action_flags[i] & ~MLX5_FLOW_ACTION_DROP)) {
18422                         return -rte_mtr_error_set(error, ENOTSUP,
18423                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
18424                                 NULL, "Drop action is mutually-exclusive "
18425                                 "with any other action");
18426                 }
18427                 /* Eswitch has few restrictions on using items and actions */
18428                 if (domain_color[i] & MLX5_MTR_DOMAIN_TRANSFER_BIT) {
18429                         if (!mlx5_flow_ext_mreg_supported(dev) &&
18430                             action_flags[i] & MLX5_FLOW_ACTION_MARK)
18431                                 return -rte_mtr_error_set(error, ENOTSUP,
18432                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18433                                         NULL, "unsupported action MARK");
18434                         if (action_flags[i] & MLX5_FLOW_ACTION_QUEUE)
18435                                 return -rte_mtr_error_set(error, ENOTSUP,
18436                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18437                                         NULL, "unsupported action QUEUE");
18438                         if (action_flags[i] & MLX5_FLOW_ACTION_RSS)
18439                                 return -rte_mtr_error_set(error, ENOTSUP,
18440                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18441                                         NULL, "unsupported action RSS");
18442                         if (!(action_flags[i] & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
18443                                 return -rte_mtr_error_set(error, ENOTSUP,
18444                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
18445                                         NULL, "no fate action is found");
18446                 } else {
18447                         if (!(action_flags[i] & MLX5_FLOW_FATE_ACTIONS) &&
18448                             (domain_color[i] & MLX5_MTR_DOMAIN_INGRESS_BIT)) {
18449                                 if ((domain_color[i] &
18450                                      MLX5_MTR_DOMAIN_EGRESS_BIT))
18451                                         domain_color[i] =
18452                                                 MLX5_MTR_DOMAIN_EGRESS_BIT;
18453                                 else
18454                                         return -rte_mtr_error_set(error,
18455                                                 ENOTSUP,
18456                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
18457                                                 NULL,
18458                                                 "no fate action is found");
18459                         }
18460                 }
18461         }
18462         if (next_mtr && *policy_mode == MLX5_MTR_POLICY_MODE_ALL) {
18463                 if (!(action_flags[RTE_COLOR_GREEN] & action_flags[RTE_COLOR_YELLOW] &
18464                       MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY))
18465                         return -rte_mtr_error_set(error, EINVAL, RTE_MTR_ERROR_TYPE_METER_POLICY,
18466                                                   NULL,
18467                                                   "Meter hierarchy supports meter action only.");
18468         }
18469         /* If both colors have RSS, the attributes should be the same. */
18470         if (flow_dv_mtr_policy_rss_compare(rss_color[RTE_COLOR_GREEN],
18471                                            rss_color[RTE_COLOR_YELLOW]))
18472                 return -rte_mtr_error_set(error, EINVAL,
18473                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
18474                                           NULL, "policy RSS attr conflict");
18475         if (rss_color[RTE_COLOR_GREEN] || rss_color[RTE_COLOR_YELLOW])
18476                 *is_rss = true;
18477         /* "domain_color[C]" is non-zero for each color, default is ALL. */
18478         if (!def_green && !def_yellow &&
18479             domain_color[RTE_COLOR_GREEN] != domain_color[RTE_COLOR_YELLOW] &&
18480             !(action_flags[RTE_COLOR_GREEN] & MLX5_FLOW_ACTION_DROP) &&
18481             !(action_flags[RTE_COLOR_YELLOW] & MLX5_FLOW_ACTION_DROP))
18482                 return -rte_mtr_error_set(error, EINVAL,
18483                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
18484                                           NULL, "policy domains conflict");
18485         /*
18486          * At least one color policy is listed in the actions, the domains
18487          * to be supported should be the intersection.
18488          */
18489         *domain_bitmap = domain_color[RTE_COLOR_GREEN] &
18490                          domain_color[RTE_COLOR_YELLOW];
18491         return 0;
18492 }
18493
18494 static int
18495 flow_dv_sync_domain(struct rte_eth_dev *dev, uint32_t domains, uint32_t flags)
18496 {
18497         struct mlx5_priv *priv = dev->data->dev_private;
18498         int ret = 0;
18499
18500         if ((domains & MLX5_DOMAIN_BIT_NIC_RX) && priv->sh->rx_domain != NULL) {
18501                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->rx_domain,
18502                                                 flags);
18503                 if (ret != 0)
18504                         return ret;
18505         }
18506         if ((domains & MLX5_DOMAIN_BIT_NIC_TX) && priv->sh->tx_domain != NULL) {
18507                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->tx_domain, flags);
18508                 if (ret != 0)
18509                         return ret;
18510         }
18511         if ((domains & MLX5_DOMAIN_BIT_FDB) && priv->sh->fdb_domain != NULL) {
18512                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->fdb_domain, flags);
18513                 if (ret != 0)
18514                         return ret;
18515         }
18516         return 0;
18517 }
18518
18519 /**
18520  * Discover the number of available flow priorities
18521  * by trying to create a flow with the highest priority value
18522  * for each possible number.
18523  *
18524  * @param[in] dev
18525  *   Ethernet device.
18526  * @param[in] vprio
18527  *   List of possible number of available priorities.
18528  * @param[in] vprio_n
18529  *   Size of @p vprio array.
18530  * @return
18531  *   On success, number of available flow priorities.
18532  *   On failure, a negative errno-style code and rte_errno is set.
18533  */
18534 static int
18535 flow_dv_discover_priorities(struct rte_eth_dev *dev,
18536                             const uint16_t *vprio, int vprio_n)
18537 {
18538         struct mlx5_priv *priv = dev->data->dev_private;
18539         struct mlx5_indexed_pool *pool = priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW];
18540         struct rte_flow_item_eth eth;
18541         struct rte_flow_item item = {
18542                 .type = RTE_FLOW_ITEM_TYPE_ETH,
18543                 .spec = &eth,
18544                 .mask = &eth,
18545         };
18546         struct mlx5_flow_dv_matcher matcher = {
18547                 .mask = {
18548                         .size = sizeof(matcher.mask.buf),
18549                 },
18550         };
18551         union mlx5_flow_tbl_key tbl_key;
18552         struct mlx5_flow flow;
18553         void *action;
18554         struct rte_flow_error error;
18555         uint8_t misc_mask;
18556         int i, err, ret = -ENOTSUP;
18557
18558         /*
18559          * Prepare a flow with a catch-all pattern and a drop action.
18560          * Use drop queue, because shared drop action may be unavailable.
18561          */
18562         action = priv->drop_queue.hrxq->action;
18563         if (action == NULL) {
18564                 DRV_LOG(ERR, "Priority discovery requires a drop action");
18565                 rte_errno = ENOTSUP;
18566                 return -rte_errno;
18567         }
18568         memset(&flow, 0, sizeof(flow));
18569         flow.handle = mlx5_ipool_zmalloc(pool, &flow.handle_idx);
18570         if (flow.handle == NULL) {
18571                 DRV_LOG(ERR, "Cannot create flow handle");
18572                 rte_errno = ENOMEM;
18573                 return -rte_errno;
18574         }
18575         flow.ingress = true;
18576         flow.dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
18577         flow.dv.actions[0] = action;
18578         flow.dv.actions_n = 1;
18579         memset(&eth, 0, sizeof(eth));
18580         flow_dv_translate_item_eth(matcher.mask.buf, flow.dv.value.buf,
18581                                    &item, /* inner */ false, /* group */ 0);
18582         matcher.crc = rte_raw_cksum(matcher.mask.buf, matcher.mask.size);
18583         for (i = 0; i < vprio_n; i++) {
18584                 /* Configure the next proposed maximum priority. */
18585                 matcher.priority = vprio[i] - 1;
18586                 memset(&tbl_key, 0, sizeof(tbl_key));
18587                 err = flow_dv_matcher_register(dev, &matcher, &tbl_key, &flow,
18588                                                /* tunnel */ NULL,
18589                                                /* group */ 0,
18590                                                &error);
18591                 if (err != 0) {
18592                         /* This action is pure SW and must always succeed. */
18593                         DRV_LOG(ERR, "Cannot register matcher");
18594                         ret = -rte_errno;
18595                         break;
18596                 }
18597                 /* Try to apply the flow to HW. */
18598                 misc_mask = flow_dv_matcher_enable(flow.dv.value.buf);
18599                 __flow_dv_adjust_buf_size(&flow.dv.value.size, misc_mask);
18600                 err = mlx5_flow_os_create_flow
18601                                 (flow.handle->dvh.matcher->matcher_object,
18602                                  (void *)&flow.dv.value, flow.dv.actions_n,
18603                                  flow.dv.actions, &flow.handle->drv_flow);
18604                 if (err == 0) {
18605                         claim_zero(mlx5_flow_os_destroy_flow
18606                                                 (flow.handle->drv_flow));
18607                         flow.handle->drv_flow = NULL;
18608                 }
18609                 claim_zero(flow_dv_matcher_release(dev, flow.handle));
18610                 if (err != 0)
18611                         break;
18612                 ret = vprio[i];
18613         }
18614         mlx5_ipool_free(pool, flow.handle_idx);
18615         /* Set rte_errno if no expected priority value matched. */
18616         if (ret < 0)
18617                 rte_errno = -ret;
18618         return ret;
18619 }
18620
18621 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
18622         .validate = flow_dv_validate,
18623         .prepare = flow_dv_prepare,
18624         .translate = flow_dv_translate,
18625         .apply = flow_dv_apply,
18626         .remove = flow_dv_remove,
18627         .destroy = flow_dv_destroy,
18628         .query = flow_dv_query,
18629         .create_mtr_tbls = flow_dv_create_mtr_tbls,
18630         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbls,
18631         .destroy_mtr_drop_tbls = flow_dv_destroy_mtr_drop_tbls,
18632         .create_meter = flow_dv_mtr_alloc,
18633         .free_meter = flow_dv_aso_mtr_release_to_pool,
18634         .validate_mtr_acts = flow_dv_validate_mtr_policy_acts,
18635         .create_mtr_acts = flow_dv_create_mtr_policy_acts,
18636         .destroy_mtr_acts = flow_dv_destroy_mtr_policy_acts,
18637         .create_policy_rules = flow_dv_create_policy_rules,
18638         .destroy_policy_rules = flow_dv_destroy_policy_rules,
18639         .create_def_policy = flow_dv_create_def_policy,
18640         .destroy_def_policy = flow_dv_destroy_def_policy,
18641         .meter_sub_policy_rss_prepare = flow_dv_meter_sub_policy_rss_prepare,
18642         .meter_hierarchy_rule_create = flow_dv_meter_hierarchy_rule_create,
18643         .destroy_sub_policy_with_rxq = flow_dv_destroy_sub_policy_with_rxq,
18644         .counter_alloc = flow_dv_counter_allocate,
18645         .counter_free = flow_dv_counter_free,
18646         .counter_query = flow_dv_counter_query,
18647         .get_aged_flows = flow_dv_get_aged_flows,
18648         .action_validate = flow_dv_action_validate,
18649         .action_create = flow_dv_action_create,
18650         .action_destroy = flow_dv_action_destroy,
18651         .action_update = flow_dv_action_update,
18652         .action_query = flow_dv_action_query,
18653         .sync_domain = flow_dv_sync_domain,
18654         .discover_priorities = flow_dv_discover_priorities,
18655         .item_create = flow_dv_item_create,
18656         .item_release = flow_dv_item_release,
18657 };
18658
18659 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */