drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_bus_pci.h>
  19 #include <rte_ip.h>
  20 #include <rte_gre.h>
  21 #include <rte_vxlan.h>
  22 #include <rte_gtp.h>
  23 #include <rte_eal_paging.h>
  24 #include <rte_mpls.h>
  25 #include <rte_mtr.h>
  26 #include <rte_mtr_driver.h>
  27 #include <rte_tailq.h>
  28
  29 #include <mlx5_glue.h>
  30 #include <mlx5_devx_cmds.h>
  31 #include <mlx5_prm.h>
  32 #include <mlx5_malloc.h>
  33
  34 #include "mlx5_defs.h"
  35 #include "mlx5.h"
  36 #include "mlx5_common_os.h"
  37 #include "mlx5_flow.h"
  38 #include "mlx5_flow_os.h"
  39 #include "mlx5_rx.h"
  40 #include "mlx5_tx.h"
  41 #include "rte_pmd_mlx5.h"
  42
  43 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  44
  45 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  46 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR_ESWITCH
  50 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  51 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  52 #endif
  53 #endif
  54
  55 #ifndef HAVE_MLX5DV_DR
  56 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  57 #endif
  58
  59 /* VLAN header definitions */
  60 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  61 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  62 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  63 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  64 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  65
  66 union flow_dv_attr {
  67         struct {
  68                 uint32_t valid:1;
  69                 uint32_t ipv4:1;
  70                 uint32_t ipv6:1;
  71                 uint32_t tcp:1;
  72                 uint32_t udp:1;
  73                 uint32_t reserved:27;
  74         };
  75         uint32_t attr;
  76 };
  77
  78 static int
  79 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
  80                              struct mlx5_flow_tbl_resource *tbl);
  81
  82 static int
  83 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  84                                      uint32_t encap_decap_idx);
  85
  86 static int
  87 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  88                                         uint32_t port_id);
  89 static void
  90 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss);
  91
  92 static int
  93 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
  94                                   uint32_t rix_jump);
  95
  96 static int16_t
  97 flow_dv_get_esw_manager_vport_id(struct rte_eth_dev *dev)
  98 {
  99         struct mlx5_priv *priv = dev->data->dev_private;
 100
 101         if (priv->pci_dev == NULL)
 102                 return 0;
 103         switch (priv->pci_dev->id.device_id) {
 104         case PCI_DEVICE_ID_MELLANOX_CONNECTX5BF:
 105         case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF:
 106         case PCI_DEVICE_ID_MELLANOX_CONNECTX7BF:
 107                 return (int16_t)0xfffe;
 108         default:
 109                 return 0;
 110         }
 111 }
 112
 113 /**
 114  * Initialize flow attributes structure according to flow items' types.
 115  *
 116  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
 117  * mode. For tunnel mode, the items to be modified are the outermost ones.
 118  *
 119  * @param[in] item
 120  *   Pointer to item specification.
 121  * @param[out] attr
 122  *   Pointer to flow attributes structure.
 123  * @param[in] dev_flow
 124  *   Pointer to the sub flow.
 125  * @param[in] tunnel_decap
 126  *   Whether action is after tunnel decapsulation.
 127  */
 128 static void
 129 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 130                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 131 {
 132         uint64_t layers = dev_flow->handle->layers;
 133
 134         /*
 135          * If layers is already initialized, it means this dev_flow is the
 136          * suffix flow, the layers flags is set by the prefix flow. Need to
 137          * use the layer flags from prefix flow as the suffix flow may not
 138          * have the user defined items as the flow is split.
 139          */
 140         if (layers) {
 141                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 142                         attr->ipv4 = 1;
 143                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 144                         attr->ipv6 = 1;
 145                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 146                         attr->tcp = 1;
 147                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 148                         attr->udp = 1;
 149                 attr->valid = 1;
 150                 return;
 151         }
 152         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 153                 uint8_t next_protocol = 0xff;
 154                 switch (item->type) {
 155                 case RTE_FLOW_ITEM_TYPE_GRE:
 156                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 157                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 158                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 159                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 160                 case RTE_FLOW_ITEM_TYPE_MPLS:
 161                         if (tunnel_decap)
 162                                 attr->attr = 0;
 163                         break;
 164                 case RTE_FLOW_ITEM_TYPE_IPV4:
 165                         if (!attr->ipv6)
 166                                 attr->ipv4 = 1;
 167                         if (item->mask != NULL &&
 168                             ((const struct rte_flow_item_ipv4 *)
 169                             item->mask)->hdr.next_proto_id)
 170                                 next_protocol =
 171                                     ((const struct rte_flow_item_ipv4 *)
 172                                       (item->spec))->hdr.next_proto_id &
 173                                     ((const struct rte_flow_item_ipv4 *)
 174                                       (item->mask))->hdr.next_proto_id;
 175                         if ((next_protocol == IPPROTO_IPIP ||
 176                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 177                                 attr->attr = 0;
 178                         break;
 179                 case RTE_FLOW_ITEM_TYPE_IPV6:
 180                         if (!attr->ipv4)
 181                                 attr->ipv6 = 1;
 182                         if (item->mask != NULL &&
 183                             ((const struct rte_flow_item_ipv6 *)
 184                             item->mask)->hdr.proto)
 185                                 next_protocol =
 186                                     ((const struct rte_flow_item_ipv6 *)
 187                                       (item->spec))->hdr.proto &
 188                                     ((const struct rte_flow_item_ipv6 *)
 189                                       (item->mask))->hdr.proto;
 190                         if ((next_protocol == IPPROTO_IPIP ||
 191                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 192                                 attr->attr = 0;
 193                         break;
 194                 case RTE_FLOW_ITEM_TYPE_UDP:
 195                         if (!attr->tcp)
 196                                 attr->udp = 1;
 197                         break;
 198                 case RTE_FLOW_ITEM_TYPE_TCP:
 199                         if (!attr->udp)
 200                                 attr->tcp = 1;
 201                         break;
 202                 default:
 203                         break;
 204                 }
 205         }
 206         attr->valid = 1;
 207 }
 208
 209 /*
 210  * Convert rte_mtr_color to mlx5 color.
 211  *
 212  * @param[in] rcol
 213  *   rte_mtr_color.
 214  *
 215  * @return
 216  *   mlx5 color.
 217  */
 218 static inline int
 219 rte_col_2_mlx5_col(enum rte_color rcol)
 220 {
 221         switch (rcol) {
 222         case RTE_COLOR_GREEN:
 223                 return MLX5_FLOW_COLOR_GREEN;
 224         case RTE_COLOR_YELLOW:
 225                 return MLX5_FLOW_COLOR_YELLOW;
 226         case RTE_COLOR_RED:
 227                 return MLX5_FLOW_COLOR_RED;
 228         default:
 229                 break;
 230         }
 231         return MLX5_FLOW_COLOR_UNDEFINED;
 232 }
 233
 234 struct field_modify_info {
 235         uint32_t size; /* Size of field in protocol header, in bytes. */
 236         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 237         enum mlx5_modification_field id;
 238 };
 239
 240 struct field_modify_info modify_eth[] = {
 241         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 242         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 243         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 244         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 245         {0, 0, 0},
 246 };
 247
 248 struct field_modify_info modify_vlan_out_first_vid[] = {
 249         /* Size in bits !!! */
 250         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 251         {0, 0, 0},
 252 };
 253
 254 struct field_modify_info modify_ipv4[] = {
 255         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 256         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 257         {4, 12, MLX5_MODI_OUT_SIPV4},
 258         {4, 16, MLX5_MODI_OUT_DIPV4},
 259         {0, 0, 0},
 260 };
 261
 262 struct field_modify_info modify_ipv6[] = {
 263         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 264         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 265         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 266         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 267         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 268         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 269         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 270         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 271         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 272         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 273         {0, 0, 0},
 274 };
 275
 276 struct field_modify_info modify_udp[] = {
 277         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 278         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 279         {0, 0, 0},
 280 };
 281
 282 struct field_modify_info modify_tcp[] = {
 283         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 284         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 285         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 286         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 287         {0, 0, 0},
 288 };
 289
 290 static const struct rte_flow_item *
 291 mlx5_flow_find_tunnel_item(const struct rte_flow_item *item)
 292 {
 293         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 294                 switch (item->type) {
 295                 default:
 296                         break;
 297                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 298                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 299                 case RTE_FLOW_ITEM_TYPE_GRE:
 300                 case RTE_FLOW_ITEM_TYPE_MPLS:
 301                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 302                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 303                         return item;
 304                 case RTE_FLOW_ITEM_TYPE_IPV4:
 305                 case RTE_FLOW_ITEM_TYPE_IPV6:
 306                         if (item[1].type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 307                             item[1].type == RTE_FLOW_ITEM_TYPE_IPV6)
 308                                 return item;
 309                         break;
 310                 }
 311         }
 312         return NULL;
 313 }
 314
 315 static void
 316 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 317                           uint8_t next_protocol, uint64_t *item_flags,
 318                           int *tunnel)
 319 {
 320         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 321                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 322         if (next_protocol == IPPROTO_IPIP) {
 323                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 324                 *tunnel = 1;
 325         }
 326         if (next_protocol == IPPROTO_IPV6) {
 327                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 328                 *tunnel = 1;
 329         }
 330 }
 331
 332 static inline struct mlx5_hlist *
 333 flow_dv_hlist_prepare(struct mlx5_dev_ctx_shared *sh, struct mlx5_hlist **phl,
 334                      const char *name, uint32_t size, bool direct_key,
 335                      bool lcores_share, void *ctx,
 336                      mlx5_list_create_cb cb_create,
 337                      mlx5_list_match_cb cb_match,
 338                      mlx5_list_remove_cb cb_remove,
 339                      mlx5_list_clone_cb cb_clone,
 340                      mlx5_list_clone_free_cb cb_clone_free)
 341 {
 342         struct mlx5_hlist *hl;
 343         struct mlx5_hlist *expected = NULL;
 344         char s[MLX5_NAME_SIZE];
 345
 346         hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
 347         if (likely(hl))
 348                 return hl;
 349         snprintf(s, sizeof(s), "%s_%s", sh->ibdev_name, name);
 350         hl = mlx5_hlist_create(s, size, direct_key, lcores_share,
 351                         ctx, cb_create, cb_match, cb_remove, cb_clone,
 352                         cb_clone_free);
 353         if (!hl) {
 354                 DRV_LOG(ERR, "%s hash creation failed", name);
 355                 rte_errno = ENOMEM;
 356                 return NULL;
 357         }
 358         if (!__atomic_compare_exchange_n(phl, &expected, hl, false,
 359                                          __ATOMIC_SEQ_CST,
 360                                          __ATOMIC_SEQ_CST)) {
 361                 mlx5_hlist_destroy(hl);
 362                 hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
 363         }
 364         return hl;
 365 }
 366
 367 /* Update VLAN's VID/PCP based on input rte_flow_action.
 368  *
 369  * @param[in] action
 370  *   Pointer to struct rte_flow_action.
 371  * @param[out] vlan
 372  *   Pointer to struct rte_vlan_hdr.
 373  */
 374 static void
 375 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 376                          struct rte_vlan_hdr *vlan)
 377 {
 378         uint16_t vlan_tci;
 379         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 380                 vlan_tci =
 381                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 382                                                action->conf)->vlan_pcp;
 383                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 384                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 385                 vlan->vlan_tci |= vlan_tci;
 386         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 387                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 388                 vlan->vlan_tci |= rte_be_to_cpu_16
 389                     (((const struct rte_flow_action_of_set_vlan_vid *)
 390                                              action->conf)->vlan_vid);
 391         }
 392 }
 393
 394 /**
 395  * Fetch 1, 2, 3 or 4 byte field from the byte array
 396  * and return as unsigned integer in host-endian format.
 397  *
 398  * @param[in] data
 399  *   Pointer to data array.
 400  * @param[in] size
 401  *   Size of field to extract.
 402  *
 403  * @return
 404  *   converted field in host endian format.
 405  */
 406 static inline uint32_t
 407 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 408 {
 409         uint32_t ret;
 410
 411         switch (size) {
 412         case 1:
 413                 ret = *data;
 414                 break;
 415         case 2:
 416                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 417                 break;
 418         case 3:
 419                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 420                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 421                 break;
 422         case 4:
 423                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 424                 break;
 425         default:
 426                 MLX5_ASSERT(false);
 427                 ret = 0;
 428                 break;
 429         }
 430         return ret;
 431 }
 432
 433 /**
 434  * Convert modify-header action to DV specification.
 435  *
 436  * Data length of each action is determined by provided field description
 437  * and the item mask. Data bit offset and width of each action is determined
 438  * by provided item mask.
 439  *
 440  * @param[in] item
 441  *   Pointer to item specification.
 442  * @param[in] field
 443  *   Pointer to field modification information.
 444  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 445  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 446  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 447  * @param[in] dcopy
 448  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 449  *   Negative offset value sets the same offset as source offset.
 450  *   size field is ignored, value is taken from source field.
 451  * @param[in,out] resource
 452  *   Pointer to the modify-header resource.
 453  * @param[in] type
 454  *   Type of modification.
 455  * @param[out] error
 456  *   Pointer to the error structure.
 457  *
 458  * @return
 459  *   0 on success, a negative errno value otherwise and rte_errno is set.
 460  */
 461 static int
 462 flow_dv_convert_modify_action(struct rte_flow_item *item,
 463                               struct field_modify_info *field,
 464                               struct field_modify_info *dcopy,
 465                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 466                               uint32_t type, struct rte_flow_error *error)
 467 {
 468         uint32_t i = resource->actions_num;
 469         struct mlx5_modification_cmd *actions = resource->actions;
 470         uint32_t carry_b = 0;
 471
 472         /*
 473          * The item and mask are provided in big-endian format.
 474          * The fields should be presented as in big-endian format either.
 475          * Mask must be always present, it defines the actual field width.
 476          */
 477         MLX5_ASSERT(item->mask);
 478         MLX5_ASSERT(field->size);
 479         do {
 480                 uint32_t size_b;
 481                 uint32_t off_b;
 482                 uint32_t mask;
 483                 uint32_t data;
 484                 bool next_field = true;
 485                 bool next_dcopy = true;
 486
 487                 if (i >= MLX5_MAX_MODIFY_NUM)
 488                         return rte_flow_error_set(error, EINVAL,
 489                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 490                                  "too many items to modify");
 491                 /* Fetch variable byte size mask from the array. */
 492                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 493                                            field->offset, field->size);
 494                 if (!mask) {
 495                         ++field;
 496                         continue;
 497                 }
 498                 /* Deduce actual data width in bits from mask value. */
 499                 off_b = rte_bsf32(mask) + carry_b;
 500                 size_b = sizeof(uint32_t) * CHAR_BIT -
 501                          off_b - __builtin_clz(mask);
 502                 MLX5_ASSERT(size_b);
 503                 actions[i] = (struct mlx5_modification_cmd) {
 504                         .action_type = type,
 505                         .field = field->id,
 506                         .offset = off_b,
 507                         .length = (size_b == sizeof(uint32_t) * CHAR_BIT) ?
 508                                 0 : size_b,
 509                 };
 510                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 511                         MLX5_ASSERT(dcopy);
 512                         actions[i].dst_field = dcopy->id;
 513                         actions[i].dst_offset =
 514                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 515                         /* Convert entire record to big-endian format. */
 516                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 517                         /*
 518                          * Destination field overflow. Copy leftovers of
 519                          * a source field to the next destination field.
 520                          */
 521                         carry_b = 0;
 522                         if ((size_b > dcopy->size * CHAR_BIT - dcopy->offset) &&
 523                             dcopy->size != 0) {
 524                                 actions[i].length =
 525                                         dcopy->size * CHAR_BIT - dcopy->offset;
 526                                 carry_b = actions[i].length;
 527                                 next_field = false;
 528                         }
 529                         /*
 530                          * Not enough bits in a source filed to fill a
 531                          * destination field. Switch to the next source.
 532                          */
 533                         if ((size_b < dcopy->size * CHAR_BIT - dcopy->offset) &&
 534                             (size_b == field->size * CHAR_BIT - off_b)) {
 535                                 actions[i].length =
 536                                         field->size * CHAR_BIT - off_b;
 537                                 dcopy->offset += actions[i].length;
 538                                 next_dcopy = false;
 539                         }
 540                         if (next_dcopy)
 541                                 ++dcopy;
 542                 } else {
 543                         MLX5_ASSERT(item->spec);
 544                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 545                                                    field->offset, field->size);
 546                         /* Shift out the trailing masked bits from data. */
 547                         data = (data & mask) >> off_b;
 548                         actions[i].data1 = rte_cpu_to_be_32(data);
 549                 }
 550                 /* Convert entire record to expected big-endian format. */
 551                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 552                 if (next_field)
 553                         ++field;
 554                 ++i;
 555         } while (field->size);
 556         if (resource->actions_num == i)
 557                 return rte_flow_error_set(error, EINVAL,
 558                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 559                                           "invalid modification flow item");
 560         resource->actions_num = i;
 561         return 0;
 562 }
 563
 564 /**
 565  * Convert modify-header set IPv4 address action to DV specification.
 566  *
 567  * @param[in,out] resource
 568  *   Pointer to the modify-header resource.
 569  * @param[in] action
 570  *   Pointer to action specification.
 571  * @param[out] error
 572  *   Pointer to the error structure.
 573  *
 574  * @return
 575  *   0 on success, a negative errno value otherwise and rte_errno is set.
 576  */
 577 static int
 578 flow_dv_convert_action_modify_ipv4
 579                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 580                          const struct rte_flow_action *action,
 581                          struct rte_flow_error *error)
 582 {
 583         const struct rte_flow_action_set_ipv4 *conf =
 584                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 585         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 586         struct rte_flow_item_ipv4 ipv4;
 587         struct rte_flow_item_ipv4 ipv4_mask;
 588
 589         memset(&ipv4, 0, sizeof(ipv4));
 590         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 591         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 592                 ipv4.hdr.src_addr = conf->ipv4_addr;
 593                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 594         } else {
 595                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 596                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 597         }
 598         item.spec = &ipv4;
 599         item.mask = &ipv4_mask;
 600         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 601                                              MLX5_MODIFICATION_TYPE_SET, error);
 602 }
 603
 604 /**
 605  * Convert modify-header set IPv6 address action to DV specification.
 606  *
 607  * @param[in,out] resource
 608  *   Pointer to the modify-header resource.
 609  * @param[in] action
 610  *   Pointer to action specification.
 611  * @param[out] error
 612  *   Pointer to the error structure.
 613  *
 614  * @return
 615  *   0 on success, a negative errno value otherwise and rte_errno is set.
 616  */
 617 static int
 618 flow_dv_convert_action_modify_ipv6
 619                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 620                          const struct rte_flow_action *action,
 621                          struct rte_flow_error *error)
 622 {
 623         const struct rte_flow_action_set_ipv6 *conf =
 624                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 625         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 626         struct rte_flow_item_ipv6 ipv6;
 627         struct rte_flow_item_ipv6 ipv6_mask;
 628
 629         memset(&ipv6, 0, sizeof(ipv6));
 630         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 631         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 632                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 633                        sizeof(ipv6.hdr.src_addr));
 634                 memcpy(&ipv6_mask.hdr.src_addr,
 635                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 636                        sizeof(ipv6.hdr.src_addr));
 637         } else {
 638                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 639                        sizeof(ipv6.hdr.dst_addr));
 640                 memcpy(&ipv6_mask.hdr.dst_addr,
 641                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 642                        sizeof(ipv6.hdr.dst_addr));
 643         }
 644         item.spec = &ipv6;
 645         item.mask = &ipv6_mask;
 646         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 647                                              MLX5_MODIFICATION_TYPE_SET, error);
 648 }
 649
 650 /**
 651  * Convert modify-header set MAC address action to DV specification.
 652  *
 653  * @param[in,out] resource
 654  *   Pointer to the modify-header resource.
 655  * @param[in] action
 656  *   Pointer to action specification.
 657  * @param[out] error
 658  *   Pointer to the error structure.
 659  *
 660  * @return
 661  *   0 on success, a negative errno value otherwise and rte_errno is set.
 662  */
 663 static int
 664 flow_dv_convert_action_modify_mac
 665                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 666                          const struct rte_flow_action *action,
 667                          struct rte_flow_error *error)
 668 {
 669         const struct rte_flow_action_set_mac *conf =
 670                 (const struct rte_flow_action_set_mac *)(action->conf);
 671         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 672         struct rte_flow_item_eth eth;
 673         struct rte_flow_item_eth eth_mask;
 674
 675         memset(&eth, 0, sizeof(eth));
 676         memset(&eth_mask, 0, sizeof(eth_mask));
 677         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 678                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 679                        sizeof(eth.src.addr_bytes));
 680                 memcpy(&eth_mask.src.addr_bytes,
 681                        &rte_flow_item_eth_mask.src.addr_bytes,
 682                        sizeof(eth_mask.src.addr_bytes));
 683         } else {
 684                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 685                        sizeof(eth.dst.addr_bytes));
 686                 memcpy(&eth_mask.dst.addr_bytes,
 687                        &rte_flow_item_eth_mask.dst.addr_bytes,
 688                        sizeof(eth_mask.dst.addr_bytes));
 689         }
 690         item.spec = &eth;
 691         item.mask = &eth_mask;
 692         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 693                                              MLX5_MODIFICATION_TYPE_SET, error);
 694 }
 695
 696 /**
 697  * Convert modify-header set VLAN VID action to DV specification.
 698  *
 699  * @param[in,out] resource
 700  *   Pointer to the modify-header resource.
 701  * @param[in] action
 702  *   Pointer to action specification.
 703  * @param[out] error
 704  *   Pointer to the error structure.
 705  *
 706  * @return
 707  *   0 on success, a negative errno value otherwise and rte_errno is set.
 708  */
 709 static int
 710 flow_dv_convert_action_modify_vlan_vid
 711                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 712                          const struct rte_flow_action *action,
 713                          struct rte_flow_error *error)
 714 {
 715         const struct rte_flow_action_of_set_vlan_vid *conf =
 716                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 717         int i = resource->actions_num;
 718         struct mlx5_modification_cmd *actions = resource->actions;
 719         struct field_modify_info *field = modify_vlan_out_first_vid;
 720
 721         if (i >= MLX5_MAX_MODIFY_NUM)
 722                 return rte_flow_error_set(error, EINVAL,
 723                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 724                          "too many items to modify");
 725         actions[i] = (struct mlx5_modification_cmd) {
 726                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 727                 .field = field->id,
 728                 .length = field->size,
 729                 .offset = field->offset,
 730         };
 731         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 732         actions[i].data1 = conf->vlan_vid;
 733         actions[i].data1 = actions[i].data1 << 16;
 734         resource->actions_num = ++i;
 735         return 0;
 736 }
 737
 738 /**
 739  * Convert modify-header set TP action to DV specification.
 740  *
 741  * @param[in,out] resource
 742  *   Pointer to the modify-header resource.
 743  * @param[in] action
 744  *   Pointer to action specification.
 745  * @param[in] items
 746  *   Pointer to rte_flow_item objects list.
 747  * @param[in] attr
 748  *   Pointer to flow attributes structure.
 749  * @param[in] dev_flow
 750  *   Pointer to the sub flow.
 751  * @param[in] tunnel_decap
 752  *   Whether action is after tunnel decapsulation.
 753  * @param[out] error
 754  *   Pointer to the error structure.
 755  *
 756  * @return
 757  *   0 on success, a negative errno value otherwise and rte_errno is set.
 758  */
 759 static int
 760 flow_dv_convert_action_modify_tp
 761                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 762                          const struct rte_flow_action *action,
 763                          const struct rte_flow_item *items,
 764                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 765                          bool tunnel_decap, struct rte_flow_error *error)
 766 {
 767         const struct rte_flow_action_set_tp *conf =
 768                 (const struct rte_flow_action_set_tp *)(action->conf);
 769         struct rte_flow_item item;
 770         struct rte_flow_item_udp udp;
 771         struct rte_flow_item_udp udp_mask;
 772         struct rte_flow_item_tcp tcp;
 773         struct rte_flow_item_tcp tcp_mask;
 774         struct field_modify_info *field;
 775
 776         if (!attr->valid)
 777                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 778         if (attr->udp) {
 779                 memset(&udp, 0, sizeof(udp));
 780                 memset(&udp_mask, 0, sizeof(udp_mask));
 781                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 782                         udp.hdr.src_port = conf->port;
 783                         udp_mask.hdr.src_port =
 784                                         rte_flow_item_udp_mask.hdr.src_port;
 785                 } else {
 786                         udp.hdr.dst_port = conf->port;
 787                         udp_mask.hdr.dst_port =
 788                                         rte_flow_item_udp_mask.hdr.dst_port;
 789                 }
 790                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 791                 item.spec = &udp;
 792                 item.mask = &udp_mask;
 793                 field = modify_udp;
 794         } else {
 795                 MLX5_ASSERT(attr->tcp);
 796                 memset(&tcp, 0, sizeof(tcp));
 797                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 798                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 799                         tcp.hdr.src_port = conf->port;
 800                         tcp_mask.hdr.src_port =
 801                                         rte_flow_item_tcp_mask.hdr.src_port;
 802                 } else {
 803                         tcp.hdr.dst_port = conf->port;
 804                         tcp_mask.hdr.dst_port =
 805                                         rte_flow_item_tcp_mask.hdr.dst_port;
 806                 }
 807                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 808                 item.spec = &tcp;
 809                 item.mask = &tcp_mask;
 810                 field = modify_tcp;
 811         }
 812         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 813                                              MLX5_MODIFICATION_TYPE_SET, error);
 814 }
 815
 816 /**
 817  * Convert modify-header set TTL action to DV specification.
 818  *
 819  * @param[in,out] resource
 820  *   Pointer to the modify-header resource.
 821  * @param[in] action
 822  *   Pointer to action specification.
 823  * @param[in] items
 824  *   Pointer to rte_flow_item objects list.
 825  * @param[in] attr
 826  *   Pointer to flow attributes structure.
 827  * @param[in] dev_flow
 828  *   Pointer to the sub flow.
 829  * @param[in] tunnel_decap
 830  *   Whether action is after tunnel decapsulation.
 831  * @param[out] error
 832  *   Pointer to the error structure.
 833  *
 834  * @return
 835  *   0 on success, a negative errno value otherwise and rte_errno is set.
 836  */
 837 static int
 838 flow_dv_convert_action_modify_ttl
 839                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 840                          const struct rte_flow_action *action,
 841                          const struct rte_flow_item *items,
 842                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 843                          bool tunnel_decap, struct rte_flow_error *error)
 844 {
 845         const struct rte_flow_action_set_ttl *conf =
 846                 (const struct rte_flow_action_set_ttl *)(action->conf);
 847         struct rte_flow_item item;
 848         struct rte_flow_item_ipv4 ipv4;
 849         struct rte_flow_item_ipv4 ipv4_mask;
 850         struct rte_flow_item_ipv6 ipv6;
 851         struct rte_flow_item_ipv6 ipv6_mask;
 852         struct field_modify_info *field;
 853
 854         if (!attr->valid)
 855                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 856         if (attr->ipv4) {
 857                 memset(&ipv4, 0, sizeof(ipv4));
 858                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 859                 ipv4.hdr.time_to_live = conf->ttl_value;
 860                 ipv4_mask.hdr.time_to_live = 0xFF;
 861                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 862                 item.spec = &ipv4;
 863                 item.mask = &ipv4_mask;
 864                 field = modify_ipv4;
 865         } else {
 866                 MLX5_ASSERT(attr->ipv6);
 867                 memset(&ipv6, 0, sizeof(ipv6));
 868                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 869                 ipv6.hdr.hop_limits = conf->ttl_value;
 870                 ipv6_mask.hdr.hop_limits = 0xFF;
 871                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 872                 item.spec = &ipv6;
 873                 item.mask = &ipv6_mask;
 874                 field = modify_ipv6;
 875         }
 876         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 877                                              MLX5_MODIFICATION_TYPE_SET, error);
 878 }
 879
 880 /**
 881  * Convert modify-header decrement TTL action to DV specification.
 882  *
 883  * @param[in,out] resource
 884  *   Pointer to the modify-header resource.
 885  * @param[in] action
 886  *   Pointer to action specification.
 887  * @param[in] items
 888  *   Pointer to rte_flow_item objects list.
 889  * @param[in] attr
 890  *   Pointer to flow attributes structure.
 891  * @param[in] dev_flow
 892  *   Pointer to the sub flow.
 893  * @param[in] tunnel_decap
 894  *   Whether action is after tunnel decapsulation.
 895  * @param[out] error
 896  *   Pointer to the error structure.
 897  *
 898  * @return
 899  *   0 on success, a negative errno value otherwise and rte_errno is set.
 900  */
 901 static int
 902 flow_dv_convert_action_modify_dec_ttl
 903                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 904                          const struct rte_flow_item *items,
 905                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 906                          bool tunnel_decap, struct rte_flow_error *error)
 907 {
 908         struct rte_flow_item item;
 909         struct rte_flow_item_ipv4 ipv4;
 910         struct rte_flow_item_ipv4 ipv4_mask;
 911         struct rte_flow_item_ipv6 ipv6;
 912         struct rte_flow_item_ipv6 ipv6_mask;
 913         struct field_modify_info *field;
 914
 915         if (!attr->valid)
 916                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 917         if (attr->ipv4) {
 918                 memset(&ipv4, 0, sizeof(ipv4));
 919                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 920                 ipv4.hdr.time_to_live = 0xFF;
 921                 ipv4_mask.hdr.time_to_live = 0xFF;
 922                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 923                 item.spec = &ipv4;
 924                 item.mask = &ipv4_mask;
 925                 field = modify_ipv4;
 926         } else {
 927                 MLX5_ASSERT(attr->ipv6);
 928                 memset(&ipv6, 0, sizeof(ipv6));
 929                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 930                 ipv6.hdr.hop_limits = 0xFF;
 931                 ipv6_mask.hdr.hop_limits = 0xFF;
 932                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 933                 item.spec = &ipv6;
 934                 item.mask = &ipv6_mask;
 935                 field = modify_ipv6;
 936         }
 937         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 938                                              MLX5_MODIFICATION_TYPE_ADD, error);
 939 }
 940
 941 /**
 942  * Convert modify-header increment/decrement TCP Sequence number
 943  * to DV specification.
 944  *
 945  * @param[in,out] resource
 946  *   Pointer to the modify-header resource.
 947  * @param[in] action
 948  *   Pointer to action specification.
 949  * @param[out] error
 950  *   Pointer to the error structure.
 951  *
 952  * @return
 953  *   0 on success, a negative errno value otherwise and rte_errno is set.
 954  */
 955 static int
 956 flow_dv_convert_action_modify_tcp_seq
 957                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 958                          const struct rte_flow_action *action,
 959                          struct rte_flow_error *error)
 960 {
 961         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 962         uint64_t value = rte_be_to_cpu_32(*conf);
 963         struct rte_flow_item item;
 964         struct rte_flow_item_tcp tcp;
 965         struct rte_flow_item_tcp tcp_mask;
 966
 967         memset(&tcp, 0, sizeof(tcp));
 968         memset(&tcp_mask, 0, sizeof(tcp_mask));
 969         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 970                 /*
 971                  * The HW has no decrement operation, only increment operation.
 972                  * To simulate decrement X from Y using increment operation
 973                  * we need to add UINT32_MAX X times to Y.
 974                  * Each adding of UINT32_MAX decrements Y by 1.
 975                  */
 976                 value *= UINT32_MAX;
 977         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 978         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 979         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 980         item.spec = &tcp;
 981         item.mask = &tcp_mask;
 982         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 983                                              MLX5_MODIFICATION_TYPE_ADD, error);
 984 }
 985
 986 /**
 987  * Convert modify-header increment/decrement TCP Acknowledgment number
 988  * to DV specification.
 989  *
 990  * @param[in,out] resource
 991  *   Pointer to the modify-header resource.
 992  * @param[in] action
 993  *   Pointer to action specification.
 994  * @param[out] error
 995  *   Pointer to the error structure.
 996  *
 997  * @return
 998  *   0 on success, a negative errno value otherwise and rte_errno is set.
 999  */
1000 static int
1001 flow_dv_convert_action_modify_tcp_ack
1002                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1003                          const struct rte_flow_action *action,
1004                          struct rte_flow_error *error)
1005 {
1006         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
1007         uint64_t value = rte_be_to_cpu_32(*conf);
1008         struct rte_flow_item item;
1009         struct rte_flow_item_tcp tcp;
1010         struct rte_flow_item_tcp tcp_mask;
1011
1012         memset(&tcp, 0, sizeof(tcp));
1013         memset(&tcp_mask, 0, sizeof(tcp_mask));
1014         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
1015                 /*
1016                  * The HW has no decrement operation, only increment operation.
1017                  * To simulate decrement X from Y using increment operation
1018                  * we need to add UINT32_MAX X times to Y.
1019                  * Each adding of UINT32_MAX decrements Y by 1.
1020                  */
1021                 value *= UINT32_MAX;
1022         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
1023         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
1024         item.type = RTE_FLOW_ITEM_TYPE_TCP;
1025         item.spec = &tcp;
1026         item.mask = &tcp_mask;
1027         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
1028                                              MLX5_MODIFICATION_TYPE_ADD, error);
1029 }
1030
1031 static enum mlx5_modification_field reg_to_field[] = {
1032         [REG_NON] = MLX5_MODI_OUT_NONE,
1033         [REG_A] = MLX5_MODI_META_DATA_REG_A,
1034         [REG_B] = MLX5_MODI_META_DATA_REG_B,
1035         [REG_C_0] = MLX5_MODI_META_REG_C_0,
1036         [REG_C_1] = MLX5_MODI_META_REG_C_1,
1037         [REG_C_2] = MLX5_MODI_META_REG_C_2,
1038         [REG_C_3] = MLX5_MODI_META_REG_C_3,
1039         [REG_C_4] = MLX5_MODI_META_REG_C_4,
1040         [REG_C_5] = MLX5_MODI_META_REG_C_5,
1041         [REG_C_6] = MLX5_MODI_META_REG_C_6,
1042         [REG_C_7] = MLX5_MODI_META_REG_C_7,
1043 };
1044
1045 /**
1046  * Convert register set to DV specification.
1047  *
1048  * @param[in,out] resource
1049  *   Pointer to the modify-header resource.
1050  * @param[in] action
1051  *   Pointer to action specification.
1052  * @param[out] error
1053  *   Pointer to the error structure.
1054  *
1055  * @return
1056  *   0 on success, a negative errno value otherwise and rte_errno is set.
1057  */
1058 static int
1059 flow_dv_convert_action_set_reg
1060                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1061                          const struct rte_flow_action *action,
1062                          struct rte_flow_error *error)
1063 {
1064         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
1065         struct mlx5_modification_cmd *actions = resource->actions;
1066         uint32_t i = resource->actions_num;
1067
1068         if (i >= MLX5_MAX_MODIFY_NUM)
1069                 return rte_flow_error_set(error, EINVAL,
1070                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1071                                           "too many items to modify");
1072         MLX5_ASSERT(conf->id != REG_NON);
1073         MLX5_ASSERT(conf->id < (enum modify_reg)RTE_DIM(reg_to_field));
1074         actions[i] = (struct mlx5_modification_cmd) {
1075                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1076                 .field = reg_to_field[conf->id],
1077                 .offset = conf->offset,
1078                 .length = conf->length,
1079         };
1080         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1081         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1082         ++i;
1083         resource->actions_num = i;
1084         return 0;
1085 }
1086
1087 /**
1088  * Convert SET_TAG action to DV specification.
1089  *
1090  * @param[in] dev
1091  *   Pointer to the rte_eth_dev structure.
1092  * @param[in,out] resource
1093  *   Pointer to the modify-header resource.
1094  * @param[in] conf
1095  *   Pointer to action specification.
1096  * @param[out] error
1097  *   Pointer to the error structure.
1098  *
1099  * @return
1100  *   0 on success, a negative errno value otherwise and rte_errno is set.
1101  */
1102 static int
1103 flow_dv_convert_action_set_tag
1104                         (struct rte_eth_dev *dev,
1105                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1106                          const struct rte_flow_action_set_tag *conf,
1107                          struct rte_flow_error *error)
1108 {
1109         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1110         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1111         struct rte_flow_item item = {
1112                 .spec = &data,
1113                 .mask = &mask,
1114         };
1115         struct field_modify_info reg_c_x[] = {
1116                 [1] = {0, 0, 0},
1117         };
1118         enum mlx5_modification_field reg_type;
1119         int ret;
1120
1121         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1122         if (ret < 0)
1123                 return ret;
1124         MLX5_ASSERT(ret != REG_NON);
1125         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1126         reg_type = reg_to_field[ret];
1127         MLX5_ASSERT(reg_type > 0);
1128         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1129         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1130                                              MLX5_MODIFICATION_TYPE_SET, error);
1131 }
1132
1133 /**
1134  * Convert internal COPY_REG action to DV specification.
1135  *
1136  * @param[in] dev
1137  *   Pointer to the rte_eth_dev structure.
1138  * @param[in,out] res
1139  *   Pointer to the modify-header resource.
1140  * @param[in] action
1141  *   Pointer to action specification.
1142  * @param[out] error
1143  *   Pointer to the error structure.
1144  *
1145  * @return
1146  *   0 on success, a negative errno value otherwise and rte_errno is set.
1147  */
1148 static int
1149 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1150                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1151                                  const struct rte_flow_action *action,
1152                                  struct rte_flow_error *error)
1153 {
1154         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1155         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1156         struct rte_flow_item item = {
1157                 .spec = NULL,
1158                 .mask = &mask,
1159         };
1160         struct field_modify_info reg_src[] = {
1161                 {4, 0, reg_to_field[conf->src]},
1162                 {0, 0, 0},
1163         };
1164         struct field_modify_info reg_dst = {
1165                 .offset = 0,
1166                 .id = reg_to_field[conf->dst],
1167         };
1168         /* Adjust reg_c[0] usage according to reported mask. */
1169         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1170                 struct mlx5_priv *priv = dev->data->dev_private;
1171                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1172
1173                 MLX5_ASSERT(reg_c0);
1174                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1175                 if (conf->dst == REG_C_0) {
1176                         /* Copy to reg_c[0], within mask only. */
1177                         reg_dst.offset = rte_bsf32(reg_c0);
1178                         mask = rte_cpu_to_be_32(reg_c0 >> reg_dst.offset);
1179                 } else {
1180                         reg_dst.offset = 0;
1181                         mask = rte_cpu_to_be_32(reg_c0);
1182                 }
1183         }
1184         return flow_dv_convert_modify_action(&item,
1185                                              reg_src, &reg_dst, res,
1186                                              MLX5_MODIFICATION_TYPE_COPY,
1187                                              error);
1188 }
1189
1190 /**
1191  * Convert MARK action to DV specification. This routine is used
1192  * in extensive metadata only and requires metadata register to be
1193  * handled. In legacy mode hardware tag resource is engaged.
1194  *
1195  * @param[in] dev
1196  *   Pointer to the rte_eth_dev structure.
1197  * @param[in] conf
1198  *   Pointer to MARK action specification.
1199  * @param[in,out] resource
1200  *   Pointer to the modify-header resource.
1201  * @param[out] error
1202  *   Pointer to the error structure.
1203  *
1204  * @return
1205  *   0 on success, a negative errno value otherwise and rte_errno is set.
1206  */
1207 static int
1208 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1209                             const struct rte_flow_action_mark *conf,
1210                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1211                             struct rte_flow_error *error)
1212 {
1213         struct mlx5_priv *priv = dev->data->dev_private;
1214         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1215                                            priv->sh->dv_mark_mask);
1216         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1217         struct rte_flow_item item = {
1218                 .spec = &data,
1219                 .mask = &mask,
1220         };
1221         struct field_modify_info reg_c_x[] = {
1222                 [1] = {0, 0, 0},
1223         };
1224         int reg;
1225
1226         if (!mask)
1227                 return rte_flow_error_set(error, EINVAL,
1228                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1229                                           NULL, "zero mark action mask");
1230         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1231         if (reg < 0)
1232                 return reg;
1233         MLX5_ASSERT(reg > 0);
1234         if (reg == REG_C_0) {
1235                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1236                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1237
1238                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1239                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1240                 mask = rte_cpu_to_be_32(mask << shl_c0);
1241         }
1242         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1243         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1244                                              MLX5_MODIFICATION_TYPE_SET, error);
1245 }
1246
1247 /**
1248  * Get metadata register index for specified steering domain.
1249  *
1250  * @param[in] dev
1251  *   Pointer to the rte_eth_dev structure.
1252  * @param[in] attr
1253  *   Attributes of flow to determine steering domain.
1254  * @param[out] error
1255  *   Pointer to the error structure.
1256  *
1257  * @return
1258  *   positive index on success, a negative errno value otherwise
1259  *   and rte_errno is set.
1260  */
1261 static enum modify_reg
1262 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1263                          const struct rte_flow_attr *attr,
1264                          struct rte_flow_error *error)
1265 {
1266         int reg =
1267                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1268                                           MLX5_METADATA_FDB :
1269                                             attr->egress ?
1270                                             MLX5_METADATA_TX :
1271                                             MLX5_METADATA_RX, 0, error);
1272         if (reg < 0)
1273                 return rte_flow_error_set(error,
1274                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1275                                           NULL, "unavailable "
1276                                           "metadata register");
1277         return reg;
1278 }
1279
1280 /**
1281  * Convert SET_META action to DV specification.
1282  *
1283  * @param[in] dev
1284  *   Pointer to the rte_eth_dev structure.
1285  * @param[in,out] resource
1286  *   Pointer to the modify-header resource.
1287  * @param[in] attr
1288  *   Attributes of flow that includes this item.
1289  * @param[in] conf
1290  *   Pointer to action specification.
1291  * @param[out] error
1292  *   Pointer to the error structure.
1293  *
1294  * @return
1295  *   0 on success, a negative errno value otherwise and rte_errno is set.
1296  */
1297 static int
1298 flow_dv_convert_action_set_meta
1299                         (struct rte_eth_dev *dev,
1300                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1301                          const struct rte_flow_attr *attr,
1302                          const struct rte_flow_action_set_meta *conf,
1303                          struct rte_flow_error *error)
1304 {
1305         uint32_t mask = rte_cpu_to_be_32(conf->mask);
1306         uint32_t data = rte_cpu_to_be_32(conf->data) & mask;
1307         struct rte_flow_item item = {
1308                 .spec = &data,
1309                 .mask = &mask,
1310         };
1311         struct field_modify_info reg_c_x[] = {
1312                 [1] = {0, 0, 0},
1313         };
1314         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1315
1316         if (reg < 0)
1317                 return reg;
1318         MLX5_ASSERT(reg != REG_NON);
1319         if (reg == REG_C_0) {
1320                 struct mlx5_priv *priv = dev->data->dev_private;
1321                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1322                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1323
1324                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1325                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1326                 mask = rte_cpu_to_be_32(mask << shl_c0);
1327         }
1328         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1329         /* The routine expects parameters in memory as big-endian ones. */
1330         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1331                                              MLX5_MODIFICATION_TYPE_SET, error);
1332 }
1333
1334 /**
1335  * Convert modify-header set IPv4 DSCP action to DV specification.
1336  *
1337  * @param[in,out] resource
1338  *   Pointer to the modify-header resource.
1339  * @param[in] action
1340  *   Pointer to action specification.
1341  * @param[out] error
1342  *   Pointer to the error structure.
1343  *
1344  * @return
1345  *   0 on success, a negative errno value otherwise and rte_errno is set.
1346  */
1347 static int
1348 flow_dv_convert_action_modify_ipv4_dscp
1349                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1350                          const struct rte_flow_action *action,
1351                          struct rte_flow_error *error)
1352 {
1353         const struct rte_flow_action_set_dscp *conf =
1354                 (const struct rte_flow_action_set_dscp *)(action->conf);
1355         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1356         struct rte_flow_item_ipv4 ipv4;
1357         struct rte_flow_item_ipv4 ipv4_mask;
1358
1359         memset(&ipv4, 0, sizeof(ipv4));
1360         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1361         ipv4.hdr.type_of_service = conf->dscp;
1362         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1363         item.spec = &ipv4;
1364         item.mask = &ipv4_mask;
1365         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1366                                              MLX5_MODIFICATION_TYPE_SET, error);
1367 }
1368
1369 /**
1370  * Convert modify-header set IPv6 DSCP action to DV specification.
1371  *
1372  * @param[in,out] resource
1373  *   Pointer to the modify-header resource.
1374  * @param[in] action
1375  *   Pointer to action specification.
1376  * @param[out] error
1377  *   Pointer to the error structure.
1378  *
1379  * @return
1380  *   0 on success, a negative errno value otherwise and rte_errno is set.
1381  */
1382 static int
1383 flow_dv_convert_action_modify_ipv6_dscp
1384                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1385                          const struct rte_flow_action *action,
1386                          struct rte_flow_error *error)
1387 {
1388         const struct rte_flow_action_set_dscp *conf =
1389                 (const struct rte_flow_action_set_dscp *)(action->conf);
1390         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1391         struct rte_flow_item_ipv6 ipv6;
1392         struct rte_flow_item_ipv6 ipv6_mask;
1393
1394         memset(&ipv6, 0, sizeof(ipv6));
1395         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1396         /*
1397          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1398          * rdma-core only accept the DSCP bits byte aligned start from
1399          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1400          * bits in IPv6 case as rdma-core requires byte aligned value.
1401          */
1402         ipv6.hdr.vtc_flow = conf->dscp;
1403         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1404         item.spec = &ipv6;
1405         item.mask = &ipv6_mask;
1406         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1407                                              MLX5_MODIFICATION_TYPE_SET, error);
1408 }
1409
1410 static int
1411 mlx5_flow_item_field_width(struct mlx5_priv *priv,
1412                            enum rte_flow_field_id field, int inherit)
1413 {
1414         switch (field) {
1415         case RTE_FLOW_FIELD_START:
1416                 return 32;
1417         case RTE_FLOW_FIELD_MAC_DST:
1418         case RTE_FLOW_FIELD_MAC_SRC:
1419                 return 48;
1420         case RTE_FLOW_FIELD_VLAN_TYPE:
1421                 return 16;
1422         case RTE_FLOW_FIELD_VLAN_ID:
1423                 return 12;
1424         case RTE_FLOW_FIELD_MAC_TYPE:
1425                 return 16;
1426         case RTE_FLOW_FIELD_IPV4_DSCP:
1427                 return 6;
1428         case RTE_FLOW_FIELD_IPV4_TTL:
1429                 return 8;
1430         case RTE_FLOW_FIELD_IPV4_SRC:
1431         case RTE_FLOW_FIELD_IPV4_DST:
1432                 return 32;
1433         case RTE_FLOW_FIELD_IPV6_DSCP:
1434                 return 6;
1435         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1436                 return 8;
1437         case RTE_FLOW_FIELD_IPV6_SRC:
1438         case RTE_FLOW_FIELD_IPV6_DST:
1439                 return 128;
1440         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1441         case RTE_FLOW_FIELD_TCP_PORT_DST:
1442                 return 16;
1443         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1444         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1445                 return 32;
1446         case RTE_FLOW_FIELD_TCP_FLAGS:
1447                 return 9;
1448         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1449         case RTE_FLOW_FIELD_UDP_PORT_DST:
1450                 return 16;
1451         case RTE_FLOW_FIELD_VXLAN_VNI:
1452         case RTE_FLOW_FIELD_GENEVE_VNI:
1453                 return 24;
1454         case RTE_FLOW_FIELD_GTP_TEID:
1455         case RTE_FLOW_FIELD_TAG:
1456                 return 32;
1457         case RTE_FLOW_FIELD_MARK:
1458                 return __builtin_popcount(priv->sh->dv_mark_mask);
1459         case RTE_FLOW_FIELD_META:
1460                 return __builtin_popcount(priv->sh->dv_meta_mask);
1461         case RTE_FLOW_FIELD_POINTER:
1462         case RTE_FLOW_FIELD_VALUE:
1463                 return inherit < 0 ? 0 : inherit;
1464         default:
1465                 MLX5_ASSERT(false);
1466         }
1467         return 0;
1468 }
1469
1470 static void
1471 mlx5_flow_field_id_to_modify_info
1472                 (const struct rte_flow_action_modify_data *data,
1473                  struct field_modify_info *info, uint32_t *mask,
1474                  uint32_t width, uint32_t *shift, struct rte_eth_dev *dev,
1475                  const struct rte_flow_attr *attr, struct rte_flow_error *error)
1476 {
1477         struct mlx5_priv *priv = dev->data->dev_private;
1478         uint32_t idx = 0;
1479         uint32_t off = 0;
1480
1481         switch (data->field) {
1482         case RTE_FLOW_FIELD_START:
1483                 /* not supported yet */
1484                 MLX5_ASSERT(false);
1485                 break;
1486         case RTE_FLOW_FIELD_MAC_DST:
1487                 off = data->offset > 16 ? data->offset - 16 : 0;
1488                 if (mask) {
1489                         if (data->offset < 16) {
1490                                 info[idx] = (struct field_modify_info){2, 4,
1491                                                 MLX5_MODI_OUT_DMAC_15_0};
1492                                 if (width < 16) {
1493                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1494                                                                  (16 - width));
1495                                         width = 0;
1496                                 } else {
1497                                         mask[idx] = RTE_BE16(0xffff);
1498                                         width -= 16;
1499                                 }
1500                                 if (!width)
1501                                         break;
1502                                 ++idx;
1503                         }
1504                         info[idx] = (struct field_modify_info){4, 0,
1505                                                 MLX5_MODI_OUT_DMAC_47_16};
1506                         mask[idx] = rte_cpu_to_be_32((0xffffffff >>
1507                                                       (32 - width)) << off);
1508                 } else {
1509                         if (data->offset < 16)
1510                                 info[idx++] = (struct field_modify_info){2, 4,
1511                                                 MLX5_MODI_OUT_DMAC_15_0};
1512                         info[idx] = (struct field_modify_info){4, 0,
1513                                                 MLX5_MODI_OUT_DMAC_47_16};
1514                 }
1515                 break;
1516         case RTE_FLOW_FIELD_MAC_SRC:
1517                 off = data->offset > 16 ? data->offset - 16 : 0;
1518                 if (mask) {
1519                         if (data->offset < 16) {
1520                                 info[idx] = (struct field_modify_info){2, 4,
1521                                                 MLX5_MODI_OUT_SMAC_15_0};
1522                                 if (width < 16) {
1523                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1524                                                                  (16 - width));
1525                                         width = 0;
1526                                 } else {
1527                                         mask[idx] = RTE_BE16(0xffff);
1528                                         width -= 16;
1529                                 }
1530                                 if (!width)
1531                                         break;
1532                                 ++idx;
1533                         }
1534                         info[idx] = (struct field_modify_info){4, 0,
1535                                                 MLX5_MODI_OUT_SMAC_47_16};
1536                         mask[idx] = rte_cpu_to_be_32((0xffffffff >>
1537                                                       (32 - width)) << off);
1538                 } else {
1539                         if (data->offset < 16)
1540                                 info[idx++] = (struct field_modify_info){2, 4,
1541                                                 MLX5_MODI_OUT_SMAC_15_0};
1542                         info[idx] = (struct field_modify_info){4, 0,
1543                                                 MLX5_MODI_OUT_SMAC_47_16};
1544                 }
1545                 break;
1546         case RTE_FLOW_FIELD_VLAN_TYPE:
1547                 /* not supported yet */
1548                 break;
1549         case RTE_FLOW_FIELD_VLAN_ID:
1550                 info[idx] = (struct field_modify_info){2, 0,
1551                                         MLX5_MODI_OUT_FIRST_VID};
1552                 if (mask)
1553                         mask[idx] = rte_cpu_to_be_16(0x0fff >> (12 - width));
1554                 break;
1555         case RTE_FLOW_FIELD_MAC_TYPE:
1556                 info[idx] = (struct field_modify_info){2, 0,
1557                                         MLX5_MODI_OUT_ETHERTYPE};
1558                 if (mask)
1559                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1560                 break;
1561         case RTE_FLOW_FIELD_IPV4_DSCP:
1562                 info[idx] = (struct field_modify_info){1, 0,
1563                                         MLX5_MODI_OUT_IP_DSCP};
1564                 if (mask)
1565                         mask[idx] = 0x3f >> (6 - width);
1566                 break;
1567         case RTE_FLOW_FIELD_IPV4_TTL:
1568                 info[idx] = (struct field_modify_info){1, 0,
1569                                         MLX5_MODI_OUT_IPV4_TTL};
1570                 if (mask)
1571                         mask[idx] = 0xff >> (8 - width);
1572                 break;
1573         case RTE_FLOW_FIELD_IPV4_SRC:
1574                 info[idx] = (struct field_modify_info){4, 0,
1575                                         MLX5_MODI_OUT_SIPV4};
1576                 if (mask)
1577                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1578                                                      (32 - width));
1579                 break;
1580         case RTE_FLOW_FIELD_IPV4_DST:
1581                 info[idx] = (struct field_modify_info){4, 0,
1582                                         MLX5_MODI_OUT_DIPV4};
1583                 if (mask)
1584                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1585                                                      (32 - width));
1586                 break;
1587         case RTE_FLOW_FIELD_IPV6_DSCP:
1588                 info[idx] = (struct field_modify_info){1, 0,
1589                                         MLX5_MODI_OUT_IP_DSCP};
1590                 if (mask)
1591                         mask[idx] = 0x3f >> (6 - width);
1592                 break;
1593         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1594                 info[idx] = (struct field_modify_info){1, 0,
1595                                         MLX5_MODI_OUT_IPV6_HOPLIMIT};
1596                 if (mask)
1597                         mask[idx] = 0xff >> (8 - width);
1598                 break;
1599         case RTE_FLOW_FIELD_IPV6_SRC:
1600                 if (mask) {
1601                         if (data->offset < 32) {
1602                                 info[idx] = (struct field_modify_info){4, 12,
1603                                                 MLX5_MODI_OUT_SIPV6_31_0};
1604                                 if (width < 32) {
1605                                         mask[idx] =
1606                                                 rte_cpu_to_be_32(0xffffffff >>
1607                                                                  (32 - width));
1608                                         width = 0;
1609                                 } else {
1610                                         mask[idx] = RTE_BE32(0xffffffff);
1611                                         width -= 32;
1612                                 }
1613                                 if (!width)
1614                                         break;
1615                                 ++idx;
1616                         }
1617                         if (data->offset < 64) {
1618                                 info[idx] = (struct field_modify_info){4, 8,
1619                                                 MLX5_MODI_OUT_SIPV6_63_32};
1620                                 if (width < 32) {
1621                                         mask[idx] =
1622                                                 rte_cpu_to_be_32(0xffffffff >>
1623                                                                  (32 - width));
1624                                         width = 0;
1625                                 } else {
1626                                         mask[idx] = RTE_BE32(0xffffffff);
1627                                         width -= 32;
1628                                 }
1629                                 if (!width)
1630                                         break;
1631                                 ++idx;
1632                         }
1633                         if (data->offset < 96) {
1634                                 info[idx] = (struct field_modify_info){4, 4,
1635                                                 MLX5_MODI_OUT_SIPV6_95_64};
1636                                 if (width < 32) {
1637                                         mask[idx] =
1638                                                 rte_cpu_to_be_32(0xffffffff >>
1639                                                                  (32 - width));
1640                                         width = 0;
1641                                 } else {
1642                                         mask[idx] = RTE_BE32(0xffffffff);
1643                                         width -= 32;
1644                                 }
1645                                 if (!width)
1646                                         break;
1647                                 ++idx;
1648                         }
1649                         info[idx] = (struct field_modify_info){4, 0,
1650                                                 MLX5_MODI_OUT_SIPV6_127_96};
1651                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1652                                                      (32 - width));
1653                 } else {
1654                         if (data->offset < 32)
1655                                 info[idx++] = (struct field_modify_info){4, 12,
1656                                                 MLX5_MODI_OUT_SIPV6_31_0};
1657                         if (data->offset < 64)
1658                                 info[idx++] = (struct field_modify_info){4, 8,
1659                                                 MLX5_MODI_OUT_SIPV6_63_32};
1660                         if (data->offset < 96)
1661                                 info[idx++] = (struct field_modify_info){4, 4,
1662                                                 MLX5_MODI_OUT_SIPV6_95_64};
1663                         if (data->offset < 128)
1664                                 info[idx++] = (struct field_modify_info){4, 0,
1665                                                 MLX5_MODI_OUT_SIPV6_127_96};
1666                 }
1667                 break;
1668         case RTE_FLOW_FIELD_IPV6_DST:
1669                 if (mask) {
1670                         if (data->offset < 32) {
1671                                 info[idx] = (struct field_modify_info){4, 12,
1672                                                 MLX5_MODI_OUT_DIPV6_31_0};
1673                                 if (width < 32) {
1674                                         mask[idx] =
1675                                                 rte_cpu_to_be_32(0xffffffff >>
1676                                                                  (32 - width));
1677                                         width = 0;
1678                                 } else {
1679                                         mask[idx] = RTE_BE32(0xffffffff);
1680                                         width -= 32;
1681                                 }
1682                                 if (!width)
1683                                         break;
1684                                 ++idx;
1685                         }
1686                         if (data->offset < 64) {
1687                                 info[idx] = (struct field_modify_info){4, 8,
1688                                                 MLX5_MODI_OUT_DIPV6_63_32};
1689                                 if (width < 32) {
1690                                         mask[idx] =
1691                                                 rte_cpu_to_be_32(0xffffffff >>
1692                                                                  (32 - width));
1693                                         width = 0;
1694                                 } else {
1695                                         mask[idx] = RTE_BE32(0xffffffff);
1696                                         width -= 32;
1697                                 }
1698                                 if (!width)
1699                                         break;
1700                                 ++idx;
1701                         }
1702                         if (data->offset < 96) {
1703                                 info[idx] = (struct field_modify_info){4, 4,
1704                                                 MLX5_MODI_OUT_DIPV6_95_64};
1705                                 if (width < 32) {
1706                                         mask[idx] =
1707                                                 rte_cpu_to_be_32(0xffffffff >>
1708                                                                  (32 - width));
1709                                         width = 0;
1710                                 } else {
1711                                         mask[idx] = RTE_BE32(0xffffffff);
1712                                         width -= 32;
1713                                 }
1714                                 if (!width)
1715                                         break;
1716                                 ++idx;
1717                         }
1718                         info[idx] = (struct field_modify_info){4, 0,
1719                                                 MLX5_MODI_OUT_DIPV6_127_96};
1720                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1721                                                      (32 - width));
1722                 } else {
1723                         if (data->offset < 32)
1724                                 info[idx++] = (struct field_modify_info){4, 12,
1725                                                 MLX5_MODI_OUT_DIPV6_31_0};
1726                         if (data->offset < 64)
1727                                 info[idx++] = (struct field_modify_info){4, 8,
1728                                                 MLX5_MODI_OUT_DIPV6_63_32};
1729                         if (data->offset < 96)
1730                                 info[idx++] = (struct field_modify_info){4, 4,
1731                                                 MLX5_MODI_OUT_DIPV6_95_64};
1732                         if (data->offset < 128)
1733                                 info[idx++] = (struct field_modify_info){4, 0,
1734                                                 MLX5_MODI_OUT_DIPV6_127_96};
1735                 }
1736                 break;
1737         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1738                 info[idx] = (struct field_modify_info){2, 0,
1739                                         MLX5_MODI_OUT_TCP_SPORT};
1740                 if (mask)
1741                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1742                 break;
1743         case RTE_FLOW_FIELD_TCP_PORT_DST:
1744                 info[idx] = (struct field_modify_info){2, 0,
1745                                         MLX5_MODI_OUT_TCP_DPORT};
1746                 if (mask)
1747                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1748                 break;
1749         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1750                 info[idx] = (struct field_modify_info){4, 0,
1751                                         MLX5_MODI_OUT_TCP_SEQ_NUM};
1752                 if (mask)
1753                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1754                                                      (32 - width));
1755                 break;
1756         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1757                 info[idx] = (struct field_modify_info){4, 0,
1758                                         MLX5_MODI_OUT_TCP_ACK_NUM};
1759                 if (mask)
1760                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1761                                                      (32 - width));
1762                 break;
1763         case RTE_FLOW_FIELD_TCP_FLAGS:
1764                 info[idx] = (struct field_modify_info){2, 0,
1765                                         MLX5_MODI_OUT_TCP_FLAGS};
1766                 if (mask)
1767                         mask[idx] = rte_cpu_to_be_16(0x1ff >> (9 - width));
1768                 break;
1769         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1770                 info[idx] = (struct field_modify_info){2, 0,
1771                                         MLX5_MODI_OUT_UDP_SPORT};
1772                 if (mask)
1773                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1774                 break;
1775         case RTE_FLOW_FIELD_UDP_PORT_DST:
1776                 info[idx] = (struct field_modify_info){2, 0,
1777                                         MLX5_MODI_OUT_UDP_DPORT};
1778                 if (mask)
1779                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1780                 break;
1781         case RTE_FLOW_FIELD_VXLAN_VNI:
1782                 /* not supported yet */
1783                 break;
1784         case RTE_FLOW_FIELD_GENEVE_VNI:
1785                 /* not supported yet*/
1786                 break;
1787         case RTE_FLOW_FIELD_GTP_TEID:
1788                 info[idx] = (struct field_modify_info){4, 0,
1789                                         MLX5_MODI_GTP_TEID};
1790                 if (mask)
1791                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1792                                                      (32 - width));
1793                 break;
1794         case RTE_FLOW_FIELD_TAG:
1795                 {
1796                         int reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG,
1797                                                    data->level, error);
1798                         if (reg < 0)
1799                                 return;
1800                         MLX5_ASSERT(reg != REG_NON);
1801                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1802                         info[idx] = (struct field_modify_info){4, 0,
1803                                                 reg_to_field[reg]};
1804                         if (mask)
1805                                 mask[idx] =
1806                                         rte_cpu_to_be_32(0xffffffff >>
1807                                                          (32 - width));
1808                 }
1809                 break;
1810         case RTE_FLOW_FIELD_MARK:
1811                 {
1812                         uint32_t mark_mask = priv->sh->dv_mark_mask;
1813                         uint32_t mark_count = __builtin_popcount(mark_mask);
1814                         int reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK,
1815                                                        0, error);
1816                         if (reg < 0)
1817                                 return;
1818                         MLX5_ASSERT(reg != REG_NON);
1819                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1820                         info[idx] = (struct field_modify_info){4, 0,
1821                                                 reg_to_field[reg]};
1822                         if (mask)
1823                                 mask[idx] = rte_cpu_to_be_32((mark_mask >>
1824                                          (mark_count - width)) & mark_mask);
1825                 }
1826                 break;
1827         case RTE_FLOW_FIELD_META:
1828                 {
1829                         uint32_t meta_mask = priv->sh->dv_meta_mask;
1830                         uint32_t meta_count = __builtin_popcount(meta_mask);
1831                         uint32_t msk_c0 =
1832                                 rte_cpu_to_be_32(priv->sh->dv_regc0_mask);
1833                         uint32_t shl_c0 = rte_bsf32(msk_c0);
1834                         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1835                         if (reg < 0)
1836                                 return;
1837                         MLX5_ASSERT(reg != REG_NON);
1838                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1839                         if (reg == REG_C_0)
1840                                 *shift = shl_c0;
1841                         info[idx] = (struct field_modify_info){4, 0,
1842                                                 reg_to_field[reg]};
1843                         if (mask)
1844                                 mask[idx] = rte_cpu_to_be_32((meta_mask >>
1845                                         (meta_count - width)) & meta_mask);
1846                 }
1847                 break;
1848         case RTE_FLOW_FIELD_POINTER:
1849         case RTE_FLOW_FIELD_VALUE:
1850         default:
1851                 MLX5_ASSERT(false);
1852                 break;
1853         }
1854 }
1855
1856 /**
1857  * Convert modify_field action to DV specification.
1858  *
1859  * @param[in] dev
1860  *   Pointer to the rte_eth_dev structure.
1861  * @param[in,out] resource
1862  *   Pointer to the modify-header resource.
1863  * @param[in] action
1864  *   Pointer to action specification.
1865  * @param[in] attr
1866  *   Attributes of flow that includes this item.
1867  * @param[out] error
1868  *   Pointer to the error structure.
1869  *
1870  * @return
1871  *   0 on success, a negative errno value otherwise and rte_errno is set.
1872  */
1873 static int
1874 flow_dv_convert_action_modify_field
1875                         (struct rte_eth_dev *dev,
1876                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1877                          const struct rte_flow_action *action,
1878                          const struct rte_flow_attr *attr,
1879                          struct rte_flow_error *error)
1880 {
1881         const struct rte_flow_action_modify_field *conf =
1882                 (const struct rte_flow_action_modify_field *)(action->conf);
1883         struct rte_flow_item item = {
1884                 .spec = NULL,
1885                 .mask = NULL
1886         };
1887         struct field_modify_info field[MLX5_ACT_MAX_MOD_FIELDS] = {
1888                                                                 {0, 0, 0} };
1889         struct field_modify_info dcopy[MLX5_ACT_MAX_MOD_FIELDS] = {
1890                                                                 {0, 0, 0} };
1891         uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0};
1892         uint32_t type;
1893         uint32_t shift = 0;
1894
1895         if (conf->src.field == RTE_FLOW_FIELD_POINTER ||
1896             conf->src.field == RTE_FLOW_FIELD_VALUE) {
1897                 type = MLX5_MODIFICATION_TYPE_SET;
1898                 /** For SET fill the destination field (field) first. */
1899                 mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask,
1900                                                   conf->width, &shift, dev,
1901                                                   attr, error);
1902                 item.spec = conf->src.field == RTE_FLOW_FIELD_POINTER ?
1903                                         (void *)(uintptr_t)conf->src.pvalue :
1904                                         (void *)(uintptr_t)&conf->src.value;
1905         } else {
1906                 type = MLX5_MODIFICATION_TYPE_COPY;
1907                 /** For COPY fill the destination field (dcopy) without mask. */
1908                 mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL,
1909                                                   conf->width, &shift, dev,
1910                                                   attr, error);
1911                 /** Then construct the source field (field) with mask. */
1912                 mlx5_flow_field_id_to_modify_info(&conf->src, field, mask,
1913                                                   conf->width, &shift,
1914                                                   dev, attr, error);
1915         }
1916         item.mask = &mask;
1917         return flow_dv_convert_modify_action(&item,
1918                         field, dcopy, resource, type, error);
1919 }
1920
1921 /**
1922  * Validate MARK item.
1923  *
1924  * @param[in] dev
1925  *   Pointer to the rte_eth_dev structure.
1926  * @param[in] item
1927  *   Item specification.
1928  * @param[in] attr
1929  *   Attributes of flow that includes this item.
1930  * @param[out] error
1931  *   Pointer to error structure.
1932  *
1933  * @return
1934  *   0 on success, a negative errno value otherwise and rte_errno is set.
1935  */
1936 static int
1937 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1938                            const struct rte_flow_item *item,
1939                            const struct rte_flow_attr *attr __rte_unused,
1940                            struct rte_flow_error *error)
1941 {
1942         struct mlx5_priv *priv = dev->data->dev_private;
1943         struct mlx5_dev_config *config = &priv->config;
1944         const struct rte_flow_item_mark *spec = item->spec;
1945         const struct rte_flow_item_mark *mask = item->mask;
1946         const struct rte_flow_item_mark nic_mask = {
1947                 .id = priv->sh->dv_mark_mask,
1948         };
1949         int ret;
1950
1951         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1952                 return rte_flow_error_set(error, ENOTSUP,
1953                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1954                                           "extended metadata feature"
1955                                           " isn't enabled");
1956         if (!mlx5_flow_ext_mreg_supported(dev))
1957                 return rte_flow_error_set(error, ENOTSUP,
1958                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1959                                           "extended metadata register"
1960                                           " isn't supported");
1961         if (!nic_mask.id)
1962                 return rte_flow_error_set(error, ENOTSUP,
1963                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1964                                           "extended metadata register"
1965                                           " isn't available");
1966         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1967         if (ret < 0)
1968                 return ret;
1969         if (!spec)
1970                 return rte_flow_error_set(error, EINVAL,
1971                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1972                                           item->spec,
1973                                           "data cannot be empty");
1974         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1975                 return rte_flow_error_set(error, EINVAL,
1976                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1977                                           &spec->id,
1978                                           "mark id exceeds the limit");
1979         if (!mask)
1980                 mask = &nic_mask;
1981         if (!mask->id)
1982                 return rte_flow_error_set(error, EINVAL,
1983                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1984                                         "mask cannot be zero");
1985
1986         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1987                                         (const uint8_t *)&nic_mask,
1988                                         sizeof(struct rte_flow_item_mark),
1989                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1990         if (ret < 0)
1991                 return ret;
1992         return 0;
1993 }
1994
1995 /**
1996  * Validate META item.
1997  *
1998  * @param[in] dev
1999  *   Pointer to the rte_eth_dev structure.
2000  * @param[in] item
2001  *   Item specification.
2002  * @param[in] attr
2003  *   Attributes of flow that includes this item.
2004  * @param[out] error
2005  *   Pointer to error structure.
2006  *
2007  * @return
2008  *   0 on success, a negative errno value otherwise and rte_errno is set.
2009  */
2010 static int
2011 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
2012                            const struct rte_flow_item *item,
2013                            const struct rte_flow_attr *attr,
2014                            struct rte_flow_error *error)
2015 {
2016         struct mlx5_priv *priv = dev->data->dev_private;
2017         struct mlx5_dev_config *config = &priv->config;
2018         const struct rte_flow_item_meta *spec = item->spec;
2019         const struct rte_flow_item_meta *mask = item->mask;
2020         struct rte_flow_item_meta nic_mask = {
2021                 .data = UINT32_MAX
2022         };
2023         int reg;
2024         int ret;
2025
2026         if (!spec)
2027                 return rte_flow_error_set(error, EINVAL,
2028                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2029                                           item->spec,
2030                                           "data cannot be empty");
2031         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
2032                 if (!mlx5_flow_ext_mreg_supported(dev))
2033                         return rte_flow_error_set(error, ENOTSUP,
2034                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2035                                           "extended metadata register"
2036                                           " isn't supported");
2037                 reg = flow_dv_get_metadata_reg(dev, attr, error);
2038                 if (reg < 0)
2039                         return reg;
2040                 if (reg == REG_NON)
2041                         return rte_flow_error_set(error, ENOTSUP,
2042                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2043                                         "unavalable extended metadata register");
2044                 if (reg == REG_B)
2045                         return rte_flow_error_set(error, ENOTSUP,
2046                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2047                                           "match on reg_b "
2048                                           "isn't supported");
2049                 if (reg != REG_A)
2050                         nic_mask.data = priv->sh->dv_meta_mask;
2051         } else {
2052                 if (attr->transfer)
2053                         return rte_flow_error_set(error, ENOTSUP,
2054                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2055                                         "extended metadata feature "
2056                                         "should be enabled when "
2057                                         "meta item is requested "
2058                                         "with e-switch mode ");
2059                 if (attr->ingress)
2060                         return rte_flow_error_set(error, ENOTSUP,
2061                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2062                                         "match on metadata for ingress "
2063                                         "is not supported in legacy "
2064                                         "metadata mode");
2065         }
2066         if (!mask)
2067                 mask = &rte_flow_item_meta_mask;
2068         if (!mask->data)
2069                 return rte_flow_error_set(error, EINVAL,
2070                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2071                                         "mask cannot be zero");
2072
2073         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2074                                         (const uint8_t *)&nic_mask,
2075                                         sizeof(struct rte_flow_item_meta),
2076                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2077         return ret;
2078 }
2079
2080 /**
2081  * Validate TAG item.
2082  *
2083  * @param[in] dev
2084  *   Pointer to the rte_eth_dev structure.
2085  * @param[in] item
2086  *   Item specification.
2087  * @param[in] attr
2088  *   Attributes of flow that includes this item.
2089  * @param[out] error
2090  *   Pointer to error structure.
2091  *
2092  * @return
2093  *   0 on success, a negative errno value otherwise and rte_errno is set.
2094  */
2095 static int
2096 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
2097                           const struct rte_flow_item *item,
2098                           const struct rte_flow_attr *attr __rte_unused,
2099                           struct rte_flow_error *error)
2100 {
2101         const struct rte_flow_item_tag *spec = item->spec;
2102         const struct rte_flow_item_tag *mask = item->mask;
2103         const struct rte_flow_item_tag nic_mask = {
2104                 .data = RTE_BE32(UINT32_MAX),
2105                 .index = 0xff,
2106         };
2107         int ret;
2108
2109         if (!mlx5_flow_ext_mreg_supported(dev))
2110                 return rte_flow_error_set(error, ENOTSUP,
2111                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2112                                           "extensive metadata register"
2113                                           " isn't supported");
2114         if (!spec)
2115                 return rte_flow_error_set(error, EINVAL,
2116                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2117                                           item->spec,
2118                                           "data cannot be empty");
2119         if (!mask)
2120                 mask = &rte_flow_item_tag_mask;
2121         if (!mask->data)
2122                 return rte_flow_error_set(error, EINVAL,
2123                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2124                                         "mask cannot be zero");
2125
2126         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2127                                         (const uint8_t *)&nic_mask,
2128                                         sizeof(struct rte_flow_item_tag),
2129                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2130         if (ret < 0)
2131                 return ret;
2132         if (mask->index != 0xff)
2133                 return rte_flow_error_set(error, EINVAL,
2134                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2135                                           "partial mask for tag index"
2136                                           " is not supported");
2137         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
2138         if (ret < 0)
2139                 return ret;
2140         MLX5_ASSERT(ret != REG_NON);
2141         return 0;
2142 }
2143
2144 /**
2145  * Validate vport item.
2146  *
2147  * @param[in] dev
2148  *   Pointer to the rte_eth_dev structure.
2149  * @param[in] item
2150  *   Item specification.
2151  * @param[in] attr
2152  *   Attributes of flow that includes this item.
2153  * @param[in] item_flags
2154  *   Bit-fields that holds the items detected until now.
2155  * @param[out] error
2156  *   Pointer to error structure.
2157  *
2158  * @return
2159  *   0 on success, a negative errno value otherwise and rte_errno is set.
2160  */
2161 static int
2162 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
2163                               const struct rte_flow_item *item,
2164                               const struct rte_flow_attr *attr,
2165                               uint64_t item_flags,
2166                               struct rte_flow_error *error)
2167 {
2168         const struct rte_flow_item_port_id *spec = item->spec;
2169         const struct rte_flow_item_port_id *mask = item->mask;
2170         const struct rte_flow_item_port_id switch_mask = {
2171                         .id = 0xffffffff,
2172         };
2173         struct mlx5_priv *esw_priv;
2174         struct mlx5_priv *dev_priv;
2175         int ret;
2176
2177         if (!attr->transfer)
2178                 return rte_flow_error_set(error, EINVAL,
2179                                           RTE_FLOW_ERROR_TYPE_ITEM,
2180                                           NULL,
2181                                           "match on port id is valid only"
2182                                           " when transfer flag is enabled");
2183         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
2184                 return rte_flow_error_set(error, ENOTSUP,
2185                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2186                                           "multiple source ports are not"
2187                                           " supported");
2188         if (!mask)
2189                 mask = &switch_mask;
2190         if (mask->id != 0xffffffff)
2191                 return rte_flow_error_set(error, ENOTSUP,
2192                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2193                                            mask,
2194                                            "no support for partial mask on"
2195                                            " \"id\" field");
2196         ret = mlx5_flow_item_acceptable
2197                                 (item, (const uint8_t *)mask,
2198                                  (const uint8_t *)&rte_flow_item_port_id_mask,
2199                                  sizeof(struct rte_flow_item_port_id),
2200                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2201         if (ret)
2202                 return ret;
2203         if (!spec)
2204                 return 0;
2205         if (spec->id == MLX5_PORT_ESW_MGR)
2206                 return 0;
2207         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
2208         if (!esw_priv)
2209                 return rte_flow_error_set(error, rte_errno,
2210                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2211                                           "failed to obtain E-Switch info for"
2212                                           " port");
2213         dev_priv = mlx5_dev_to_eswitch_info(dev);
2214         if (!dev_priv)
2215                 return rte_flow_error_set(error, rte_errno,
2216                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2217                                           NULL,
2218                                           "failed to obtain E-Switch info");
2219         if (esw_priv->domain_id != dev_priv->domain_id)
2220                 return rte_flow_error_set(error, EINVAL,
2221                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2222                                           "cannot match on a port from a"
2223                                           " different E-Switch");
2224         return 0;
2225 }
2226
2227 /**
2228  * Validate VLAN item.
2229  *
2230  * @param[in] item
2231  *   Item specification.
2232  * @param[in] item_flags
2233  *   Bit-fields that holds the items detected until now.
2234  * @param[in] dev
2235  *   Ethernet device flow is being created on.
2236  * @param[out] error
2237  *   Pointer to error structure.
2238  *
2239  * @return
2240  *   0 on success, a negative errno value otherwise and rte_errno is set.
2241  */
2242 static int
2243 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
2244                            uint64_t item_flags,
2245                            struct rte_eth_dev *dev,
2246                            struct rte_flow_error *error)
2247 {
2248         const struct rte_flow_item_vlan *mask = item->mask;
2249         const struct rte_flow_item_vlan nic_mask = {
2250                 .tci = RTE_BE16(UINT16_MAX),
2251                 .inner_type = RTE_BE16(UINT16_MAX),
2252                 .has_more_vlan = 1,
2253         };
2254         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2255         int ret;
2256         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2257                                         MLX5_FLOW_LAYER_INNER_L4) :
2258                                        (MLX5_FLOW_LAYER_OUTER_L3 |
2259                                         MLX5_FLOW_LAYER_OUTER_L4);
2260         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2261                                         MLX5_FLOW_LAYER_OUTER_VLAN;
2262
2263         if (item_flags & vlanm)
2264                 return rte_flow_error_set(error, EINVAL,
2265                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2266                                           "multiple VLAN layers not supported");
2267         else if ((item_flags & l34m) != 0)
2268                 return rte_flow_error_set(error, EINVAL,
2269                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2270                                           "VLAN cannot follow L3/L4 layer");
2271         if (!mask)
2272                 mask = &rte_flow_item_vlan_mask;
2273         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2274                                         (const uint8_t *)&nic_mask,
2275                                         sizeof(struct rte_flow_item_vlan),
2276                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2277         if (ret)
2278                 return ret;
2279         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2280                 struct mlx5_priv *priv = dev->data->dev_private;
2281
2282                 if (priv->vmwa_context) {
2283                         /*
2284                          * Non-NULL context means we have a virtual machine
2285                          * and SR-IOV enabled, we have to create VLAN interface
2286                          * to make hypervisor to setup E-Switch vport
2287                          * context correctly. We avoid creating the multiple
2288                          * VLAN interfaces, so we cannot support VLAN tag mask.
2289                          */
2290                         return rte_flow_error_set(error, EINVAL,
2291                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2292                                                   item,
2293                                                   "VLAN tag mask is not"
2294                                                   " supported in virtual"
2295                                                   " environment");
2296                 }
2297         }
2298         return 0;
2299 }
2300
2301 /*
2302  * GTP flags are contained in 1 byte of the format:
2303  * -------------------------------------------
2304  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
2305  * |-----------------------------------------|
2306  * | value | Version | PT | Res | E | S | PN |
2307  * -------------------------------------------
2308  *
2309  * Matching is supported only for GTP flags E, S, PN.
2310  */
2311 #define MLX5_GTP_FLAGS_MASK     0x07
2312
2313 /**
2314  * Validate GTP item.
2315  *
2316  * @param[in] dev
2317  *   Pointer to the rte_eth_dev structure.
2318  * @param[in] item
2319  *   Item specification.
2320  * @param[in] item_flags
2321  *   Bit-fields that holds the items detected until now.
2322  * @param[out] error
2323  *   Pointer to error structure.
2324  *
2325  * @return
2326  *   0 on success, a negative errno value otherwise and rte_errno is set.
2327  */
2328 static int
2329 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
2330                           const struct rte_flow_item *item,
2331                           uint64_t item_flags,
2332                           struct rte_flow_error *error)
2333 {
2334         struct mlx5_priv *priv = dev->data->dev_private;
2335         const struct rte_flow_item_gtp *spec = item->spec;
2336         const struct rte_flow_item_gtp *mask = item->mask;
2337         const struct rte_flow_item_gtp nic_mask = {
2338                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
2339                 .msg_type = 0xff,
2340                 .teid = RTE_BE32(0xffffffff),
2341         };
2342
2343         if (!priv->config.hca_attr.tunnel_stateless_gtp)
2344                 return rte_flow_error_set(error, ENOTSUP,
2345                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2346                                           "GTP support is not enabled");
2347         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2348                 return rte_flow_error_set(error, ENOTSUP,
2349                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2350                                           "multiple tunnel layers not"
2351                                           " supported");
2352         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2353                 return rte_flow_error_set(error, EINVAL,
2354                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2355                                           "no outer UDP layer found");
2356         if (!mask)
2357                 mask = &rte_flow_item_gtp_mask;
2358         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
2359                 return rte_flow_error_set(error, ENOTSUP,
2360                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2361                                           "Match is supported for GTP"
2362                                           " flags only");
2363         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2364                                          (const uint8_t *)&nic_mask,
2365                                          sizeof(struct rte_flow_item_gtp),
2366                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2367 }
2368
2369 /**
2370  * Validate GTP PSC item.
2371  *
2372  * @param[in] item
2373  *   Item specification.
2374  * @param[in] last_item
2375  *   Previous validated item in the pattern items.
2376  * @param[in] gtp_item
2377  *   Previous GTP item specification.
2378  * @param[in] attr
2379  *   Pointer to flow attributes.
2380  * @param[out] error
2381  *   Pointer to error structure.
2382  *
2383  * @return
2384  *   0 on success, a negative errno value otherwise and rte_errno is set.
2385  */
2386 static int
2387 flow_dv_validate_item_gtp_psc(const struct rte_flow_item *item,
2388                               uint64_t last_item,
2389                               const struct rte_flow_item *gtp_item,
2390                               const struct rte_flow_attr *attr,
2391                               struct rte_flow_error *error)
2392 {
2393         const struct rte_flow_item_gtp *gtp_spec;
2394         const struct rte_flow_item_gtp *gtp_mask;
2395         const struct rte_flow_item_gtp_psc *mask;
2396         const struct rte_flow_item_gtp_psc nic_mask = {
2397                 .hdr.type = 0xF,
2398                 .hdr.qfi = 0x3F,
2399         };
2400
2401         if (!gtp_item || !(last_item & MLX5_FLOW_LAYER_GTP))
2402                 return rte_flow_error_set
2403                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2404                          "GTP PSC item must be preceded with GTP item");
2405         gtp_spec = gtp_item->spec;
2406         gtp_mask = gtp_item->mask ? gtp_item->mask : &rte_flow_item_gtp_mask;
2407         /* GTP spec and E flag is requested to match zero. */
2408         if (gtp_spec &&
2409                 (gtp_mask->v_pt_rsv_flags &
2410                 ~gtp_spec->v_pt_rsv_flags & MLX5_GTP_EXT_HEADER_FLAG))
2411                 return rte_flow_error_set
2412                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2413                          "GTP E flag must be 1 to match GTP PSC");
2414         /* Check the flow is not created in group zero. */
2415         if (!attr->transfer && !attr->group)
2416                 return rte_flow_error_set
2417                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2418                          "GTP PSC is not supported for group 0");
2419         /* GTP spec is here and E flag is requested to match zero. */
2420         if (!item->spec)
2421                 return 0;
2422         mask = item->mask ? item->mask : &rte_flow_item_gtp_psc_mask;
2423         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2424                                          (const uint8_t *)&nic_mask,
2425                                          sizeof(struct rte_flow_item_gtp_psc),
2426                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2427 }
2428
2429 /**
2430  * Validate IPV4 item.
2431  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
2432  * add specific validation of fragment_offset field,
2433  *
2434  * @param[in] item
2435  *   Item specification.
2436  * @param[in] item_flags
2437  *   Bit-fields that holds the items detected until now.
2438  * @param[out] error
2439  *   Pointer to error structure.
2440  *
2441  * @return
2442  *   0 on success, a negative errno value otherwise and rte_errno is set.
2443  */
2444 static int
2445 flow_dv_validate_item_ipv4(struct rte_eth_dev *dev,
2446                            const struct rte_flow_item *item,
2447                            uint64_t item_flags, uint64_t last_item,
2448                            uint16_t ether_type, struct rte_flow_error *error)
2449 {
2450         int ret;
2451         struct mlx5_priv *priv = dev->data->dev_private;
2452         const struct rte_flow_item_ipv4 *spec = item->spec;
2453         const struct rte_flow_item_ipv4 *last = item->last;
2454         const struct rte_flow_item_ipv4 *mask = item->mask;
2455         rte_be16_t fragment_offset_spec = 0;
2456         rte_be16_t fragment_offset_last = 0;
2457         struct rte_flow_item_ipv4 nic_ipv4_mask = {
2458                 .hdr = {
2459                         .src_addr = RTE_BE32(0xffffffff),
2460                         .dst_addr = RTE_BE32(0xffffffff),
2461                         .type_of_service = 0xff,
2462                         .fragment_offset = RTE_BE16(0xffff),
2463                         .next_proto_id = 0xff,
2464                         .time_to_live = 0xff,
2465                 },
2466         };
2467
2468         if (mask && (mask->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK)) {
2469                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2470                 bool ihl_cap = !tunnel ? priv->config.hca_attr.outer_ipv4_ihl :
2471                                priv->config.hca_attr.inner_ipv4_ihl;
2472                 if (!ihl_cap)
2473                         return rte_flow_error_set(error, ENOTSUP,
2474                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2475                                                   item,
2476                                                   "IPV4 ihl offload not supported");
2477                 nic_ipv4_mask.hdr.version_ihl = mask->hdr.version_ihl;
2478         }
2479         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
2480                                            ether_type, &nic_ipv4_mask,
2481                                            MLX5_ITEM_RANGE_ACCEPTED, error);
2482         if (ret < 0)
2483                 return ret;
2484         if (spec && mask)
2485                 fragment_offset_spec = spec->hdr.fragment_offset &
2486                                        mask->hdr.fragment_offset;
2487         if (!fragment_offset_spec)
2488                 return 0;
2489         /*
2490          * spec and mask are valid, enforce using full mask to make sure the
2491          * complete value is used correctly.
2492          */
2493         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2494                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2495                 return rte_flow_error_set(error, EINVAL,
2496                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2497                                           item, "must use full mask for"
2498                                           " fragment_offset");
2499         /*
2500          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
2501          * indicating this is 1st fragment of fragmented packet.
2502          * This is not yet supported in MLX5, return appropriate error message.
2503          */
2504         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
2505                 return rte_flow_error_set(error, ENOTSUP,
2506                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2507                                           "match on first fragment not "
2508                                           "supported");
2509         if (fragment_offset_spec && !last)
2510                 return rte_flow_error_set(error, ENOTSUP,
2511                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2512                                           "specified value not supported");
2513         /* spec and last are valid, validate the specified range. */
2514         fragment_offset_last = last->hdr.fragment_offset &
2515                                mask->hdr.fragment_offset;
2516         /*
2517          * Match on fragment_offset spec 0x2001 and last 0x3fff
2518          * means MF is 1 and frag-offset is > 0.
2519          * This packet is fragment 2nd and onward, excluding last.
2520          * This is not yet supported in MLX5, return appropriate
2521          * error message.
2522          */
2523         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
2524             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2525                 return rte_flow_error_set(error, ENOTSUP,
2526                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2527                                           last, "match on following "
2528                                           "fragments not supported");
2529         /*
2530          * Match on fragment_offset spec 0x0001 and last 0x1fff
2531          * means MF is 0 and frag-offset is > 0.
2532          * This packet is last fragment of fragmented packet.
2533          * This is not yet supported in MLX5, return appropriate
2534          * error message.
2535          */
2536         if (fragment_offset_spec == RTE_BE16(1) &&
2537             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
2538                 return rte_flow_error_set(error, ENOTSUP,
2539                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2540                                           last, "match on last "
2541                                           "fragment not supported");
2542         /*
2543          * Match on fragment_offset spec 0x0001 and last 0x3fff
2544          * means MF and/or frag-offset is not 0.
2545          * This is a fragmented packet.
2546          * Other range values are invalid and rejected.
2547          */
2548         if (!(fragment_offset_spec == RTE_BE16(1) &&
2549               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
2550                 return rte_flow_error_set(error, ENOTSUP,
2551                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2552                                           "specified range not supported");
2553         return 0;
2554 }
2555
2556 /**
2557  * Validate IPV6 fragment extension item.
2558  *
2559  * @param[in] item
2560  *   Item specification.
2561  * @param[in] item_flags
2562  *   Bit-fields that holds the items detected until now.
2563  * @param[out] error
2564  *   Pointer to error structure.
2565  *
2566  * @return
2567  *   0 on success, a negative errno value otherwise and rte_errno is set.
2568  */
2569 static int
2570 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
2571                                     uint64_t item_flags,
2572                                     struct rte_flow_error *error)
2573 {
2574         const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
2575         const struct rte_flow_item_ipv6_frag_ext *last = item->last;
2576         const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
2577         rte_be16_t frag_data_spec = 0;
2578         rte_be16_t frag_data_last = 0;
2579         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2580         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2581                                       MLX5_FLOW_LAYER_OUTER_L4;
2582         int ret = 0;
2583         struct rte_flow_item_ipv6_frag_ext nic_mask = {
2584                 .hdr = {
2585                         .next_header = 0xff,
2586                         .frag_data = RTE_BE16(0xffff),
2587                 },
2588         };
2589
2590         if (item_flags & l4m)
2591                 return rte_flow_error_set(error, EINVAL,
2592                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2593                                           "ipv6 fragment extension item cannot "
2594                                           "follow L4 item.");
2595         if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
2596             (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2597                 return rte_flow_error_set(error, EINVAL,
2598                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2599                                           "ipv6 fragment extension item must "
2600                                           "follow ipv6 item");
2601         if (spec && mask)
2602                 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
2603         if (!frag_data_spec)
2604                 return 0;
2605         /*
2606          * spec and mask are valid, enforce using full mask to make sure the
2607          * complete value is used correctly.
2608          */
2609         if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
2610                                 RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2611                 return rte_flow_error_set(error, EINVAL,
2612                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2613                                           item, "must use full mask for"
2614                                           " frag_data");
2615         /*
2616          * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
2617          * This is 1st fragment of fragmented packet.
2618          */
2619         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
2620                 return rte_flow_error_set(error, ENOTSUP,
2621                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2622                                           "match on first fragment not "
2623                                           "supported");
2624         if (frag_data_spec && !last)
2625                 return rte_flow_error_set(error, EINVAL,
2626                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2627                                           "specified value not supported");
2628         ret = mlx5_flow_item_acceptable
2629                                 (item, (const uint8_t *)mask,
2630                                  (const uint8_t *)&nic_mask,
2631                                  sizeof(struct rte_flow_item_ipv6_frag_ext),
2632                                  MLX5_ITEM_RANGE_ACCEPTED, error);
2633         if (ret)
2634                 return ret;
2635         /* spec and last are valid, validate the specified range. */
2636         frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
2637         /*
2638          * Match on frag_data spec 0x0009 and last 0xfff9
2639          * means M is 1 and frag-offset is > 0.
2640          * This packet is fragment 2nd and onward, excluding last.
2641          * This is not yet supported in MLX5, return appropriate
2642          * error message.
2643          */
2644         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
2645                                        RTE_IPV6_EHDR_MF_MASK) &&
2646             frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2647                 return rte_flow_error_set(error, ENOTSUP,
2648                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2649                                           last, "match on following "
2650                                           "fragments not supported");
2651         /*
2652          * Match on frag_data spec 0x0008 and last 0xfff8
2653          * means M is 0 and frag-offset is > 0.
2654          * This packet is last fragment of fragmented packet.
2655          * This is not yet supported in MLX5, return appropriate
2656          * error message.
2657          */
2658         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
2659             frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
2660                 return rte_flow_error_set(error, ENOTSUP,
2661                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2662                                           last, "match on last "
2663                                           "fragment not supported");
2664         /* Other range values are invalid and rejected. */
2665         return rte_flow_error_set(error, EINVAL,
2666                                   RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2667                                   "specified range not supported");
2668 }
2669
2670 /*
2671  * Validate ASO CT item.
2672  *
2673  * @param[in] dev
2674  *   Pointer to the rte_eth_dev structure.
2675  * @param[in] item
2676  *   Item specification.
2677  * @param[in] item_flags
2678  *   Pointer to bit-fields that holds the items detected until now.
2679  * @param[out] error
2680  *   Pointer to error structure.
2681  *
2682  * @return
2683  *   0 on success, a negative errno value otherwise and rte_errno is set.
2684  */
2685 static int
2686 flow_dv_validate_item_aso_ct(struct rte_eth_dev *dev,
2687                              const struct rte_flow_item *item,
2688                              uint64_t *item_flags,
2689                              struct rte_flow_error *error)
2690 {
2691         const struct rte_flow_item_conntrack *spec = item->spec;
2692         const struct rte_flow_item_conntrack *mask = item->mask;
2693         RTE_SET_USED(dev);
2694         uint32_t flags;
2695
2696         if (*item_flags & MLX5_FLOW_LAYER_ASO_CT)
2697                 return rte_flow_error_set(error, EINVAL,
2698                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2699                                           "Only one CT is supported");
2700         if (!mask)
2701                 mask = &rte_flow_item_conntrack_mask;
2702         flags = spec->flags & mask->flags;
2703         if ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID) &&
2704             ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID) ||
2705              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD) ||
2706              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)))
2707                 return rte_flow_error_set(error, EINVAL,
2708                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2709                                           "Conflict status bits");
2710         /* State change also needs to be considered. */
2711         *item_flags |= MLX5_FLOW_LAYER_ASO_CT;
2712         return 0;
2713 }
2714
2715 /**
2716  * Validate the pop VLAN action.
2717  *
2718  * @param[in] dev
2719  *   Pointer to the rte_eth_dev structure.
2720  * @param[in] action_flags
2721  *   Holds the actions detected until now.
2722  * @param[in] action
2723  *   Pointer to the pop vlan action.
2724  * @param[in] item_flags
2725  *   The items found in this flow rule.
2726  * @param[in] attr
2727  *   Pointer to flow attributes.
2728  * @param[out] error
2729  *   Pointer to error structure.
2730  *
2731  * @return
2732  *   0 on success, a negative errno value otherwise and rte_errno is set.
2733  */
2734 static int
2735 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
2736                                  uint64_t action_flags,
2737                                  const struct rte_flow_action *action,
2738                                  uint64_t item_flags,
2739                                  const struct rte_flow_attr *attr,
2740                                  struct rte_flow_error *error)
2741 {
2742         const struct mlx5_priv *priv = dev->data->dev_private;
2743         struct mlx5_dev_ctx_shared *sh = priv->sh;
2744         bool direction_error = false;
2745
2746         if (!priv->sh->pop_vlan_action)
2747                 return rte_flow_error_set(error, ENOTSUP,
2748                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2749                                           NULL,
2750                                           "pop vlan action is not supported");
2751         /* Pop VLAN is not supported in egress except for CX6 FDB mode. */
2752         if (attr->transfer) {
2753                 bool fdb_tx = priv->representor_id != UINT16_MAX;
2754                 bool is_cx5 = sh->steering_format_version ==
2755                     MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5;
2756
2757                 if (fdb_tx && is_cx5)
2758                         direction_error = true;
2759         } else if (attr->egress) {
2760                 direction_error = true;
2761         }
2762         if (direction_error)
2763                 return rte_flow_error_set(error, ENOTSUP,
2764                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2765                                           NULL,
2766                                           "pop vlan action not supported for egress");
2767         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
2768                 return rte_flow_error_set(error, ENOTSUP,
2769                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2770                                           "no support for multiple VLAN "
2771                                           "actions");
2772         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
2773         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
2774             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
2775                 return rte_flow_error_set(error, ENOTSUP,
2776                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2777                                           NULL,
2778                                           "cannot pop vlan after decap without "
2779                                           "match on inner vlan in the flow");
2780         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
2781         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
2782             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2783                 return rte_flow_error_set(error, ENOTSUP,
2784                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2785                                           NULL,
2786                                           "cannot pop vlan without a "
2787                                           "match on (outer) vlan in the flow");
2788         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2789                 return rte_flow_error_set(error, EINVAL,
2790                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2791                                           "wrong action order, port_id should "
2792                                           "be after pop VLAN action");
2793         if (!attr->transfer && priv->representor)
2794                 return rte_flow_error_set(error, ENOTSUP,
2795                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2796                                           "pop vlan action for VF representor "
2797                                           "not supported on NIC table");
2798         return 0;
2799 }
2800
2801 /**
2802  * Get VLAN default info from vlan match info.
2803  *
2804  * @param[in] items
2805  *   the list of item specifications.
2806  * @param[out] vlan
2807  *   pointer VLAN info to fill to.
2808  *
2809  * @return
2810  *   0 on success, a negative errno value otherwise and rte_errno is set.
2811  */
2812 static void
2813 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2814                                   struct rte_vlan_hdr *vlan)
2815 {
2816         const struct rte_flow_item_vlan nic_mask = {
2817                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2818                                 MLX5DV_FLOW_VLAN_VID_MASK),
2819                 .inner_type = RTE_BE16(0xffff),
2820         };
2821
2822         if (items == NULL)
2823                 return;
2824         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2825                 int type = items->type;
2826
2827                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2828                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2829                         break;
2830         }
2831         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2832                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2833                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2834
2835                 /* If VLAN item in pattern doesn't contain data, return here. */
2836                 if (!vlan_v)
2837                         return;
2838                 if (!vlan_m)
2839                         vlan_m = &nic_mask;
2840                 /* Only full match values are accepted */
2841                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2842                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2843                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2844                         vlan->vlan_tci |=
2845                                 rte_be_to_cpu_16(vlan_v->tci &
2846                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2847                 }
2848                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2849                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2850                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2851                         vlan->vlan_tci |=
2852                                 rte_be_to_cpu_16(vlan_v->tci &
2853                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2854                 }
2855                 if (vlan_m->inner_type == nic_mask.inner_type)
2856                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2857                                                            vlan_m->inner_type);
2858         }
2859 }
2860
2861 /**
2862  * Validate the push VLAN action.
2863  *
2864  * @param[in] dev
2865  *   Pointer to the rte_eth_dev structure.
2866  * @param[in] action_flags
2867  *   Holds the actions detected until now.
2868  * @param[in] item_flags
2869  *   The items found in this flow rule.
2870  * @param[in] action
2871  *   Pointer to the action structure.
2872  * @param[in] attr
2873  *   Pointer to flow attributes
2874  * @param[out] error
2875  *   Pointer to error structure.
2876  *
2877  * @return
2878  *   0 on success, a negative errno value otherwise and rte_errno is set.
2879  */
2880 static int
2881 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2882                                   uint64_t action_flags,
2883                                   const struct rte_flow_item_vlan *vlan_m,
2884                                   const struct rte_flow_action *action,
2885                                   const struct rte_flow_attr *attr,
2886                                   struct rte_flow_error *error)
2887 {
2888         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2889         const struct mlx5_priv *priv = dev->data->dev_private;
2890         struct mlx5_dev_ctx_shared *sh = priv->sh;
2891         bool direction_error = false;
2892
2893         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2894             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2895                 return rte_flow_error_set(error, EINVAL,
2896                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2897                                           "invalid vlan ethertype");
2898         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2899                 return rte_flow_error_set(error, EINVAL,
2900                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2901                                           "wrong action order, port_id should "
2902                                           "be after push VLAN");
2903         /* Push VLAN is not supported in ingress except for CX6 FDB mode. */
2904         if (attr->transfer) {
2905                 bool fdb_tx = priv->representor_id != UINT16_MAX;
2906                 bool is_cx5 = sh->steering_format_version ==
2907                     MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5;
2908
2909                 if (!fdb_tx && is_cx5)
2910                         direction_error = true;
2911         } else if (attr->ingress) {
2912                 direction_error = true;
2913         }
2914         if (direction_error)
2915                 return rte_flow_error_set(error, ENOTSUP,
2916                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
2917                                           NULL,
2918                                           "push vlan action not supported for ingress");
2919         if (!attr->transfer && priv->representor)
2920                 return rte_flow_error_set(error, ENOTSUP,
2921                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2922                                           "push vlan action for VF representor "
2923                                           "not supported on NIC table");
2924         if (vlan_m &&
2925             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2926             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2927                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2928             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2929             !(mlx5_flow_find_action
2930                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2931                 return rte_flow_error_set(error, EINVAL,
2932                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2933                                           "not full match mask on VLAN PCP and "
2934                                           "there is no of_set_vlan_pcp action, "
2935                                           "push VLAN action cannot figure out "
2936                                           "PCP value");
2937         if (vlan_m &&
2938             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2939             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2940                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2941             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2942             !(mlx5_flow_find_action
2943                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2944                 return rte_flow_error_set(error, EINVAL,
2945                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2946                                           "not full match mask on VLAN VID and "
2947                                           "there is no of_set_vlan_vid action, "
2948                                           "push VLAN action cannot figure out "
2949                                           "VID value");
2950         (void)attr;
2951         return 0;
2952 }
2953
2954 /**
2955  * Validate the set VLAN PCP.
2956  *
2957  * @param[in] action_flags
2958  *   Holds the actions detected until now.
2959  * @param[in] actions
2960  *   Pointer to the list of actions remaining in the flow rule.
2961  * @param[out] error
2962  *   Pointer to error structure.
2963  *
2964  * @return
2965  *   0 on success, a negative errno value otherwise and rte_errno is set.
2966  */
2967 static int
2968 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2969                                      const struct rte_flow_action actions[],
2970                                      struct rte_flow_error *error)
2971 {
2972         const struct rte_flow_action *action = actions;
2973         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2974
2975         if (conf->vlan_pcp > 7)
2976                 return rte_flow_error_set(error, EINVAL,
2977                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2978                                           "VLAN PCP value is too big");
2979         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2980                 return rte_flow_error_set(error, ENOTSUP,
2981                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2982                                           "set VLAN PCP action must follow "
2983                                           "the push VLAN action");
2984         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2985                 return rte_flow_error_set(error, ENOTSUP,
2986                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2987                                           "Multiple VLAN PCP modification are "
2988                                           "not supported");
2989         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2990                 return rte_flow_error_set(error, EINVAL,
2991                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2992                                           "wrong action order, port_id should "
2993                                           "be after set VLAN PCP");
2994         return 0;
2995 }
2996
2997 /**
2998  * Validate the set VLAN VID.
2999  *
3000  * @param[in] item_flags
3001  *   Holds the items detected in this rule.
3002  * @param[in] action_flags
3003  *   Holds the actions detected until now.
3004  * @param[in] actions
3005  *   Pointer to the list of actions remaining in the flow rule.
3006  * @param[out] error
3007  *   Pointer to error structure.
3008  *
3009  * @return
3010  *   0 on success, a negative errno value otherwise and rte_errno is set.
3011  */
3012 static int
3013 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
3014                                      uint64_t action_flags,
3015                                      const struct rte_flow_action actions[],
3016                                      struct rte_flow_error *error)
3017 {
3018         const struct rte_flow_action *action = actions;
3019         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
3020
3021         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
3022                 return rte_flow_error_set(error, EINVAL,
3023                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3024                                           "VLAN VID value is too big");
3025         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
3026             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
3027                 return rte_flow_error_set(error, ENOTSUP,
3028                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3029                                           "set VLAN VID action must follow push"
3030                                           " VLAN action or match on VLAN item");
3031         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
3032                 return rte_flow_error_set(error, ENOTSUP,
3033                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3034                                           "Multiple VLAN VID modifications are "
3035                                           "not supported");
3036         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
3037                 return rte_flow_error_set(error, EINVAL,
3038                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3039                                           "wrong action order, port_id should "
3040                                           "be after set VLAN VID");
3041         return 0;
3042 }
3043
3044 /*
3045  * Validate the FLAG action.
3046  *
3047  * @param[in] dev
3048  *   Pointer to the rte_eth_dev structure.
3049  * @param[in] action_flags
3050  *   Holds the actions detected until now.
3051  * @param[in] attr
3052  *   Pointer to flow attributes
3053  * @param[out] error
3054  *   Pointer to error structure.
3055  *
3056  * @return
3057  *   0 on success, a negative errno value otherwise and rte_errno is set.
3058  */
3059 static int
3060 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
3061                              uint64_t action_flags,
3062                              const struct rte_flow_attr *attr,
3063                              struct rte_flow_error *error)
3064 {
3065         struct mlx5_priv *priv = dev->data->dev_private;
3066         struct mlx5_dev_config *config = &priv->config;
3067         int ret;
3068
3069         /* Fall back if no extended metadata register support. */
3070         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3071                 return mlx5_flow_validate_action_flag(action_flags, attr,
3072                                                       error);
3073         /* Extensive metadata mode requires registers. */
3074         if (!mlx5_flow_ext_mreg_supported(dev))
3075                 return rte_flow_error_set(error, ENOTSUP,
3076                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3077                                           "no metadata registers "
3078                                           "to support flag action");
3079         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
3080                 return rte_flow_error_set(error, ENOTSUP,
3081                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3082                                           "extended metadata register"
3083                                           " isn't available");
3084         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3085         if (ret < 0)
3086                 return ret;
3087         MLX5_ASSERT(ret > 0);
3088         if (action_flags & MLX5_FLOW_ACTION_MARK)
3089                 return rte_flow_error_set(error, EINVAL,
3090                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3091                                           "can't mark and flag in same flow");
3092         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3093                 return rte_flow_error_set(error, EINVAL,
3094                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3095                                           "can't have 2 flag"
3096                                           " actions in same flow");
3097         return 0;
3098 }
3099
3100 /**
3101  * Validate MARK action.
3102  *
3103  * @param[in] dev
3104  *   Pointer to the rte_eth_dev structure.
3105  * @param[in] action
3106  *   Pointer to action.
3107  * @param[in] action_flags
3108  *   Holds the actions detected until now.
3109  * @param[in] attr
3110  *   Pointer to flow attributes
3111  * @param[out] error
3112  *   Pointer to error structure.
3113  *
3114  * @return
3115  *   0 on success, a negative errno value otherwise and rte_errno is set.
3116  */
3117 static int
3118 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
3119                              const struct rte_flow_action *action,
3120                              uint64_t action_flags,
3121                              const struct rte_flow_attr *attr,
3122                              struct rte_flow_error *error)
3123 {
3124         struct mlx5_priv *priv = dev->data->dev_private;
3125         struct mlx5_dev_config *config = &priv->config;
3126         const struct rte_flow_action_mark *mark = action->conf;
3127         int ret;
3128
3129         if (is_tunnel_offload_active(dev))
3130                 return rte_flow_error_set(error, ENOTSUP,
3131                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3132                                           "no mark action "
3133                                           "if tunnel offload active");
3134         /* Fall back if no extended metadata register support. */
3135         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3136                 return mlx5_flow_validate_action_mark(action, action_flags,
3137                                                       attr, error);
3138         /* Extensive metadata mode requires registers. */
3139         if (!mlx5_flow_ext_mreg_supported(dev))
3140                 return rte_flow_error_set(error, ENOTSUP,
3141                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3142                                           "no metadata registers "
3143                                           "to support mark action");
3144         if (!priv->sh->dv_mark_mask)
3145                 return rte_flow_error_set(error, ENOTSUP,
3146                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3147                                           "extended metadata register"
3148                                           " isn't available");
3149         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3150         if (ret < 0)
3151                 return ret;
3152         MLX5_ASSERT(ret > 0);
3153         if (!mark)
3154                 return rte_flow_error_set(error, EINVAL,
3155                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3156                                           "configuration cannot be null");
3157         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
3158                 return rte_flow_error_set(error, EINVAL,
3159                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3160                                           &mark->id,
3161                                           "mark id exceeds the limit");
3162         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3163                 return rte_flow_error_set(error, EINVAL,
3164                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3165                                           "can't flag and mark in same flow");
3166         if (action_flags & MLX5_FLOW_ACTION_MARK)
3167                 return rte_flow_error_set(error, EINVAL,
3168                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3169                                           "can't have 2 mark actions in same"
3170                                           " flow");
3171         return 0;
3172 }
3173
3174 /**
3175  * Validate SET_META action.
3176  *
3177  * @param[in] dev
3178  *   Pointer to the rte_eth_dev structure.
3179  * @param[in] action
3180  *   Pointer to the action structure.
3181  * @param[in] action_flags
3182  *   Holds the actions detected until now.
3183  * @param[in] attr
3184  *   Pointer to flow attributes
3185  * @param[out] error
3186  *   Pointer to error structure.
3187  *
3188  * @return
3189  *   0 on success, a negative errno value otherwise and rte_errno is set.
3190  */
3191 static int
3192 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
3193                                  const struct rte_flow_action *action,
3194                                  uint64_t action_flags __rte_unused,
3195                                  const struct rte_flow_attr *attr,
3196                                  struct rte_flow_error *error)
3197 {
3198         const struct rte_flow_action_set_meta *conf;
3199         uint32_t nic_mask = UINT32_MAX;
3200         int reg;
3201
3202         if (!mlx5_flow_ext_mreg_supported(dev))
3203                 return rte_flow_error_set(error, ENOTSUP,
3204                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3205                                           "extended metadata register"
3206                                           " isn't supported");
3207         reg = flow_dv_get_metadata_reg(dev, attr, error);
3208         if (reg < 0)
3209                 return reg;
3210         if (reg == REG_NON)
3211                 return rte_flow_error_set(error, ENOTSUP,
3212                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3213                                           "unavalable extended metadata register");
3214         if (reg != REG_A && reg != REG_B) {
3215                 struct mlx5_priv *priv = dev->data->dev_private;
3216
3217                 nic_mask = priv->sh->dv_meta_mask;
3218         }
3219         if (!(action->conf))
3220                 return rte_flow_error_set(error, EINVAL,
3221                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3222                                           "configuration cannot be null");
3223         conf = (const struct rte_flow_action_set_meta *)action->conf;
3224         if (!conf->mask)
3225                 return rte_flow_error_set(error, EINVAL,
3226                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3227                                           "zero mask doesn't have any effect");
3228         if (conf->mask & ~nic_mask)
3229                 return rte_flow_error_set(error, EINVAL,
3230                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3231                                           "meta data must be within reg C0");
3232         return 0;
3233 }
3234
3235 /**
3236  * Validate SET_TAG action.
3237  *
3238  * @param[in] dev
3239  *   Pointer to the rte_eth_dev structure.
3240  * @param[in] action
3241  *   Pointer to the action structure.
3242  * @param[in] action_flags
3243  *   Holds the actions detected until now.
3244  * @param[in] attr
3245  *   Pointer to flow attributes
3246  * @param[out] error
3247  *   Pointer to error structure.
3248  *
3249  * @return
3250  *   0 on success, a negative errno value otherwise and rte_errno is set.
3251  */
3252 static int
3253 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
3254                                 const struct rte_flow_action *action,
3255                                 uint64_t action_flags,
3256                                 const struct rte_flow_attr *attr,
3257                                 struct rte_flow_error *error)
3258 {
3259         const struct rte_flow_action_set_tag *conf;
3260         const uint64_t terminal_action_flags =
3261                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
3262                 MLX5_FLOW_ACTION_RSS;
3263         int ret;
3264
3265         if (!mlx5_flow_ext_mreg_supported(dev))
3266                 return rte_flow_error_set(error, ENOTSUP,
3267                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3268                                           "extensive metadata register"
3269                                           " isn't supported");
3270         if (!(action->conf))
3271                 return rte_flow_error_set(error, EINVAL,
3272                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3273                                           "configuration cannot be null");
3274         conf = (const struct rte_flow_action_set_tag *)action->conf;
3275         if (!conf->mask)
3276                 return rte_flow_error_set(error, EINVAL,
3277                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3278                                           "zero mask doesn't have any effect");
3279         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
3280         if (ret < 0)
3281                 return ret;
3282         if (!attr->transfer && attr->ingress &&
3283             (action_flags & terminal_action_flags))
3284                 return rte_flow_error_set(error, EINVAL,
3285                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3286                                           "set_tag has no effect"
3287                                           " with terminal actions");
3288         return 0;
3289 }
3290
3291 /**
3292  * Validate count action.
3293  *
3294  * @param[in] dev
3295  *   Pointer to rte_eth_dev structure.
3296  * @param[in] shared
3297  *   Indicator if action is shared.
3298  * @param[in] action_flags
3299  *   Holds the actions detected until now.
3300  * @param[out] error
3301  *   Pointer to error structure.
3302  *
3303  * @return
3304  *   0 on success, a negative errno value otherwise and rte_errno is set.
3305  */
3306 static int
3307 flow_dv_validate_action_count(struct rte_eth_dev *dev, bool shared,
3308                               uint64_t action_flags,
3309                               struct rte_flow_error *error)
3310 {
3311         struct mlx5_priv *priv = dev->data->dev_private;
3312
3313         if (!priv->sh->devx)
3314                 goto notsup_err;
3315         if (action_flags & MLX5_FLOW_ACTION_COUNT)
3316                 return rte_flow_error_set(error, EINVAL,
3317                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3318                                           "duplicate count actions set");
3319         if (shared && (action_flags & MLX5_FLOW_ACTION_AGE) &&
3320             !priv->sh->flow_hit_aso_en)
3321                 return rte_flow_error_set(error, EINVAL,
3322                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3323                                           "old age and shared count combination is not supported");
3324 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
3325         return 0;
3326 #endif
3327 notsup_err:
3328         return rte_flow_error_set
3329                       (error, ENOTSUP,
3330                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3331                        NULL,
3332                        "count action not supported");
3333 }
3334
3335 /**
3336  * Validate the L2 encap action.
3337  *
3338  * @param[in] dev
3339  *   Pointer to the rte_eth_dev structure.
3340  * @param[in] action_flags
3341  *   Holds the actions detected until now.
3342  * @param[in] action
3343  *   Pointer to the action structure.
3344  * @param[in] attr
3345  *   Pointer to flow attributes.
3346  * @param[out] error
3347  *   Pointer to error structure.
3348  *
3349  * @return
3350  *   0 on success, a negative errno value otherwise and rte_errno is set.
3351  */
3352 static int
3353 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
3354                                  uint64_t action_flags,
3355                                  const struct rte_flow_action *action,
3356                                  const struct rte_flow_attr *attr,
3357                                  struct rte_flow_error *error)
3358 {
3359         const struct mlx5_priv *priv = dev->data->dev_private;
3360
3361         if (!(action->conf))
3362                 return rte_flow_error_set(error, EINVAL,
3363                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3364                                           "configuration cannot be null");
3365         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3366                 return rte_flow_error_set(error, EINVAL,
3367                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3368                                           "can only have a single encap action "
3369                                           "in a flow");
3370         if (!attr->transfer && priv->representor)
3371                 return rte_flow_error_set(error, ENOTSUP,
3372                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3373                                           "encap action for VF representor "
3374                                           "not supported on NIC table");
3375         return 0;
3376 }
3377
3378 /**
3379  * Validate a decap action.
3380  *
3381  * @param[in] dev
3382  *   Pointer to the rte_eth_dev structure.
3383  * @param[in] action_flags
3384  *   Holds the actions detected until now.
3385  * @param[in] action
3386  *   Pointer to the action structure.
3387  * @param[in] item_flags
3388  *   Holds the items detected.
3389  * @param[in] attr
3390  *   Pointer to flow attributes
3391  * @param[out] error
3392  *   Pointer to error structure.
3393  *
3394  * @return
3395  *   0 on success, a negative errno value otherwise and rte_errno is set.
3396  */
3397 static int
3398 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
3399                               uint64_t action_flags,
3400                               const struct rte_flow_action *action,
3401                               const uint64_t item_flags,
3402                               const struct rte_flow_attr *attr,
3403                               struct rte_flow_error *error)
3404 {
3405         const struct mlx5_priv *priv = dev->data->dev_private;
3406
3407         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
3408             !priv->config.decap_en)
3409                 return rte_flow_error_set(error, ENOTSUP,
3410                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3411                                           "decap is not enabled");
3412         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
3413                 return rte_flow_error_set(error, ENOTSUP,
3414                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3415                                           action_flags &
3416                                           MLX5_FLOW_ACTION_DECAP ? "can only "
3417                                           "have a single decap action" : "decap "
3418                                           "after encap is not supported");
3419         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
3420                 return rte_flow_error_set(error, EINVAL,
3421                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3422                                           "can't have decap action after"
3423                                           " modify action");
3424         if (attr->egress)
3425                 return rte_flow_error_set(error, ENOTSUP,
3426                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
3427                                           NULL,
3428                                           "decap action not supported for "
3429                                           "egress");
3430         if (!attr->transfer && priv->representor)
3431                 return rte_flow_error_set(error, ENOTSUP,
3432                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3433                                           "decap action for VF representor "
3434                                           "not supported on NIC table");
3435         if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_DECAP &&
3436             !(item_flags & MLX5_FLOW_LAYER_VXLAN))
3437                 return rte_flow_error_set(error, ENOTSUP,
3438                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3439                                 "VXLAN item should be present for VXLAN decap");
3440         return 0;
3441 }
3442
3443 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
3444
3445 /**
3446  * Validate the raw encap and decap actions.
3447  *
3448  * @param[in] dev
3449  *   Pointer to the rte_eth_dev structure.
3450  * @param[in] decap
3451  *   Pointer to the decap action.
3452  * @param[in] encap
3453  *   Pointer to the encap action.
3454  * @param[in] attr
3455  *   Pointer to flow attributes
3456  * @param[in/out] action_flags
3457  *   Holds the actions detected until now.
3458  * @param[out] actions_n
3459  *   pointer to the number of actions counter.
3460  * @param[in] action
3461  *   Pointer to the action structure.
3462  * @param[in] item_flags
3463  *   Holds the items detected.
3464  * @param[out] error
3465  *   Pointer to error structure.
3466  *
3467  * @return
3468  *   0 on success, a negative errno value otherwise and rte_errno is set.
3469  */
3470 static int
3471 flow_dv_validate_action_raw_encap_decap
3472         (struct rte_eth_dev *dev,
3473          const struct rte_flow_action_raw_decap *decap,
3474          const struct rte_flow_action_raw_encap *encap,
3475          const struct rte_flow_attr *attr, uint64_t *action_flags,
3476          int *actions_n, const struct rte_flow_action *action,
3477          uint64_t item_flags, struct rte_flow_error *error)
3478 {
3479         const struct mlx5_priv *priv = dev->data->dev_private;
3480         int ret;
3481
3482         if (encap && (!encap->size || !encap->data))
3483                 return rte_flow_error_set(error, EINVAL,
3484                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3485                                           "raw encap data cannot be empty");
3486         if (decap && encap) {
3487                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
3488                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3489                         /* L3 encap. */
3490                         decap = NULL;
3491                 else if (encap->size <=
3492                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3493                            decap->size >
3494                            MLX5_ENCAPSULATION_DECISION_SIZE)
3495                         /* L3 decap. */
3496                         encap = NULL;
3497                 else if (encap->size >
3498                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3499                            decap->size >
3500                            MLX5_ENCAPSULATION_DECISION_SIZE)
3501                         /* 2 L2 actions: encap and decap. */
3502                         ;
3503                 else
3504                         return rte_flow_error_set(error,
3505                                 ENOTSUP,
3506                                 RTE_FLOW_ERROR_TYPE_ACTION,
3507                                 NULL, "unsupported too small "
3508                                 "raw decap and too small raw "
3509                                 "encap combination");
3510         }
3511         if (decap) {
3512                 ret = flow_dv_validate_action_decap(dev, *action_flags, action,
3513                                                     item_flags, attr, error);
3514                 if (ret < 0)
3515                         return ret;
3516                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
3517                 ++(*actions_n);
3518         }
3519         if (encap) {
3520                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
3521                         return rte_flow_error_set(error, ENOTSUP,
3522                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3523                                                   NULL,
3524                                                   "small raw encap size");
3525                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
3526                         return rte_flow_error_set(error, EINVAL,
3527                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3528                                                   NULL,
3529                                                   "more than one encap action");
3530                 if (!attr->transfer && priv->representor)
3531                         return rte_flow_error_set
3532                                         (error, ENOTSUP,
3533                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3534                                          "encap action for VF representor "
3535                                          "not supported on NIC table");
3536                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
3537                 ++(*actions_n);
3538         }
3539         return 0;
3540 }
3541
3542 /*
3543  * Validate the ASO CT action.
3544  *
3545  * @param[in] dev
3546  *   Pointer to the rte_eth_dev structure.
3547  * @param[in] action_flags
3548  *   Holds the actions detected until now.
3549  * @param[in] item_flags
3550  *   The items found in this flow rule.
3551  * @param[in] attr
3552  *   Pointer to flow attributes.
3553  * @param[out] error
3554  *   Pointer to error structure.
3555  *
3556  * @return
3557  *   0 on success, a negative errno value otherwise and rte_errno is set.
3558  */
3559 static int
3560 flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev,
3561                                uint64_t action_flags,
3562                                uint64_t item_flags,
3563                                const struct rte_flow_attr *attr,
3564                                struct rte_flow_error *error)
3565 {
3566         RTE_SET_USED(dev);
3567
3568         if (attr->group == 0 && !attr->transfer)
3569                 return rte_flow_error_set(error, ENOTSUP,
3570                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3571                                           NULL,
3572                                           "Only support non-root table");
3573         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
3574                 return rte_flow_error_set(error, ENOTSUP,
3575                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3576                                           "CT cannot follow a fate action");
3577         if ((action_flags & MLX5_FLOW_ACTION_METER) ||
3578             (action_flags & MLX5_FLOW_ACTION_AGE))
3579                 return rte_flow_error_set(error, EINVAL,
3580                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3581                                           "Only one ASO action is supported");
3582         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3583                 return rte_flow_error_set(error, EINVAL,
3584                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3585                                           "Encap cannot exist before CT");
3586         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
3587                 return rte_flow_error_set(error, EINVAL,
3588                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3589                                           "Not a outer TCP packet");
3590         return 0;
3591 }
3592
3593 int
3594 flow_dv_encap_decap_match_cb(void *tool_ctx __rte_unused,
3595                              struct mlx5_list_entry *entry, void *cb_ctx)
3596 {
3597         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3598         struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
3599         struct mlx5_flow_dv_encap_decap_resource *resource;
3600
3601         resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource,
3602                                 entry);
3603         if (resource->reformat_type == ctx_resource->reformat_type &&
3604             resource->ft_type == ctx_resource->ft_type &&
3605             resource->flags == ctx_resource->flags &&
3606             resource->size == ctx_resource->size &&
3607             !memcmp((const void *)resource->buf,
3608                     (const void *)ctx_resource->buf,
3609                     resource->size))
3610                 return 0;
3611         return -1;
3612 }
3613
3614 struct mlx5_list_entry *
3615 flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
3616 {
3617         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3618         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3619         struct mlx5dv_dr_domain *domain;
3620         struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
3621         struct mlx5_flow_dv_encap_decap_resource *resource;
3622         uint32_t idx;
3623         int ret;
3624
3625         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3626                 domain = sh->fdb_domain;
3627         else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3628                 domain = sh->rx_domain;
3629         else
3630                 domain = sh->tx_domain;
3631         /* Register new encap/decap resource. */
3632         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &idx);
3633         if (!resource) {
3634                 rte_flow_error_set(ctx->error, ENOMEM,
3635                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3636                                    "cannot allocate resource memory");
3637                 return NULL;
3638         }
3639         *resource = *ctx_resource;
3640         resource->idx = idx;
3641         ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->cdev->ctx,
3642                                                               domain, resource,
3643                                                              &resource->action);
3644         if (ret) {
3645                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
3646                 rte_flow_error_set(ctx->error, ENOMEM,
3647                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3648                                    NULL, "cannot create action");
3649                 return NULL;
3650         }
3651
3652         return &resource->entry;
3653 }
3654
3655 struct mlx5_list_entry *
3656 flow_dv_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
3657                              void *cb_ctx)
3658 {
3659         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3660         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3661         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
3662         uint32_t idx;
3663
3664         cache_resource = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
3665                                            &idx);
3666         if (!cache_resource) {
3667                 rte_flow_error_set(ctx->error, ENOMEM,
3668                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3669                                    "cannot allocate resource memory");
3670                 return NULL;
3671         }
3672         memcpy(cache_resource, oentry, sizeof(*cache_resource));
3673         cache_resource->idx = idx;
3674         return &cache_resource->entry;
3675 }
3676
3677 void
3678 flow_dv_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3679 {
3680         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3681         struct mlx5_flow_dv_encap_decap_resource *res =
3682                                        container_of(entry, typeof(*res), entry);
3683
3684         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
3685 }
3686
3687 /**
3688  * Find existing encap/decap resource or create and register a new one.
3689  *
3690  * @param[in, out] dev
3691  *   Pointer to rte_eth_dev structure.
3692  * @param[in, out] resource
3693  *   Pointer to encap/decap resource.
3694  * @parm[in, out] dev_flow
3695  *   Pointer to the dev_flow.
3696  * @param[out] error
3697  *   pointer to error structure.
3698  *
3699  * @return
3700  *   0 on success otherwise -errno and errno is set.
3701  */
3702 static int
3703 flow_dv_encap_decap_resource_register
3704                         (struct rte_eth_dev *dev,
3705                          struct mlx5_flow_dv_encap_decap_resource *resource,
3706                          struct mlx5_flow *dev_flow,
3707                          struct rte_flow_error *error)
3708 {
3709         struct mlx5_priv *priv = dev->data->dev_private;
3710         struct mlx5_dev_ctx_shared *sh = priv->sh;
3711         struct mlx5_list_entry *entry;
3712         union {
3713                 struct {
3714                         uint32_t ft_type:8;
3715                         uint32_t refmt_type:8;
3716                         /*
3717                          * Header reformat actions can be shared between
3718                          * non-root tables. One bit to indicate non-root
3719                          * table or not.
3720                          */
3721                         uint32_t is_root:1;
3722                         uint32_t reserve:15;
3723                 };
3724                 uint32_t v32;
3725         } encap_decap_key = {
3726                 {
3727                         .ft_type = resource->ft_type,
3728                         .refmt_type = resource->reformat_type,
3729                         .is_root = !!dev_flow->dv.group,
3730                         .reserve = 0,
3731                 }
3732         };
3733         struct mlx5_flow_cb_ctx ctx = {
3734                 .error = error,
3735                 .data = resource,
3736         };
3737         struct mlx5_hlist *encaps_decaps;
3738         uint64_t key64;
3739
3740         encaps_decaps = flow_dv_hlist_prepare(sh, &sh->encaps_decaps,
3741                                 "encaps_decaps",
3742                                 MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
3743                                 true, true, sh,
3744                                 flow_dv_encap_decap_create_cb,
3745                                 flow_dv_encap_decap_match_cb,
3746                                 flow_dv_encap_decap_remove_cb,
3747                                 flow_dv_encap_decap_clone_cb,
3748                                 flow_dv_encap_decap_clone_free_cb);
3749         if (unlikely(!encaps_decaps))
3750                 return -rte_errno;
3751         resource->flags = dev_flow->dv.group ? 0 : 1;
3752         key64 =  __rte_raw_cksum(&encap_decap_key.v32,
3753                                  sizeof(encap_decap_key.v32), 0);
3754         if (resource->reformat_type !=
3755             MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 &&
3756             resource->size)
3757                 key64 = __rte_raw_cksum(resource->buf, resource->size, key64);
3758         entry = mlx5_hlist_register(encaps_decaps, key64, &ctx);
3759         if (!entry)
3760                 return -rte_errno;
3761         resource = container_of(entry, typeof(*resource), entry);
3762         dev_flow->dv.encap_decap = resource;
3763         dev_flow->handle->dvh.rix_encap_decap = resource->idx;
3764         return 0;
3765 }
3766
3767 /**
3768  * Find existing table jump resource or create and register a new one.
3769  *
3770  * @param[in, out] dev
3771  *   Pointer to rte_eth_dev structure.
3772  * @param[in, out] tbl
3773  *   Pointer to flow table resource.
3774  * @parm[in, out] dev_flow
3775  *   Pointer to the dev_flow.
3776  * @param[out] error
3777  *   pointer to error structure.
3778  *
3779  * @return
3780  *   0 on success otherwise -errno and errno is set.
3781  */
3782 static int
3783 flow_dv_jump_tbl_resource_register
3784                         (struct rte_eth_dev *dev __rte_unused,
3785                          struct mlx5_flow_tbl_resource *tbl,
3786                          struct mlx5_flow *dev_flow,
3787                          struct rte_flow_error *error __rte_unused)
3788 {
3789         struct mlx5_flow_tbl_data_entry *tbl_data =
3790                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
3791
3792         MLX5_ASSERT(tbl);
3793         MLX5_ASSERT(tbl_data->jump.action);
3794         dev_flow->handle->rix_jump = tbl_data->idx;
3795         dev_flow->dv.jump = &tbl_data->jump;
3796         return 0;
3797 }
3798
3799 int
3800 flow_dv_port_id_match_cb(void *tool_ctx __rte_unused,
3801                          struct mlx5_list_entry *entry, void *cb_ctx)
3802 {
3803         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3804         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3805         struct mlx5_flow_dv_port_id_action_resource *res =
3806                                        container_of(entry, typeof(*res), entry);
3807
3808         return ref->port_id != res->port_id;
3809 }
3810
3811 struct mlx5_list_entry *
3812 flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx)
3813 {
3814         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3815         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3816         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3817         struct mlx5_flow_dv_port_id_action_resource *resource;
3818         uint32_t idx;
3819         int ret;
3820
3821         /* Register new port id action resource. */
3822         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
3823         if (!resource) {
3824                 rte_flow_error_set(ctx->error, ENOMEM,
3825                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3826                                    "cannot allocate port_id action memory");
3827                 return NULL;
3828         }
3829         *resource = *ref;
3830         ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain,
3831                                                         ref->port_id,
3832                                                         &resource->action);
3833         if (ret) {
3834                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx);
3835                 rte_flow_error_set(ctx->error, ENOMEM,
3836                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3837                                    "cannot create action");
3838                 return NULL;
3839         }
3840         resource->idx = idx;
3841         return &resource->entry;
3842 }
3843
3844 struct mlx5_list_entry *
3845 flow_dv_port_id_clone_cb(void *tool_ctx,
3846                          struct mlx5_list_entry *entry __rte_unused,
3847                          void *cb_ctx)
3848 {
3849         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3850         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3851         struct mlx5_flow_dv_port_id_action_resource *resource;
3852         uint32_t idx;
3853
3854         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
3855         if (!resource) {
3856                 rte_flow_error_set(ctx->error, ENOMEM,
3857                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3858                                    "cannot allocate port_id action memory");
3859                 return NULL;
3860         }
3861         memcpy(resource, entry, sizeof(*resource));
3862         resource->idx = idx;
3863         return &resource->entry;
3864 }
3865
3866 void
3867 flow_dv_port_id_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3868 {
3869         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3870         struct mlx5_flow_dv_port_id_action_resource *resource =
3871                                   container_of(entry, typeof(*resource), entry);
3872
3873         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
3874 }
3875
3876 /**
3877  * Find existing table port ID resource or create and register a new one.
3878  *
3879  * @param[in, out] dev
3880  *   Pointer to rte_eth_dev structure.
3881  * @param[in, out] ref
3882  *   Pointer to port ID action resource reference.
3883  * @parm[in, out] dev_flow
3884  *   Pointer to the dev_flow.
3885  * @param[out] error
3886  *   pointer to error structure.
3887  *
3888  * @return
3889  *   0 on success otherwise -errno and errno is set.
3890  */
3891 static int
3892 flow_dv_port_id_action_resource_register
3893                         (struct rte_eth_dev *dev,
3894                          struct mlx5_flow_dv_port_id_action_resource *ref,
3895                          struct mlx5_flow *dev_flow,
3896                          struct rte_flow_error *error)
3897 {
3898         struct mlx5_priv *priv = dev->data->dev_private;
3899         struct mlx5_list_entry *entry;
3900         struct mlx5_flow_dv_port_id_action_resource *resource;
3901         struct mlx5_flow_cb_ctx ctx = {
3902                 .error = error,
3903                 .data = ref,
3904         };
3905
3906         entry = mlx5_list_register(priv->sh->port_id_action_list, &ctx);
3907         if (!entry)
3908                 return -rte_errno;
3909         resource = container_of(entry, typeof(*resource), entry);
3910         dev_flow->dv.port_id_action = resource;
3911         dev_flow->handle->rix_port_id_action = resource->idx;
3912         return 0;
3913 }
3914
3915 int
3916 flow_dv_push_vlan_match_cb(void *tool_ctx __rte_unused,
3917                            struct mlx5_list_entry *entry, void *cb_ctx)
3918 {
3919         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3920         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3921         struct mlx5_flow_dv_push_vlan_action_resource *res =
3922                                        container_of(entry, typeof(*res), entry);
3923
3924         return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
3925 }
3926
3927 struct mlx5_list_entry *
3928 flow_dv_push_vlan_create_cb(void *tool_ctx, void *cb_ctx)
3929 {
3930         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3931         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3932         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3933         struct mlx5_flow_dv_push_vlan_action_resource *resource;
3934         struct mlx5dv_dr_domain *domain;
3935         uint32_t idx;
3936         int ret;
3937
3938         /* Register new port id action resource. */
3939         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
3940         if (!resource) {
3941                 rte_flow_error_set(ctx->error, ENOMEM,
3942                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3943                                    "cannot allocate push_vlan action memory");
3944                 return NULL;
3945         }
3946         *resource = *ref;
3947         if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3948                 domain = sh->fdb_domain;
3949         else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3950                 domain = sh->rx_domain;
3951         else
3952                 domain = sh->tx_domain;
3953         ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag,
3954                                                         &resource->action);
3955         if (ret) {
3956                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
3957                 rte_flow_error_set(ctx->error, ENOMEM,
3958                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3959                                    "cannot create push vlan action");
3960                 return NULL;
3961         }
3962         resource->idx = idx;
3963         return &resource->entry;
3964 }
3965
3966 struct mlx5_list_entry *
3967 flow_dv_push_vlan_clone_cb(void *tool_ctx,
3968                            struct mlx5_list_entry *entry __rte_unused,
3969                            void *cb_ctx)
3970 {
3971         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3972         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3973         struct mlx5_flow_dv_push_vlan_action_resource *resource;
3974         uint32_t idx;
3975
3976         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
3977         if (!resource) {
3978                 rte_flow_error_set(ctx->error, ENOMEM,
3979                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3980                                    "cannot allocate push_vlan action memory");
3981                 return NULL;
3982         }
3983         memcpy(resource, entry, sizeof(*resource));
3984         resource->idx = idx;
3985         return &resource->entry;
3986 }
3987
3988 void
3989 flow_dv_push_vlan_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3990 {
3991         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3992         struct mlx5_flow_dv_push_vlan_action_resource *resource =
3993                                   container_of(entry, typeof(*resource), entry);
3994
3995         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
3996 }
3997
3998 /**
3999  * Find existing push vlan resource or create and register a new one.
4000  *
4001  * @param [in, out] dev
4002  *   Pointer to rte_eth_dev structure.
4003  * @param[in, out] ref
4004  *   Pointer to port ID action resource reference.
4005  * @parm[in, out] dev_flow
4006  *   Pointer to the dev_flow.
4007  * @param[out] error
4008  *   pointer to error structure.
4009  *
4010  * @return
4011  *   0 on success otherwise -errno and errno is set.
4012  */
4013 static int
4014 flow_dv_push_vlan_action_resource_register
4015                        (struct rte_eth_dev *dev,
4016                         struct mlx5_flow_dv_push_vlan_action_resource *ref,
4017                         struct mlx5_flow *dev_flow,
4018                         struct rte_flow_error *error)
4019 {
4020         struct mlx5_priv *priv = dev->data->dev_private;
4021         struct mlx5_flow_dv_push_vlan_action_resource *resource;
4022         struct mlx5_list_entry *entry;
4023         struct mlx5_flow_cb_ctx ctx = {
4024                 .error = error,
4025                 .data = ref,
4026         };
4027
4028         entry = mlx5_list_register(priv->sh->push_vlan_action_list, &ctx);
4029         if (!entry)
4030                 return -rte_errno;
4031         resource = container_of(entry, typeof(*resource), entry);
4032
4033         dev_flow->handle->dvh.rix_push_vlan = resource->idx;
4034         dev_flow->dv.push_vlan_res = resource;
4035         return 0;
4036 }
4037
4038 /**
4039  * Get the size of specific rte_flow_item_type hdr size
4040  *
4041  * @param[in] item_type
4042  *   Tested rte_flow_item_type.
4043  *
4044  * @return
4045  *   sizeof struct item_type, 0 if void or irrelevant.
4046  */
4047 static size_t
4048 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
4049 {
4050         size_t retval;
4051
4052         switch (item_type) {
4053         case RTE_FLOW_ITEM_TYPE_ETH:
4054                 retval = sizeof(struct rte_ether_hdr);
4055                 break;
4056         case RTE_FLOW_ITEM_TYPE_VLAN:
4057                 retval = sizeof(struct rte_vlan_hdr);
4058                 break;
4059         case RTE_FLOW_ITEM_TYPE_IPV4:
4060                 retval = sizeof(struct rte_ipv4_hdr);
4061                 break;
4062         case RTE_FLOW_ITEM_TYPE_IPV6:
4063                 retval = sizeof(struct rte_ipv6_hdr);
4064                 break;
4065         case RTE_FLOW_ITEM_TYPE_UDP:
4066                 retval = sizeof(struct rte_udp_hdr);
4067                 break;
4068         case RTE_FLOW_ITEM_TYPE_TCP:
4069                 retval = sizeof(struct rte_tcp_hdr);
4070                 break;
4071         case RTE_FLOW_ITEM_TYPE_VXLAN:
4072         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4073                 retval = sizeof(struct rte_vxlan_hdr);
4074                 break;
4075         case RTE_FLOW_ITEM_TYPE_GRE:
4076         case RTE_FLOW_ITEM_TYPE_NVGRE:
4077                 retval = sizeof(struct rte_gre_hdr);
4078                 break;
4079         case RTE_FLOW_ITEM_TYPE_MPLS:
4080                 retval = sizeof(struct rte_mpls_hdr);
4081                 break;
4082         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
4083         default:
4084                 retval = 0;
4085                 break;
4086         }
4087         return retval;
4088 }
4089
4090 #define MLX5_ENCAP_IPV4_VERSION         0x40
4091 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
4092 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
4093 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
4094 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
4095 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
4096 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
4097
4098 /**
4099  * Convert the encap action data from list of rte_flow_item to raw buffer
4100  *
4101  * @param[in] items
4102  *   Pointer to rte_flow_item objects list.
4103  * @param[out] buf
4104  *   Pointer to the output buffer.
4105  * @param[out] size
4106  *   Pointer to the output buffer size.
4107  * @param[out] error
4108  *   Pointer to the error structure.
4109  *
4110  * @return
4111  *   0 on success, a negative errno value otherwise and rte_errno is set.
4112  */
4113 static int
4114 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
4115                            size_t *size, struct rte_flow_error *error)
4116 {
4117         struct rte_ether_hdr *eth = NULL;
4118         struct rte_vlan_hdr *vlan = NULL;
4119         struct rte_ipv4_hdr *ipv4 = NULL;
4120         struct rte_ipv6_hdr *ipv6 = NULL;
4121         struct rte_udp_hdr *udp = NULL;
4122         struct rte_vxlan_hdr *vxlan = NULL;
4123         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
4124         struct rte_gre_hdr *gre = NULL;
4125         size_t len;
4126         size_t temp_size = 0;
4127
4128         if (!items)
4129                 return rte_flow_error_set(error, EINVAL,
4130                                           RTE_FLOW_ERROR_TYPE_ACTION,
4131                                           NULL, "invalid empty data");
4132         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4133                 len = flow_dv_get_item_hdr_len(items->type);
4134                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
4135                         return rte_flow_error_set(error, EINVAL,
4136                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4137                                                   (void *)items->type,
4138                                                   "items total size is too big"
4139                                                   " for encap action");
4140                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
4141                 switch (items->type) {
4142                 case RTE_FLOW_ITEM_TYPE_ETH:
4143                         eth = (struct rte_ether_hdr *)&buf[temp_size];
4144                         break;
4145                 case RTE_FLOW_ITEM_TYPE_VLAN:
4146                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
4147                         if (!eth)
4148                                 return rte_flow_error_set(error, EINVAL,
4149                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4150                                                 (void *)items->type,
4151                                                 "eth header not found");
4152                         if (!eth->ether_type)
4153                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
4154                         break;
4155                 case RTE_FLOW_ITEM_TYPE_IPV4:
4156                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
4157                         if (!vlan && !eth)
4158                                 return rte_flow_error_set(error, EINVAL,
4159                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4160                                                 (void *)items->type,
4161                                                 "neither eth nor vlan"
4162                                                 " header found");
4163                         if (vlan && !vlan->eth_proto)
4164                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4165                         else if (eth && !eth->ether_type)
4166                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4167                         if (!ipv4->version_ihl)
4168                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
4169                                                     MLX5_ENCAP_IPV4_IHL_MIN;
4170                         if (!ipv4->time_to_live)
4171                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
4172                         break;
4173                 case RTE_FLOW_ITEM_TYPE_IPV6:
4174                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
4175                         if (!vlan && !eth)
4176                                 return rte_flow_error_set(error, EINVAL,
4177                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4178                                                 (void *)items->type,
4179                                                 "neither eth nor vlan"
4180                                                 " header found");
4181                         if (vlan && !vlan->eth_proto)
4182                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4183                         else if (eth && !eth->ether_type)
4184                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4185                         if (!ipv6->vtc_flow)
4186                                 ipv6->vtc_flow =
4187                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
4188                         if (!ipv6->hop_limits)
4189                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
4190                         break;
4191                 case RTE_FLOW_ITEM_TYPE_UDP:
4192                         udp = (struct rte_udp_hdr *)&buf[temp_size];
4193                         if (!ipv4 && !ipv6)
4194                                 return rte_flow_error_set(error, EINVAL,
4195                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4196                                                 (void *)items->type,
4197                                                 "ip header not found");
4198                         if (ipv4 && !ipv4->next_proto_id)
4199                                 ipv4->next_proto_id = IPPROTO_UDP;
4200                         else if (ipv6 && !ipv6->proto)
4201                                 ipv6->proto = IPPROTO_UDP;
4202                         break;
4203                 case RTE_FLOW_ITEM_TYPE_VXLAN:
4204                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
4205                         if (!udp)
4206                                 return rte_flow_error_set(error, EINVAL,
4207                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4208                                                 (void *)items->type,
4209                                                 "udp header not found");
4210                         if (!udp->dst_port)
4211                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
4212                         if (!vxlan->vx_flags)
4213                                 vxlan->vx_flags =
4214                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
4215                         break;
4216                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4217                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
4218                         if (!udp)
4219                                 return rte_flow_error_set(error, EINVAL,
4220                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4221                                                 (void *)items->type,
4222                                                 "udp header not found");
4223                         if (!vxlan_gpe->proto)
4224                                 return rte_flow_error_set(error, EINVAL,
4225                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4226                                                 (void *)items->type,
4227                                                 "next protocol not found");
4228                         if (!udp->dst_port)
4229                                 udp->dst_port =
4230                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
4231                         if (!vxlan_gpe->vx_flags)
4232                                 vxlan_gpe->vx_flags =
4233                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
4234                         break;
4235                 case RTE_FLOW_ITEM_TYPE_GRE:
4236                 case RTE_FLOW_ITEM_TYPE_NVGRE:
4237                         gre = (struct rte_gre_hdr *)&buf[temp_size];
4238                         if (!gre->proto)
4239                                 return rte_flow_error_set(error, EINVAL,
4240                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4241                                                 (void *)items->type,
4242                                                 "next protocol not found");
4243                         if (!ipv4 && !ipv6)
4244                                 return rte_flow_error_set(error, EINVAL,
4245                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4246                                                 (void *)items->type,
4247                                                 "ip header not found");
4248                         if (ipv4 && !ipv4->next_proto_id)
4249                                 ipv4->next_proto_id = IPPROTO_GRE;
4250                         else if (ipv6 && !ipv6->proto)
4251                                 ipv6->proto = IPPROTO_GRE;
4252                         break;
4253                 case RTE_FLOW_ITEM_TYPE_VOID:
4254                         break;
4255                 default:
4256                         return rte_flow_error_set(error, EINVAL,
4257                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4258                                                   (void *)items->type,
4259                                                   "unsupported item type");
4260                         break;
4261                 }
4262                 temp_size += len;
4263         }
4264         *size = temp_size;
4265         return 0;
4266 }
4267
4268 static int
4269 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
4270 {
4271         struct rte_ether_hdr *eth = NULL;
4272         struct rte_vlan_hdr *vlan = NULL;
4273         struct rte_ipv6_hdr *ipv6 = NULL;
4274         struct rte_udp_hdr *udp = NULL;
4275         char *next_hdr;
4276         uint16_t proto;
4277
4278         eth = (struct rte_ether_hdr *)data;
4279         next_hdr = (char *)(eth + 1);
4280         proto = RTE_BE16(eth->ether_type);
4281
4282         /* VLAN skipping */
4283         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
4284                 vlan = (struct rte_vlan_hdr *)next_hdr;
4285                 proto = RTE_BE16(vlan->eth_proto);
4286                 next_hdr += sizeof(struct rte_vlan_hdr);
4287         }
4288
4289         /* HW calculates IPv4 csum. no need to proceed */
4290         if (proto == RTE_ETHER_TYPE_IPV4)
4291                 return 0;
4292
4293         /* non IPv4/IPv6 header. not supported */
4294         if (proto != RTE_ETHER_TYPE_IPV6) {
4295                 return rte_flow_error_set(error, ENOTSUP,
4296                                           RTE_FLOW_ERROR_TYPE_ACTION,
4297                                           NULL, "Cannot offload non IPv4/IPv6");
4298         }
4299
4300         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
4301
4302         /* ignore non UDP */
4303         if (ipv6->proto != IPPROTO_UDP)
4304                 return 0;
4305
4306         udp = (struct rte_udp_hdr *)(ipv6 + 1);
4307         udp->dgram_cksum = 0;
4308
4309         return 0;
4310 }
4311
4312 /**
4313  * Convert L2 encap action to DV specification.
4314  *
4315  * @param[in] dev
4316  *   Pointer to rte_eth_dev structure.
4317  * @param[in] action
4318  *   Pointer to action structure.
4319  * @param[in, out] dev_flow
4320  *   Pointer to the mlx5_flow.
4321  * @param[in] transfer
4322  *   Mark if the flow is E-Switch flow.
4323  * @param[out] error
4324  *   Pointer to the error structure.
4325  *
4326  * @return
4327  *   0 on success, a negative errno value otherwise and rte_errno is set.
4328  */
4329 static int
4330 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
4331                                const struct rte_flow_action *action,
4332                                struct mlx5_flow *dev_flow,
4333                                uint8_t transfer,
4334                                struct rte_flow_error *error)
4335 {
4336         const struct rte_flow_item *encap_data;
4337         const struct rte_flow_action_raw_encap *raw_encap_data;
4338         struct mlx5_flow_dv_encap_decap_resource res = {
4339                 .reformat_type =
4340                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
4341                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4342                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
4343         };
4344
4345         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
4346                 raw_encap_data =
4347                         (const struct rte_flow_action_raw_encap *)action->conf;
4348                 res.size = raw_encap_data->size;
4349                 memcpy(res.buf, raw_encap_data->data, res.size);
4350         } else {
4351                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
4352                         encap_data =
4353                                 ((const struct rte_flow_action_vxlan_encap *)
4354                                                 action->conf)->definition;
4355                 else
4356                         encap_data =
4357                                 ((const struct rte_flow_action_nvgre_encap *)
4358                                                 action->conf)->definition;
4359                 if (flow_dv_convert_encap_data(encap_data, res.buf,
4360                                                &res.size, error))
4361                         return -rte_errno;
4362         }
4363         if (flow_dv_zero_encap_udp_csum(res.buf, error))
4364                 return -rte_errno;
4365         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4366                 return rte_flow_error_set(error, EINVAL,
4367                                           RTE_FLOW_ERROR_TYPE_ACTION,
4368                                           NULL, "can't create L2 encap action");
4369         return 0;
4370 }
4371
4372 /**
4373  * Convert L2 decap action to DV specification.
4374  *
4375  * @param[in] dev
4376  *   Pointer to rte_eth_dev structure.
4377  * @param[in, out] dev_flow
4378  *   Pointer to the mlx5_flow.
4379  * @param[in] transfer
4380  *   Mark if the flow is E-Switch flow.
4381  * @param[out] error
4382  *   Pointer to the error structure.
4383  *
4384  * @return
4385  *   0 on success, a negative errno value otherwise and rte_errno is set.
4386  */
4387 static int
4388 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
4389                                struct mlx5_flow *dev_flow,
4390                                uint8_t transfer,
4391                                struct rte_flow_error *error)
4392 {
4393         struct mlx5_flow_dv_encap_decap_resource res = {
4394                 .size = 0,
4395                 .reformat_type =
4396                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
4397                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4398                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
4399         };
4400
4401         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4402                 return rte_flow_error_set(error, EINVAL,
4403                                           RTE_FLOW_ERROR_TYPE_ACTION,
4404                                           NULL, "can't create L2 decap action");
4405         return 0;
4406 }
4407
4408 /**
4409  * Convert raw decap/encap (L3 tunnel) action to DV specification.
4410  *
4411  * @param[in] dev
4412  *   Pointer to rte_eth_dev structure.
4413  * @param[in] action
4414  *   Pointer to action structure.
4415  * @param[in, out] dev_flow
4416  *   Pointer to the mlx5_flow.
4417  * @param[in] attr
4418  *   Pointer to the flow attributes.
4419  * @param[out] error
4420  *   Pointer to the error structure.
4421  *
4422  * @return
4423  *   0 on success, a negative errno value otherwise and rte_errno is set.
4424  */
4425 static int
4426 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
4427                                 const struct rte_flow_action *action,
4428                                 struct mlx5_flow *dev_flow,
4429                                 const struct rte_flow_attr *attr,
4430                                 struct rte_flow_error *error)
4431 {
4432         const struct rte_flow_action_raw_encap *encap_data;
4433         struct mlx5_flow_dv_encap_decap_resource res;
4434
4435         memset(&res, 0, sizeof(res));
4436         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
4437         res.size = encap_data->size;
4438         memcpy(res.buf, encap_data->data, res.size);
4439         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
4440                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
4441                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
4442         if (attr->transfer)
4443                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4444         else
4445                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4446                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4447         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4448                 return rte_flow_error_set(error, EINVAL,
4449                                           RTE_FLOW_ERROR_TYPE_ACTION,
4450                                           NULL, "can't create encap action");
4451         return 0;
4452 }
4453
4454 /**
4455  * Create action push VLAN.
4456  *
4457  * @param[in] dev
4458  *   Pointer to rte_eth_dev structure.
4459  * @param[in] attr
4460  *   Pointer to the flow attributes.
4461  * @param[in] vlan
4462  *   Pointer to the vlan to push to the Ethernet header.
4463  * @param[in, out] dev_flow
4464  *   Pointer to the mlx5_flow.
4465  * @param[out] error
4466  *   Pointer to the error structure.
4467  *
4468  * @return
4469  *   0 on success, a negative errno value otherwise and rte_errno is set.
4470  */
4471 static int
4472 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
4473                                 const struct rte_flow_attr *attr,
4474                                 const struct rte_vlan_hdr *vlan,
4475                                 struct mlx5_flow *dev_flow,
4476                                 struct rte_flow_error *error)
4477 {
4478         struct mlx5_flow_dv_push_vlan_action_resource res;
4479
4480         memset(&res, 0, sizeof(res));
4481         res.vlan_tag =
4482                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
4483                                  vlan->vlan_tci);
4484         if (attr->transfer)
4485                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4486         else
4487                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4488                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4489         return flow_dv_push_vlan_action_resource_register
4490                                             (dev, &res, dev_flow, error);
4491 }
4492
4493 /**
4494  * Validate the modify-header actions.
4495  *
4496  * @param[in] action_flags
4497  *   Holds the actions detected until now.
4498  * @param[in] action
4499  *   Pointer to the modify action.
4500  * @param[out] error
4501  *   Pointer to error structure.
4502  *
4503  * @return
4504  *   0 on success, a negative errno value otherwise and rte_errno is set.
4505  */
4506 static int
4507 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
4508                                    const struct rte_flow_action *action,
4509                                    struct rte_flow_error *error)
4510 {
4511         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
4512                 return rte_flow_error_set(error, EINVAL,
4513                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4514                                           NULL, "action configuration not set");
4515         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
4516                 return rte_flow_error_set(error, EINVAL,
4517                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4518                                           "can't have encap action before"
4519                                           " modify action");
4520         return 0;
4521 }
4522
4523 /**
4524  * Validate the modify-header MAC address actions.
4525  *
4526  * @param[in] action_flags
4527  *   Holds the actions detected until now.
4528  * @param[in] action
4529  *   Pointer to the modify action.
4530  * @param[in] item_flags
4531  *   Holds the items detected.
4532  * @param[out] error
4533  *   Pointer to error structure.
4534  *
4535  * @return
4536  *   0 on success, a negative errno value otherwise and rte_errno is set.
4537  */
4538 static int
4539 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
4540                                    const struct rte_flow_action *action,
4541                                    const uint64_t item_flags,
4542                                    struct rte_flow_error *error)
4543 {
4544         int ret = 0;
4545
4546         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4547         if (!ret) {
4548                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
4549                         return rte_flow_error_set(error, EINVAL,
4550                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4551                                                   NULL,
4552                                                   "no L2 item in pattern");
4553         }
4554         return ret;
4555 }
4556
4557 /**
4558  * Validate the modify-header IPv4 address actions.
4559  *
4560  * @param[in] action_flags
4561  *   Holds the actions detected until now.
4562  * @param[in] action
4563  *   Pointer to the modify action.
4564  * @param[in] item_flags
4565  *   Holds the items detected.
4566  * @param[out] error
4567  *   Pointer to error structure.
4568  *
4569  * @return
4570  *   0 on success, a negative errno value otherwise and rte_errno is set.
4571  */
4572 static int
4573 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
4574                                     const struct rte_flow_action *action,
4575                                     const uint64_t item_flags,
4576                                     struct rte_flow_error *error)
4577 {
4578         int ret = 0;
4579         uint64_t layer;
4580
4581         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4582         if (!ret) {
4583                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4584                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
4585                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
4586                 if (!(item_flags & layer))
4587                         return rte_flow_error_set(error, EINVAL,
4588                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4589                                                   NULL,
4590                                                   "no ipv4 item in pattern");
4591         }
4592         return ret;
4593 }
4594
4595 /**
4596  * Validate the modify-header IPv6 address actions.
4597  *
4598  * @param[in] action_flags
4599  *   Holds the actions detected until now.
4600  * @param[in] action
4601  *   Pointer to the modify action.
4602  * @param[in] item_flags
4603  *   Holds the items detected.
4604  * @param[out] error
4605  *   Pointer to error structure.
4606  *
4607  * @return
4608  *   0 on success, a negative errno value otherwise and rte_errno is set.
4609  */
4610 static int
4611 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
4612                                     const struct rte_flow_action *action,
4613                                     const uint64_t item_flags,
4614                                     struct rte_flow_error *error)
4615 {
4616         int ret = 0;
4617         uint64_t layer;
4618
4619         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4620         if (!ret) {
4621                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4622                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
4623                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
4624                 if (!(item_flags & layer))
4625                         return rte_flow_error_set(error, EINVAL,
4626                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4627                                                   NULL,
4628                                                   "no ipv6 item in pattern");
4629         }
4630         return ret;
4631 }
4632
4633 /**
4634  * Validate the modify-header TP actions.
4635  *
4636  * @param[in] action_flags
4637  *   Holds the actions detected until now.
4638  * @param[in] action
4639  *   Pointer to the modify action.
4640  * @param[in] item_flags
4641  *   Holds the items detected.
4642  * @param[out] error
4643  *   Pointer to error structure.
4644  *
4645  * @return
4646  *   0 on success, a negative errno value otherwise and rte_errno is set.
4647  */
4648 static int
4649 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
4650                                   const struct rte_flow_action *action,
4651                                   const uint64_t item_flags,
4652                                   struct rte_flow_error *error)
4653 {
4654         int ret = 0;
4655         uint64_t layer;
4656
4657         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4658         if (!ret) {
4659                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4660                                  MLX5_FLOW_LAYER_INNER_L4 :
4661                                  MLX5_FLOW_LAYER_OUTER_L4;
4662                 if (!(item_flags & layer))
4663                         return rte_flow_error_set(error, EINVAL,
4664                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4665                                                   NULL, "no transport layer "
4666                                                   "in pattern");
4667         }
4668         return ret;
4669 }
4670
4671 /**
4672  * Validate the modify-header actions of increment/decrement
4673  * TCP Sequence-number.
4674  *
4675  * @param[in] action_flags
4676  *   Holds the actions detected until now.
4677  * @param[in] action
4678  *   Pointer to the modify action.
4679  * @param[in] item_flags
4680  *   Holds the items detected.
4681  * @param[out] error
4682  *   Pointer to error structure.
4683  *
4684  * @return
4685  *   0 on success, a negative errno value otherwise and rte_errno is set.
4686  */
4687 static int
4688 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
4689                                        const struct rte_flow_action *action,
4690                                        const uint64_t item_flags,
4691                                        struct rte_flow_error *error)
4692 {
4693         int ret = 0;
4694         uint64_t layer;
4695
4696         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4697         if (!ret) {
4698                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4699                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4700                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4701                 if (!(item_flags & layer))
4702                         return rte_flow_error_set(error, EINVAL,
4703                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4704                                                   NULL, "no TCP item in"
4705                                                   " pattern");
4706                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
4707                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
4708                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
4709                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
4710                         return rte_flow_error_set(error, EINVAL,
4711                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4712                                                   NULL,
4713                                                   "cannot decrease and increase"
4714                                                   " TCP sequence number"
4715                                                   " at the same time");
4716         }
4717         return ret;
4718 }
4719
4720 /**
4721  * Validate the modify-header actions of increment/decrement
4722  * TCP Acknowledgment number.
4723  *
4724  * @param[in] action_flags
4725  *   Holds the actions detected until now.
4726  * @param[in] action
4727  *   Pointer to the modify action.
4728  * @param[in] item_flags
4729  *   Holds the items detected.
4730  * @param[out] error
4731  *   Pointer to error structure.
4732  *
4733  * @return
4734  *   0 on success, a negative errno value otherwise and rte_errno is set.
4735  */
4736 static int
4737 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
4738                                        const struct rte_flow_action *action,
4739                                        const uint64_t item_flags,
4740                                        struct rte_flow_error *error)
4741 {
4742         int ret = 0;
4743         uint64_t layer;
4744
4745         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4746         if (!ret) {
4747                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4748                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4749                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4750                 if (!(item_flags & layer))
4751                         return rte_flow_error_set(error, EINVAL,
4752                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4753                                                   NULL, "no TCP item in"
4754                                                   " pattern");
4755                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
4756                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
4757                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
4758                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
4759                         return rte_flow_error_set(error, EINVAL,
4760                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4761                                                   NULL,
4762                                                   "cannot decrease and increase"
4763                                                   " TCP acknowledgment number"
4764                                                   " at the same time");
4765         }
4766         return ret;
4767 }
4768
4769 /**
4770  * Validate the modify-header TTL actions.
4771  *
4772  * @param[in] action_flags
4773  *   Holds the actions detected until now.
4774  * @param[in] action
4775  *   Pointer to the modify action.
4776  * @param[in] item_flags
4777  *   Holds the items detected.
4778  * @param[out] error
4779  *   Pointer to error structure.
4780  *
4781  * @return
4782  *   0 on success, a negative errno value otherwise and rte_errno is set.
4783  */
4784 static int
4785 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
4786                                    const struct rte_flow_action *action,
4787                                    const uint64_t item_flags,
4788                                    struct rte_flow_error *error)
4789 {
4790         int ret = 0;
4791         uint64_t layer;
4792
4793         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4794         if (!ret) {
4795                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4796                                  MLX5_FLOW_LAYER_INNER_L3 :
4797                                  MLX5_FLOW_LAYER_OUTER_L3;
4798                 if (!(item_flags & layer))
4799                         return rte_flow_error_set(error, EINVAL,
4800                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4801                                                   NULL,
4802                                                   "no IP protocol in pattern");
4803         }
4804         return ret;
4805 }
4806
4807 /**
4808  * Validate the generic modify field actions.
4809  * @param[in] dev
4810  *   Pointer to the rte_eth_dev structure.
4811  * @param[in] action_flags
4812  *   Holds the actions detected until now.
4813  * @param[in] action
4814  *   Pointer to the modify action.
4815  * @param[in] attr
4816  *   Pointer to the flow attributes.
4817  * @param[out] error
4818  *   Pointer to error structure.
4819  *
4820  * @return
4821  *   Number of header fields to modify (0 or more) on success,
4822  *   a negative errno value otherwise and rte_errno is set.
4823  */
4824 static int
4825 flow_dv_validate_action_modify_field(struct rte_eth_dev *dev,
4826                                    const uint64_t action_flags,
4827                                    const struct rte_flow_action *action,
4828                                    const struct rte_flow_attr *attr,
4829                                    struct rte_flow_error *error)
4830 {
4831         int ret = 0;
4832         struct mlx5_priv *priv = dev->data->dev_private;
4833         struct mlx5_dev_config *config = &priv->config;
4834         const struct rte_flow_action_modify_field *action_modify_field =
4835                 action->conf;
4836         uint32_t dst_width = mlx5_flow_item_field_width(priv,
4837                                 action_modify_field->dst.field, -1);
4838         uint32_t src_width = mlx5_flow_item_field_width(priv,
4839                                 action_modify_field->src.field, dst_width);
4840
4841         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4842         if (ret)
4843                 return ret;
4844
4845         if (action_modify_field->width == 0)
4846                 return rte_flow_error_set(error, EINVAL,
4847                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4848                                 "no bits are requested to be modified");
4849         else if (action_modify_field->width > dst_width ||
4850                  action_modify_field->width > src_width)
4851                 return rte_flow_error_set(error, EINVAL,
4852                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4853                                 "cannot modify more bits than"
4854                                 " the width of a field");
4855         if (action_modify_field->dst.field != RTE_FLOW_FIELD_VALUE &&
4856             action_modify_field->dst.field != RTE_FLOW_FIELD_POINTER) {
4857                 if ((action_modify_field->dst.offset +
4858                      action_modify_field->width > dst_width) ||
4859                     (action_modify_field->dst.offset % 32))
4860                         return rte_flow_error_set(error, EINVAL,
4861                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4862                                         "destination offset is too big"
4863                                         " or not aligned to 4 bytes");
4864                 if (action_modify_field->dst.level &&
4865                     action_modify_field->dst.field != RTE_FLOW_FIELD_TAG)
4866                         return rte_flow_error_set(error, ENOTSUP,
4867                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4868                                         "inner header fields modification"
4869                                         " is not supported");
4870         }
4871         if (action_modify_field->src.field != RTE_FLOW_FIELD_VALUE &&
4872             action_modify_field->src.field != RTE_FLOW_FIELD_POINTER) {
4873                 if (!attr->transfer && !attr->group)
4874                         return rte_flow_error_set(error, ENOTSUP,
4875                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4876                                         "modify field action is not"
4877                                         " supported for group 0");
4878                 if ((action_modify_field->src.offset +
4879                      action_modify_field->width > src_width) ||
4880                     (action_modify_field->src.offset % 32))
4881                         return rte_flow_error_set(error, EINVAL,
4882                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4883                                         "source offset is too big"
4884                                         " or not aligned to 4 bytes");
4885                 if (action_modify_field->src.level &&
4886                     action_modify_field->src.field != RTE_FLOW_FIELD_TAG)
4887                         return rte_flow_error_set(error, ENOTSUP,
4888                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4889                                         "inner header fields modification"
4890                                         " is not supported");
4891         }
4892         if ((action_modify_field->dst.field ==
4893              action_modify_field->src.field) &&
4894             (action_modify_field->dst.level ==
4895              action_modify_field->src.level))
4896                 return rte_flow_error_set(error, EINVAL,
4897                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4898                                 "source and destination fields"
4899                                 " cannot be the same");
4900         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VALUE ||
4901             action_modify_field->dst.field == RTE_FLOW_FIELD_POINTER ||
4902             action_modify_field->dst.field == RTE_FLOW_FIELD_MARK)
4903                 return rte_flow_error_set(error, EINVAL,
4904                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4905                                 "mark, immediate value or a pointer to it"
4906                                 " cannot be used as a destination");
4907         if (action_modify_field->dst.field == RTE_FLOW_FIELD_START ||
4908             action_modify_field->src.field == RTE_FLOW_FIELD_START)
4909                 return rte_flow_error_set(error, ENOTSUP,
4910                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4911                                 "modifications of an arbitrary"
4912                                 " place in a packet is not supported");
4913         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VLAN_TYPE ||
4914             action_modify_field->src.field == RTE_FLOW_FIELD_VLAN_TYPE)
4915                 return rte_flow_error_set(error, ENOTSUP,
4916                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4917                                 "modifications of the 802.1Q Tag"
4918                                 " Identifier is not supported");
4919         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VXLAN_VNI ||
4920             action_modify_field->src.field == RTE_FLOW_FIELD_VXLAN_VNI)
4921                 return rte_flow_error_set(error, ENOTSUP,
4922                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4923                                 "modifications of the VXLAN Network"
4924                                 " Identifier is not supported");
4925         if (action_modify_field->dst.field == RTE_FLOW_FIELD_GENEVE_VNI ||
4926             action_modify_field->src.field == RTE_FLOW_FIELD_GENEVE_VNI)
4927                 return rte_flow_error_set(error, ENOTSUP,
4928                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4929                                 "modifications of the GENEVE Network"
4930                                 " Identifier is not supported");
4931         if (action_modify_field->dst.field == RTE_FLOW_FIELD_MARK ||
4932             action_modify_field->src.field == RTE_FLOW_FIELD_MARK ||
4933             action_modify_field->dst.field == RTE_FLOW_FIELD_META ||
4934             action_modify_field->src.field == RTE_FLOW_FIELD_META) {
4935                 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
4936                     !mlx5_flow_ext_mreg_supported(dev))
4937                         return rte_flow_error_set(error, ENOTSUP,
4938                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4939                                         "cannot modify mark or metadata without"
4940                                         " extended metadata register support");
4941         }
4942         if (action_modify_field->operation != RTE_FLOW_MODIFY_SET)
4943                 return rte_flow_error_set(error, ENOTSUP,
4944                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4945                                 "add and sub operations"
4946                                 " are not supported");
4947         return (action_modify_field->width / 32) +
4948                !!(action_modify_field->width % 32);
4949 }
4950
4951 /**
4952  * Validate jump action.
4953  *
4954  * @param[in] action
4955  *   Pointer to the jump action.
4956  * @param[in] action_flags
4957  *   Holds the actions detected until now.
4958  * @param[in] attributes
4959  *   Pointer to flow attributes
4960  * @param[in] external
4961  *   Action belongs to flow rule created by request external to PMD.
4962  * @param[out] error
4963  *   Pointer to error structure.
4964  *
4965  * @return
4966  *   0 on success, a negative errno value otherwise and rte_errno is set.
4967  */
4968 static int
4969 flow_dv_validate_action_jump(struct rte_eth_dev *dev,
4970                              const struct mlx5_flow_tunnel *tunnel,
4971                              const struct rte_flow_action *action,
4972                              uint64_t action_flags,
4973                              const struct rte_flow_attr *attributes,
4974                              bool external, struct rte_flow_error *error)
4975 {
4976         uint32_t target_group, table;
4977         int ret = 0;
4978         struct flow_grp_info grp_info = {
4979                 .external = !!external,
4980                 .transfer = !!attributes->transfer,
4981                 .fdb_def_rule = 1,
4982                 .std_tbl_fix = 0
4983         };
4984         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
4985                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4986                 return rte_flow_error_set(error, EINVAL,
4987                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4988                                           "can't have 2 fate actions in"
4989                                           " same flow");
4990         if (!action->conf)
4991                 return rte_flow_error_set(error, EINVAL,
4992                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4993                                           NULL, "action configuration not set");
4994         target_group =
4995                 ((const struct rte_flow_action_jump *)action->conf)->group;
4996         ret = mlx5_flow_group_to_table(dev, tunnel, target_group, &table,
4997                                        &grp_info, error);
4998         if (ret)
4999                 return ret;
5000         if (attributes->group == target_group &&
5001             !(action_flags & (MLX5_FLOW_ACTION_TUNNEL_SET |
5002                               MLX5_FLOW_ACTION_TUNNEL_MATCH)))
5003                 return rte_flow_error_set(error, EINVAL,
5004                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5005                                           "target group must be other than"
5006                                           " the current flow group");
5007         return 0;
5008 }
5009
5010 /*
5011  * Validate action PORT_ID / REPRESENTED_PORT.
5012  *
5013  * @param[in] dev
5014  *   Pointer to rte_eth_dev structure.
5015  * @param[in] action_flags
5016  *   Bit-fields that holds the actions detected until now.
5017  * @param[in] action
5018  *   PORT_ID / REPRESENTED_PORT action structure.
5019  * @param[in] attr
5020  *   Attributes of flow that includes this action.
5021  * @param[out] error
5022  *   Pointer to error structure.
5023  *
5024  * @return
5025  *   0 on success, a negative errno value otherwise and rte_errno is set.
5026  */
5027 static int
5028 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
5029                                 uint64_t action_flags,
5030                                 const struct rte_flow_action *action,
5031                                 const struct rte_flow_attr *attr,
5032                                 struct rte_flow_error *error)
5033 {
5034         const struct rte_flow_action_port_id *port_id;
5035         const struct rte_flow_action_ethdev *ethdev;
5036         struct mlx5_priv *act_priv;
5037         struct mlx5_priv *dev_priv;
5038         uint16_t port;
5039
5040         if (!attr->transfer)
5041                 return rte_flow_error_set(error, ENOTSUP,
5042                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5043                                           NULL,
5044                                           "port action is valid in transfer"
5045                                           " mode only");
5046         if (!action || !action->conf)
5047                 return rte_flow_error_set(error, ENOTSUP,
5048                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
5049                                           NULL,
5050                                           "port action parameters must be"
5051                                           " specified");
5052         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
5053                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5054                 return rte_flow_error_set(error, EINVAL,
5055                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5056                                           "can have only one fate actions in"
5057                                           " a flow");
5058         dev_priv = mlx5_dev_to_eswitch_info(dev);
5059         if (!dev_priv)
5060                 return rte_flow_error_set(error, rte_errno,
5061                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5062                                           NULL,
5063                                           "failed to obtain E-Switch info");
5064         switch (action->type) {
5065         case RTE_FLOW_ACTION_TYPE_PORT_ID:
5066                 port_id = action->conf;
5067                 port = port_id->original ? dev->data->port_id : port_id->id;
5068                 break;
5069         case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
5070                 ethdev = action->conf;
5071                 port = ethdev->port_id;
5072                 break;
5073         default:
5074                 MLX5_ASSERT(false);
5075                 return rte_flow_error_set
5076                                 (error, EINVAL,
5077                                  RTE_FLOW_ERROR_TYPE_ACTION, action,
5078                                  "unknown E-Switch action");
5079         }
5080         act_priv = mlx5_port_to_eswitch_info(port, false);
5081         if (!act_priv)
5082                 return rte_flow_error_set
5083                                 (error, rte_errno,
5084                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, action->conf,
5085                                  "failed to obtain E-Switch port id for port");
5086         if (act_priv->domain_id != dev_priv->domain_id)
5087                 return rte_flow_error_set
5088                                 (error, EINVAL,
5089                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5090                                  "port does not belong to"
5091                                  " E-Switch being configured");
5092         return 0;
5093 }
5094
5095 /**
5096  * Get the maximum number of modify header actions.
5097  *
5098  * @param dev
5099  *   Pointer to rte_eth_dev structure.
5100  * @param root
5101  *   Whether action is on root table.
5102  *
5103  * @return
5104  *   Max number of modify header actions device can support.
5105  */
5106 static inline unsigned int
5107 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
5108                               bool root)
5109 {
5110         /*
5111          * There's no way to directly query the max capacity from FW.
5112          * The maximal value on root table should be assumed to be supported.
5113          */
5114         if (!root)
5115                 return MLX5_MAX_MODIFY_NUM;
5116         else
5117                 return MLX5_ROOT_TBL_MODIFY_NUM;
5118 }
5119
5120 /**
5121  * Validate the meter action.
5122  *
5123  * @param[in] dev
5124  *   Pointer to rte_eth_dev structure.
5125  * @param[in] action_flags
5126  *   Bit-fields that holds the actions detected until now.
5127  * @param[in] action
5128  *   Pointer to the meter action.
5129  * @param[in] attr
5130  *   Attributes of flow that includes this action.
5131  * @param[in] port_id_item
5132  *   Pointer to item indicating port id.
5133  * @param[out] error
5134  *   Pointer to error structure.
5135  *
5136  * @return
5137  *   0 on success, a negative errno value otherwise and rte_ernno is set.
5138  */
5139 static int
5140 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
5141                                 uint64_t action_flags,
5142                                 const struct rte_flow_action *action,
5143                                 const struct rte_flow_attr *attr,
5144                                 const struct rte_flow_item *port_id_item,
5145                                 bool *def_policy,
5146                                 struct rte_flow_error *error)
5147 {
5148         struct mlx5_priv *priv = dev->data->dev_private;
5149         const struct rte_flow_action_meter *am = action->conf;
5150         struct mlx5_flow_meter_info *fm;
5151         struct mlx5_flow_meter_policy *mtr_policy;
5152         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
5153
5154         if (!am)
5155                 return rte_flow_error_set(error, EINVAL,
5156                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5157                                           "meter action conf is NULL");
5158
5159         if (action_flags & MLX5_FLOW_ACTION_METER)
5160                 return rte_flow_error_set(error, ENOTSUP,
5161                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5162                                           "meter chaining not support");
5163         if (action_flags & MLX5_FLOW_ACTION_JUMP)
5164                 return rte_flow_error_set(error, ENOTSUP,
5165                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5166                                           "meter with jump not support");
5167         if (!priv->mtr_en)
5168                 return rte_flow_error_set(error, ENOTSUP,
5169                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5170                                           NULL,
5171                                           "meter action not supported");
5172         fm = mlx5_flow_meter_find(priv, am->mtr_id, NULL);
5173         if (!fm)
5174                 return rte_flow_error_set(error, EINVAL,
5175                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5176                                           "Meter not found");
5177         /* aso meter can always be shared by different domains */
5178         if (fm->ref_cnt && !priv->sh->meter_aso_en &&
5179             !(fm->transfer == attr->transfer ||
5180               (!fm->ingress && !attr->ingress && attr->egress) ||
5181               (!fm->egress && !attr->egress && attr->ingress)))
5182                 return rte_flow_error_set(error, EINVAL,
5183                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5184                         "Flow attributes domain are either invalid "
5185                         "or have a domain conflict with current "
5186                         "meter attributes");
5187         if (fm->def_policy) {
5188                 if (!((attr->transfer &&
5189                         mtrmng->def_policy[MLX5_MTR_DOMAIN_TRANSFER]) ||
5190                         (attr->egress &&
5191                         mtrmng->def_policy[MLX5_MTR_DOMAIN_EGRESS]) ||
5192                         (attr->ingress &&
5193                         mtrmng->def_policy[MLX5_MTR_DOMAIN_INGRESS])))
5194                         return rte_flow_error_set(error, EINVAL,
5195                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5196                                           "Flow attributes domain "
5197                                           "have a conflict with current "
5198                                           "meter domain attributes");
5199                 *def_policy = true;
5200         } else {
5201                 mtr_policy = mlx5_flow_meter_policy_find(dev,
5202                                                 fm->policy_id, NULL);
5203                 if (!mtr_policy)
5204                         return rte_flow_error_set(error, EINVAL,
5205                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5206                                           "Invalid policy id for meter ");
5207                 if (!((attr->transfer && mtr_policy->transfer) ||
5208                         (attr->egress && mtr_policy->egress) ||
5209                         (attr->ingress && mtr_policy->ingress)))
5210                         return rte_flow_error_set(error, EINVAL,
5211                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5212                                           "Flow attributes domain "
5213                                           "have a conflict with current "
5214                                           "meter domain attributes");
5215                 if (attr->transfer && mtr_policy->dev) {
5216                         /**
5217                          * When policy has fate action of port_id,
5218                          * the flow should have the same src port as policy.
5219                          */
5220                         struct mlx5_priv *policy_port_priv =
5221                                         mtr_policy->dev->data->dev_private;
5222                         int32_t flow_src_port = priv->representor_id;
5223
5224                         if (port_id_item) {
5225                                 const struct rte_flow_item_port_id *spec =
5226                                                         port_id_item->spec;
5227                                 struct mlx5_priv *port_priv =
5228                                         mlx5_port_to_eswitch_info(spec->id,
5229                                                                   false);
5230                                 if (!port_priv)
5231                                         return rte_flow_error_set(error,
5232                                                 rte_errno,
5233                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5234                                                 spec,
5235                                                 "Failed to get port info.");
5236                                 flow_src_port = port_priv->representor_id;
5237                         }
5238                         if (flow_src_port != policy_port_priv->representor_id)
5239                                 return rte_flow_error_set(error,
5240                                                 rte_errno,
5241                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5242                                                 NULL,
5243                                                 "Flow and meter policy "
5244                                                 "have different src port.");
5245                 }
5246                 *def_policy = false;
5247         }
5248         return 0;
5249 }
5250
5251 /**
5252  * Validate the age action.
5253  *
5254  * @param[in] action_flags
5255  *   Holds the actions detected until now.
5256  * @param[in] action
5257  *   Pointer to the age action.
5258  * @param[in] dev
5259  *   Pointer to the Ethernet device structure.
5260  * @param[out] error
5261  *   Pointer to error structure.
5262  *
5263  * @return
5264  *   0 on success, a negative errno value otherwise and rte_errno is set.
5265  */
5266 static int
5267 flow_dv_validate_action_age(uint64_t action_flags,
5268                             const struct rte_flow_action *action,
5269                             struct rte_eth_dev *dev,
5270                             struct rte_flow_error *error)
5271 {
5272         struct mlx5_priv *priv = dev->data->dev_private;
5273         const struct rte_flow_action_age *age = action->conf;
5274
5275         if (!priv->sh->devx || (priv->sh->cmng.counter_fallback &&
5276             !priv->sh->aso_age_mng))
5277                 return rte_flow_error_set(error, ENOTSUP,
5278                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5279                                           NULL,
5280                                           "age action not supported");
5281         if (!(action->conf))
5282                 return rte_flow_error_set(error, EINVAL,
5283                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5284                                           "configuration cannot be null");
5285         if (!(age->timeout))
5286                 return rte_flow_error_set(error, EINVAL,
5287                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5288                                           "invalid timeout value 0");
5289         if (action_flags & MLX5_FLOW_ACTION_AGE)
5290                 return rte_flow_error_set(error, EINVAL,
5291                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5292                                           "duplicate age actions set");
5293         return 0;
5294 }
5295
5296 /**
5297  * Validate the modify-header IPv4 DSCP actions.
5298  *
5299  * @param[in] action_flags
5300  *   Holds the actions detected until now.
5301  * @param[in] action
5302  *   Pointer to the modify action.
5303  * @param[in] item_flags
5304  *   Holds the items detected.
5305  * @param[out] error
5306  *   Pointer to error structure.
5307  *
5308  * @return
5309  *   0 on success, a negative errno value otherwise and rte_errno is set.
5310  */
5311 static int
5312 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
5313                                          const struct rte_flow_action *action,
5314                                          const uint64_t item_flags,
5315                                          struct rte_flow_error *error)
5316 {
5317         int ret = 0;
5318
5319         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5320         if (!ret) {
5321                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
5322                         return rte_flow_error_set(error, EINVAL,
5323                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5324                                                   NULL,
5325                                                   "no ipv4 item in pattern");
5326         }
5327         return ret;
5328 }
5329
5330 /**
5331  * Validate the modify-header IPv6 DSCP actions.
5332  *
5333  * @param[in] action_flags
5334  *   Holds the actions detected until now.
5335  * @param[in] action
5336  *   Pointer to the modify action.
5337  * @param[in] item_flags
5338  *   Holds the items detected.
5339  * @param[out] error
5340  *   Pointer to error structure.
5341  *
5342  * @return
5343  *   0 on success, a negative errno value otherwise and rte_errno is set.
5344  */
5345 static int
5346 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
5347                                          const struct rte_flow_action *action,
5348                                          const uint64_t item_flags,
5349                                          struct rte_flow_error *error)
5350 {
5351         int ret = 0;
5352
5353         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5354         if (!ret) {
5355                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
5356                         return rte_flow_error_set(error, EINVAL,
5357                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5358                                                   NULL,
5359                                                   "no ipv6 item in pattern");
5360         }
5361         return ret;
5362 }
5363
5364 int
5365 flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
5366                         struct mlx5_list_entry *entry, void *cb_ctx)
5367 {
5368         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5369         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5370         struct mlx5_flow_dv_modify_hdr_resource *resource =
5371                                   container_of(entry, typeof(*resource), entry);
5372         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5373
5374         key_len += ref->actions_num * sizeof(ref->actions[0]);
5375         return ref->actions_num != resource->actions_num ||
5376                memcmp(&ref->ft_type, &resource->ft_type, key_len);
5377 }
5378
5379 static struct mlx5_indexed_pool *
5380 flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
5381 {
5382         struct mlx5_indexed_pool *ipool = __atomic_load_n
5383                                      (&sh->mdh_ipools[index], __ATOMIC_SEQ_CST);
5384
5385         if (!ipool) {
5386                 struct mlx5_indexed_pool *expected = NULL;
5387                 struct mlx5_indexed_pool_config cfg =
5388                     (struct mlx5_indexed_pool_config) {
5389                        .size = sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
5390                                                                    (index + 1) *
5391                                            sizeof(struct mlx5_modification_cmd),
5392                        .trunk_size = 64,
5393                        .grow_trunk = 3,
5394                        .grow_shift = 2,
5395                        .need_lock = 1,
5396                        .release_mem_en = !!sh->reclaim_mode,
5397                        .per_core_cache = sh->reclaim_mode ? 0 : (1 << 16),
5398                        .malloc = mlx5_malloc,
5399                        .free = mlx5_free,
5400                        .type = "mlx5_modify_action_resource",
5401                 };
5402
5403                 cfg.size = RTE_ALIGN(cfg.size, sizeof(ipool));
5404                 ipool = mlx5_ipool_create(&cfg);
5405                 if (!ipool)
5406                         return NULL;
5407                 if (!__atomic_compare_exchange_n(&sh->mdh_ipools[index],
5408                                                  &expected, ipool, false,
5409                                                  __ATOMIC_SEQ_CST,
5410                                                  __ATOMIC_SEQ_CST)) {
5411                         mlx5_ipool_destroy(ipool);
5412                         ipool = __atomic_load_n(&sh->mdh_ipools[index],
5413                                                 __ATOMIC_SEQ_CST);
5414                 }
5415         }
5416         return ipool;
5417 }
5418
5419 struct mlx5_list_entry *
5420 flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
5421 {
5422         struct mlx5_dev_ctx_shared *sh = tool_ctx;
5423         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5424         struct mlx5dv_dr_domain *ns;
5425         struct mlx5_flow_dv_modify_hdr_resource *entry;
5426         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5427         struct mlx5_indexed_pool *ipool = flow_dv_modify_ipool_get(sh,
5428                                                           ref->actions_num - 1);
5429         int ret;
5430         uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
5431         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5432         uint32_t idx;
5433
5434         if (unlikely(!ipool)) {
5435                 rte_flow_error_set(ctx->error, ENOMEM,
5436                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5437                                    NULL, "cannot allocate modify ipool");
5438                 return NULL;
5439         }
5440         entry = mlx5_ipool_zmalloc(ipool, &idx);
5441         if (!entry) {
5442                 rte_flow_error_set(ctx->error, ENOMEM,
5443                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5444                                    "cannot allocate resource memory");
5445                 return NULL;
5446         }
5447         rte_memcpy(&entry->ft_type,
5448                    RTE_PTR_ADD(ref, offsetof(typeof(*ref), ft_type)),
5449                    key_len + data_len);
5450         if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
5451                 ns = sh->fdb_domain;
5452         else if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
5453                 ns = sh->tx_domain;
5454         else
5455                 ns = sh->rx_domain;
5456         ret = mlx5_flow_os_create_flow_action_modify_header
5457                                         (sh->cdev->ctx, ns, entry,
5458                                          data_len, &entry->action);
5459         if (ret) {
5460                 mlx5_ipool_free(sh->mdh_ipools[ref->actions_num - 1], idx);
5461                 rte_flow_error_set(ctx->error, ENOMEM,
5462                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5463                                    NULL, "cannot create modification action");
5464                 return NULL;
5465         }
5466         entry->idx = idx;
5467         return &entry->entry;
5468 }
5469
5470 struct mlx5_list_entry *
5471 flow_dv_modify_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
5472                         void *cb_ctx)
5473 {
5474         struct mlx5_dev_ctx_shared *sh = tool_ctx;
5475         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5476         struct mlx5_flow_dv_modify_hdr_resource *entry;
5477         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5478         uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
5479         uint32_t idx;
5480
5481         entry = mlx5_ipool_malloc(sh->mdh_ipools[ref->actions_num - 1],
5482                                   &idx);
5483         if (!entry) {
5484                 rte_flow_error_set(ctx->error, ENOMEM,
5485                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5486                                    "cannot allocate resource memory");
5487                 return NULL;
5488         }
5489         memcpy(entry, oentry, sizeof(*entry) + data_len);
5490         entry->idx = idx;
5491         return &entry->entry;
5492 }
5493
5494 void
5495 flow_dv_modify_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
5496 {
5497         struct mlx5_dev_ctx_shared *sh = tool_ctx;
5498         struct mlx5_flow_dv_modify_hdr_resource *res =
5499                 container_of(entry, typeof(*res), entry);
5500
5501         mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
5502 }
5503
5504 /**
5505  * Validate the sample action.
5506  *
5507  * @param[in, out] action_flags
5508  *   Holds the actions detected until now.
5509  * @param[in] action
5510  *   Pointer to the sample action.
5511  * @param[in] dev
5512  *   Pointer to the Ethernet device structure.
5513  * @param[in] attr
5514  *   Attributes of flow that includes this action.
5515  * @param[in] item_flags
5516  *   Holds the items detected.
5517  * @param[in] rss
5518  *   Pointer to the RSS action.
5519  * @param[out] sample_rss
5520  *   Pointer to the RSS action in sample action list.
5521  * @param[out] count
5522  *   Pointer to the COUNT action in sample action list.
5523  * @param[out] fdb_mirror_limit
5524  *   Pointer to the FDB mirror limitation flag.
5525  * @param[out] error
5526  *   Pointer to error structure.
5527  *
5528  * @return
5529  *   0 on success, a negative errno value otherwise and rte_errno is set.
5530  */
5531 static int
5532 flow_dv_validate_action_sample(uint64_t *action_flags,
5533                                const struct rte_flow_action *action,
5534                                struct rte_eth_dev *dev,
5535                                const struct rte_flow_attr *attr,
5536                                uint64_t item_flags,
5537                                const struct rte_flow_action_rss *rss,
5538                                const struct rte_flow_action_rss **sample_rss,
5539                                const struct rte_flow_action_count **count,
5540                                int *fdb_mirror_limit,
5541                                struct rte_flow_error *error)
5542 {
5543         struct mlx5_priv *priv = dev->data->dev_private;
5544         struct mlx5_dev_config *dev_conf = &priv->config;
5545         const struct rte_flow_action_sample *sample = action->conf;
5546         const struct rte_flow_action *act;
5547         uint64_t sub_action_flags = 0;
5548         uint16_t queue_index = 0xFFFF;
5549         int actions_n = 0;
5550         int ret;
5551
5552         if (!sample)
5553                 return rte_flow_error_set(error, EINVAL,
5554                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5555                                           "configuration cannot be NULL");
5556         if (sample->ratio == 0)
5557                 return rte_flow_error_set(error, EINVAL,
5558                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5559                                           "ratio value starts from 1");
5560         if (!priv->sh->devx || (sample->ratio > 0 && !priv->sampler_en))
5561                 return rte_flow_error_set(error, ENOTSUP,
5562                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5563                                           NULL,
5564                                           "sample action not supported");
5565         if (*action_flags & MLX5_FLOW_ACTION_SAMPLE)
5566                 return rte_flow_error_set(error, EINVAL,
5567                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5568                                           "Multiple sample actions not "
5569                                           "supported");
5570         if (*action_flags & MLX5_FLOW_ACTION_METER)
5571                 return rte_flow_error_set(error, EINVAL,
5572                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5573                                           "wrong action order, meter should "
5574                                           "be after sample action");
5575         if (*action_flags & MLX5_FLOW_ACTION_JUMP)
5576                 return rte_flow_error_set(error, EINVAL,
5577                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5578                                           "wrong action order, jump should "
5579                                           "be after sample action");
5580         act = sample->actions;
5581         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
5582                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5583                         return rte_flow_error_set(error, ENOTSUP,
5584                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5585                                                   act, "too many actions");
5586                 switch (act->type) {
5587                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5588                         ret = mlx5_flow_validate_action_queue(act,
5589                                                               sub_action_flags,
5590                                                               dev,
5591                                                               attr, error);
5592                         if (ret < 0)
5593                                 return ret;
5594                         queue_index = ((const struct rte_flow_action_queue *)
5595                                                         (act->conf))->index;
5596                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
5597                         ++actions_n;
5598                         break;
5599                 case RTE_FLOW_ACTION_TYPE_RSS:
5600                         *sample_rss = act->conf;
5601                         ret = mlx5_flow_validate_action_rss(act,
5602                                                             sub_action_flags,
5603                                                             dev, attr,
5604                                                             item_flags,
5605                                                             error);
5606                         if (ret < 0)
5607                                 return ret;
5608                         if (rss && *sample_rss &&
5609                             ((*sample_rss)->level != rss->level ||
5610                             (*sample_rss)->types != rss->types))
5611                                 return rte_flow_error_set(error, ENOTSUP,
5612                                         RTE_FLOW_ERROR_TYPE_ACTION,
5613                                         NULL,
5614                                         "Can't use the different RSS types "
5615                                         "or level in the same flow");
5616                         if (*sample_rss != NULL && (*sample_rss)->queue_num)
5617                                 queue_index = (*sample_rss)->queue[0];
5618                         sub_action_flags |= MLX5_FLOW_ACTION_RSS;
5619                         ++actions_n;
5620                         break;
5621                 case RTE_FLOW_ACTION_TYPE_MARK:
5622                         ret = flow_dv_validate_action_mark(dev, act,
5623                                                            sub_action_flags,
5624                                                            attr, error);
5625                         if (ret < 0)
5626                                 return ret;
5627                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
5628                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
5629                                                 MLX5_FLOW_ACTION_MARK_EXT;
5630                         else
5631                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
5632                         ++actions_n;
5633                         break;
5634                 case RTE_FLOW_ACTION_TYPE_COUNT:
5635                         ret = flow_dv_validate_action_count
5636                                 (dev, false, *action_flags | sub_action_flags,
5637                                  error);
5638                         if (ret < 0)
5639                                 return ret;
5640                         *count = act->conf;
5641                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
5642                         *action_flags |= MLX5_FLOW_ACTION_COUNT;
5643                         ++actions_n;
5644                         break;
5645                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5646                 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
5647                         ret = flow_dv_validate_action_port_id(dev,
5648                                                               sub_action_flags,
5649                                                               act,
5650                                                               attr,
5651                                                               error);
5652                         if (ret)
5653                                 return ret;
5654                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5655                         ++actions_n;
5656                         break;
5657                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5658                         ret = flow_dv_validate_action_raw_encap_decap
5659                                 (dev, NULL, act->conf, attr, &sub_action_flags,
5660                                  &actions_n, action, item_flags, error);
5661                         if (ret < 0)
5662                                 return ret;
5663                         ++actions_n;
5664                         break;
5665                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5666                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5667                         ret = flow_dv_validate_action_l2_encap(dev,
5668                                                                sub_action_flags,
5669                                                                act, attr,
5670                                                                error);
5671                         if (ret < 0)
5672                                 return ret;
5673                         sub_action_flags |= MLX5_FLOW_ACTION_ENCAP;
5674                         ++actions_n;
5675                         break;
5676                 default:
5677                         return rte_flow_error_set(error, ENOTSUP,
5678                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5679                                                   NULL,
5680                                                   "Doesn't support optional "
5681                                                   "action");
5682                 }
5683         }
5684         if (attr->ingress && !attr->transfer) {
5685                 if (!(sub_action_flags & (MLX5_FLOW_ACTION_QUEUE |
5686                                           MLX5_FLOW_ACTION_RSS)))
5687                         return rte_flow_error_set(error, EINVAL,
5688                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5689                                                   NULL,
5690                                                   "Ingress must has a dest "
5691                                                   "QUEUE for Sample");
5692         } else if (attr->egress && !attr->transfer) {
5693                 return rte_flow_error_set(error, ENOTSUP,
5694                                           RTE_FLOW_ERROR_TYPE_ACTION,
5695                                           NULL,
5696                                           "Sample Only support Ingress "
5697                                           "or E-Switch");
5698         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
5699                 MLX5_ASSERT(attr->transfer);
5700                 if (sample->ratio > 1)
5701                         return rte_flow_error_set(error, ENOTSUP,
5702                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5703                                                   NULL,
5704                                                   "E-Switch doesn't support "
5705                                                   "any optional action "
5706                                                   "for sampling");
5707                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
5708                         return rte_flow_error_set(error, ENOTSUP,
5709                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5710                                                   NULL,
5711                                                   "unsupported action QUEUE");
5712                 if (sub_action_flags & MLX5_FLOW_ACTION_RSS)
5713                         return rte_flow_error_set(error, ENOTSUP,
5714                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5715                                                   NULL,
5716                                                   "unsupported action QUEUE");
5717                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
5718                         return rte_flow_error_set(error, EINVAL,
5719                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5720                                                   NULL,
5721                                                   "E-Switch must has a dest "
5722                                                   "port for mirroring");
5723                 if (!priv->config.hca_attr.reg_c_preserve &&
5724                      priv->representor_id != UINT16_MAX)
5725                         *fdb_mirror_limit = 1;
5726         }
5727         /* Continue validation for Xcap actions.*/
5728         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
5729             (queue_index == 0xFFFF ||
5730              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
5731                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
5732                      MLX5_FLOW_XCAP_ACTIONS)
5733                         return rte_flow_error_set(error, ENOTSUP,
5734                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5735                                                   NULL, "encap and decap "
5736                                                   "combination aren't "
5737                                                   "supported");
5738                 if (!attr->transfer && attr->ingress && (sub_action_flags &
5739                                                         MLX5_FLOW_ACTION_ENCAP))
5740                         return rte_flow_error_set(error, ENOTSUP,
5741                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5742                                                   NULL, "encap is not supported"
5743                                                   " for ingress traffic");
5744         }
5745         return 0;
5746 }
5747
5748 /**
5749  * Find existing modify-header resource or create and register a new one.
5750  *
5751  * @param dev[in, out]
5752  *   Pointer to rte_eth_dev structure.
5753  * @param[in, out] resource
5754  *   Pointer to modify-header resource.
5755  * @parm[in, out] dev_flow
5756  *   Pointer to the dev_flow.
5757  * @param[out] error
5758  *   pointer to error structure.
5759  *
5760  * @return
5761  *   0 on success otherwise -errno and errno is set.
5762  */
5763 static int
5764 flow_dv_modify_hdr_resource_register
5765                         (struct rte_eth_dev *dev,
5766                          struct mlx5_flow_dv_modify_hdr_resource *resource,
5767                          struct mlx5_flow *dev_flow,
5768                          struct rte_flow_error *error)
5769 {
5770         struct mlx5_priv *priv = dev->data->dev_private;
5771         struct mlx5_dev_ctx_shared *sh = priv->sh;
5772         uint32_t key_len = sizeof(*resource) -
5773                            offsetof(typeof(*resource), ft_type) +
5774                            resource->actions_num * sizeof(resource->actions[0]);
5775         struct mlx5_list_entry *entry;
5776         struct mlx5_flow_cb_ctx ctx = {
5777                 .error = error,
5778                 .data = resource,
5779         };
5780         struct mlx5_hlist *modify_cmds;
5781         uint64_t key64;
5782
5783         modify_cmds = flow_dv_hlist_prepare(sh, &sh->modify_cmds,
5784                                 "hdr_modify",
5785                                 MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
5786                                 true, false, sh,
5787                                 flow_dv_modify_create_cb,
5788                                 flow_dv_modify_match_cb,
5789                                 flow_dv_modify_remove_cb,
5790                                 flow_dv_modify_clone_cb,
5791                                 flow_dv_modify_clone_free_cb);
5792         if (unlikely(!modify_cmds))
5793                 return -rte_errno;
5794         resource->root = !dev_flow->dv.group;
5795         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
5796                                                                 resource->root))
5797                 return rte_flow_error_set(error, EOVERFLOW,
5798                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5799                                           "too many modify header items");
5800         key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0);
5801         entry = mlx5_hlist_register(modify_cmds, key64, &ctx);
5802         if (!entry)
5803                 return -rte_errno;
5804         resource = container_of(entry, typeof(*resource), entry);
5805         dev_flow->handle->dvh.modify_hdr = resource;
5806         return 0;
5807 }
5808
5809 /**
5810  * Get DV flow counter by index.
5811  *
5812  * @param[in] dev
5813  *   Pointer to the Ethernet device structure.
5814  * @param[in] idx
5815  *   mlx5 flow counter index in the container.
5816  * @param[out] ppool
5817  *   mlx5 flow counter pool in the container.
5818  *
5819  * @return
5820  *   Pointer to the counter, NULL otherwise.
5821  */
5822 static struct mlx5_flow_counter *
5823 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
5824                            uint32_t idx,
5825                            struct mlx5_flow_counter_pool **ppool)
5826 {
5827         struct mlx5_priv *priv = dev->data->dev_private;
5828         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5829         struct mlx5_flow_counter_pool *pool;
5830
5831         /* Decrease to original index and clear shared bit. */
5832         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
5833         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cmng->n);
5834         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
5835         MLX5_ASSERT(pool);
5836         if (ppool)
5837                 *ppool = pool;
5838         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
5839 }
5840
5841 /**
5842  * Check the devx counter belongs to the pool.
5843  *
5844  * @param[in] pool
5845  *   Pointer to the counter pool.
5846  * @param[in] id
5847  *   The counter devx ID.
5848  *
5849  * @return
5850  *   True if counter belongs to the pool, false otherwise.
5851  */
5852 static bool
5853 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
5854 {
5855         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
5856                    MLX5_COUNTERS_PER_POOL;
5857
5858         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
5859                 return true;
5860         return false;
5861 }
5862
5863 /**
5864  * Get a pool by devx counter ID.
5865  *
5866  * @param[in] cmng
5867  *   Pointer to the counter management.
5868  * @param[in] id
5869  *   The counter devx ID.
5870  *
5871  * @return
5872  *   The counter pool pointer if exists, NULL otherwise,
5873  */
5874 static struct mlx5_flow_counter_pool *
5875 flow_dv_find_pool_by_id(struct mlx5_flow_counter_mng *cmng, int id)
5876 {
5877         uint32_t i;
5878         struct mlx5_flow_counter_pool *pool = NULL;
5879
5880         rte_spinlock_lock(&cmng->pool_update_sl);
5881         /* Check last used pool. */
5882         if (cmng->last_pool_idx != POOL_IDX_INVALID &&
5883             flow_dv_is_counter_in_pool(cmng->pools[cmng->last_pool_idx], id)) {
5884                 pool = cmng->pools[cmng->last_pool_idx];
5885                 goto out;
5886         }
5887         /* ID out of range means no suitable pool in the container. */
5888         if (id > cmng->max_id || id < cmng->min_id)
5889                 goto out;
5890         /*
5891          * Find the pool from the end of the container, since mostly counter
5892          * ID is sequence increasing, and the last pool should be the needed
5893          * one.
5894          */
5895         i = cmng->n_valid;
5896         while (i--) {
5897                 struct mlx5_flow_counter_pool *pool_tmp = cmng->pools[i];
5898
5899                 if (flow_dv_is_counter_in_pool(pool_tmp, id)) {
5900                         pool = pool_tmp;
5901                         break;
5902                 }
5903         }
5904 out:
5905         rte_spinlock_unlock(&cmng->pool_update_sl);
5906         return pool;
5907 }
5908
5909 /**
5910  * Resize a counter container.
5911  *
5912  * @param[in] dev
5913  *   Pointer to the Ethernet device structure.
5914  *
5915  * @return
5916  *   0 on success, otherwise negative errno value and rte_errno is set.
5917  */
5918 static int
5919 flow_dv_container_resize(struct rte_eth_dev *dev)
5920 {
5921         struct mlx5_priv *priv = dev->data->dev_private;
5922         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5923         void *old_pools = cmng->pools;
5924         uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
5925         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
5926         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
5927
5928         if (!pools) {
5929                 rte_errno = ENOMEM;
5930                 return -ENOMEM;
5931         }
5932         if (old_pools)
5933                 memcpy(pools, old_pools, cmng->n *
5934                                        sizeof(struct mlx5_flow_counter_pool *));
5935         cmng->n = resize;
5936         cmng->pools = pools;
5937         if (old_pools)
5938                 mlx5_free(old_pools);
5939         return 0;
5940 }
5941
5942 /**
5943  * Query a devx flow counter.
5944  *
5945  * @param[in] dev
5946  *   Pointer to the Ethernet device structure.
5947  * @param[in] counter
5948  *   Index to the flow counter.
5949  * @param[out] pkts
5950  *   The statistics value of packets.
5951  * @param[out] bytes
5952  *   The statistics value of bytes.
5953  *
5954  * @return
5955  *   0 on success, otherwise a negative errno value and rte_errno is set.
5956  */
5957 static inline int
5958 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
5959                      uint64_t *bytes)
5960 {
5961         struct mlx5_priv *priv = dev->data->dev_private;
5962         struct mlx5_flow_counter_pool *pool = NULL;
5963         struct mlx5_flow_counter *cnt;
5964         int offset;
5965
5966         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
5967         MLX5_ASSERT(pool);
5968         if (priv->sh->cmng.counter_fallback)
5969                 return mlx5_devx_cmd_flow_counter_query(cnt->dcs_when_active, 0,
5970                                         0, pkts, bytes, 0, NULL, NULL, 0);
5971         rte_spinlock_lock(&pool->sl);
5972         if (!pool->raw) {
5973                 *pkts = 0;
5974                 *bytes = 0;
5975         } else {
5976                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
5977                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
5978                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
5979         }
5980         rte_spinlock_unlock(&pool->sl);
5981         return 0;
5982 }
5983
5984 /**
5985  * Create and initialize a new counter pool.
5986  *
5987  * @param[in] dev
5988  *   Pointer to the Ethernet device structure.
5989  * @param[out] dcs
5990  *   The devX counter handle.
5991  * @param[in] age
5992  *   Whether the pool is for counter that was allocated for aging.
5993  * @param[in/out] cont_cur
5994  *   Pointer to the container pointer, it will be update in pool resize.
5995  *
5996  * @return
5997  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
5998  */
5999 static struct mlx5_flow_counter_pool *
6000 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
6001                     uint32_t age)
6002 {
6003         struct mlx5_priv *priv = dev->data->dev_private;
6004         struct mlx5_flow_counter_pool *pool;
6005         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6006         bool fallback = priv->sh->cmng.counter_fallback;
6007         uint32_t size = sizeof(*pool);
6008
6009         size += MLX5_COUNTERS_PER_POOL * MLX5_CNT_SIZE;
6010         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * MLX5_AGE_SIZE);
6011         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
6012         if (!pool) {
6013                 rte_errno = ENOMEM;
6014                 return NULL;
6015         }
6016         pool->raw = NULL;
6017         pool->is_aged = !!age;
6018         pool->query_gen = 0;
6019         pool->min_dcs = dcs;
6020         rte_spinlock_init(&pool->sl);
6021         rte_spinlock_init(&pool->csl);
6022         TAILQ_INIT(&pool->counters[0]);
6023         TAILQ_INIT(&pool->counters[1]);
6024         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
6025         rte_spinlock_lock(&cmng->pool_update_sl);
6026         pool->index = cmng->n_valid;
6027         if (pool->index == cmng->n && flow_dv_container_resize(dev)) {
6028                 mlx5_free(pool);
6029                 rte_spinlock_unlock(&cmng->pool_update_sl);
6030                 return NULL;
6031         }
6032         cmng->pools[pool->index] = pool;
6033         cmng->n_valid++;
6034         if (unlikely(fallback)) {
6035                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
6036
6037                 if (base < cmng->min_id)
6038                         cmng->min_id = base;
6039                 if (base > cmng->max_id)
6040                         cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
6041                 cmng->last_pool_idx = pool->index;
6042         }
6043         rte_spinlock_unlock(&cmng->pool_update_sl);
6044         return pool;
6045 }
6046
6047 /**
6048  * Prepare a new counter and/or a new counter pool.
6049  *
6050  * @param[in] dev
6051  *   Pointer to the Ethernet device structure.
6052  * @param[out] cnt_free
6053  *   Where to put the pointer of a new counter.
6054  * @param[in] age
6055  *   Whether the pool is for counter that was allocated for aging.
6056  *
6057  * @return
6058  *   The counter pool pointer and @p cnt_free is set on success,
6059  *   NULL otherwise and rte_errno is set.
6060  */
6061 static struct mlx5_flow_counter_pool *
6062 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
6063                              struct mlx5_flow_counter **cnt_free,
6064                              uint32_t age)
6065 {
6066         struct mlx5_priv *priv = dev->data->dev_private;
6067         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6068         struct mlx5_flow_counter_pool *pool;
6069         struct mlx5_counters tmp_tq;
6070         struct mlx5_devx_obj *dcs = NULL;
6071         struct mlx5_flow_counter *cnt;
6072         enum mlx5_counter_type cnt_type =
6073                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
6074         bool fallback = priv->sh->cmng.counter_fallback;
6075         uint32_t i;
6076
6077         if (fallback) {
6078                 /* bulk_bitmap must be 0 for single counter allocation. */
6079                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->cdev->ctx, 0);
6080                 if (!dcs)
6081                         return NULL;
6082                 pool = flow_dv_find_pool_by_id(cmng, dcs->id);
6083                 if (!pool) {
6084                         pool = flow_dv_pool_create(dev, dcs, age);
6085                         if (!pool) {
6086                                 mlx5_devx_cmd_destroy(dcs);
6087                                 return NULL;
6088                         }
6089                 }
6090                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
6091                 cnt = MLX5_POOL_GET_CNT(pool, i);
6092                 cnt->pool = pool;
6093                 cnt->dcs_when_free = dcs;
6094                 *cnt_free = cnt;
6095                 return pool;
6096         }
6097         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->cdev->ctx, 0x4);
6098         if (!dcs) {
6099                 rte_errno = ENODATA;
6100                 return NULL;
6101         }
6102         pool = flow_dv_pool_create(dev, dcs, age);
6103         if (!pool) {
6104                 mlx5_devx_cmd_destroy(dcs);
6105                 return NULL;
6106         }
6107         TAILQ_INIT(&tmp_tq);
6108         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
6109                 cnt = MLX5_POOL_GET_CNT(pool, i);
6110                 cnt->pool = pool;
6111                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
6112         }
6113         rte_spinlock_lock(&cmng->csl[cnt_type]);
6114         TAILQ_CONCAT(&cmng->counters[cnt_type], &tmp_tq, next);
6115         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6116         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
6117         (*cnt_free)->pool = pool;
6118         return pool;
6119 }
6120
6121 /**
6122  * Allocate a flow counter.
6123  *
6124  * @param[in] dev
6125  *   Pointer to the Ethernet device structure.
6126  * @param[in] age
6127  *   Whether the counter was allocated for aging.
6128  *
6129  * @return
6130  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6131  */
6132 static uint32_t
6133 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t age)
6134 {
6135         struct mlx5_priv *priv = dev->data->dev_private;
6136         struct mlx5_flow_counter_pool *pool = NULL;
6137         struct mlx5_flow_counter *cnt_free = NULL;
6138         bool fallback = priv->sh->cmng.counter_fallback;
6139         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6140         enum mlx5_counter_type cnt_type =
6141                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
6142         uint32_t cnt_idx;
6143
6144         if (!priv->sh->devx) {
6145                 rte_errno = ENOTSUP;
6146                 return 0;
6147         }
6148         /* Get free counters from container. */
6149         rte_spinlock_lock(&cmng->csl[cnt_type]);
6150         cnt_free = TAILQ_FIRST(&cmng->counters[cnt_type]);
6151         if (cnt_free)
6152                 TAILQ_REMOVE(&cmng->counters[cnt_type], cnt_free, next);
6153         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6154         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, age))
6155                 goto err;
6156         pool = cnt_free->pool;
6157         if (fallback)
6158                 cnt_free->dcs_when_active = cnt_free->dcs_when_free;
6159         /* Create a DV counter action only in the first time usage. */
6160         if (!cnt_free->action) {
6161                 uint16_t offset;
6162                 struct mlx5_devx_obj *dcs;
6163                 int ret;
6164
6165                 if (!fallback) {
6166                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
6167                         dcs = pool->min_dcs;
6168                 } else {
6169                         offset = 0;
6170                         dcs = cnt_free->dcs_when_free;
6171                 }
6172                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
6173                                                             &cnt_free->action);
6174                 if (ret) {
6175                         rte_errno = errno;
6176                         goto err;
6177                 }
6178         }
6179         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
6180                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
6181         /* Update the counter reset values. */
6182         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
6183                                  &cnt_free->bytes))
6184                 goto err;
6185         if (!fallback && !priv->sh->cmng.query_thread_on)
6186                 /* Start the asynchronous batch query by the host thread. */
6187                 mlx5_set_query_alarm(priv->sh);
6188         /*
6189          * When the count action isn't shared (by ID), shared_info field is
6190          * used for indirect action API's refcnt.
6191          * When the counter action is not shared neither by ID nor by indirect
6192          * action API, shared info must be 1.
6193          */
6194         cnt_free->shared_info.refcnt = 1;
6195         return cnt_idx;
6196 err:
6197         if (cnt_free) {
6198                 cnt_free->pool = pool;
6199                 if (fallback)
6200                         cnt_free->dcs_when_free = cnt_free->dcs_when_active;
6201                 rte_spinlock_lock(&cmng->csl[cnt_type]);
6202                 TAILQ_INSERT_TAIL(&cmng->counters[cnt_type], cnt_free, next);
6203                 rte_spinlock_unlock(&cmng->csl[cnt_type]);
6204         }
6205         return 0;
6206 }
6207
6208 /**
6209  * Get age param from counter index.
6210  *
6211  * @param[in] dev
6212  *   Pointer to the Ethernet device structure.
6213  * @param[in] counter
6214  *   Index to the counter handler.
6215  *
6216  * @return
6217  *   The aging parameter specified for the counter index.
6218  */
6219 static struct mlx5_age_param*
6220 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
6221                                 uint32_t counter)
6222 {
6223         struct mlx5_flow_counter *cnt;
6224         struct mlx5_flow_counter_pool *pool = NULL;
6225
6226         flow_dv_counter_get_by_idx(dev, counter, &pool);
6227         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
6228         cnt = MLX5_POOL_GET_CNT(pool, counter);
6229         return MLX5_CNT_TO_AGE(cnt);
6230 }
6231
6232 /**
6233  * Remove a flow counter from aged counter list.
6234  *
6235  * @param[in] dev
6236  *   Pointer to the Ethernet device structure.
6237  * @param[in] counter
6238  *   Index to the counter handler.
6239  * @param[in] cnt
6240  *   Pointer to the counter handler.
6241  */
6242 static void
6243 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
6244                                 uint32_t counter, struct mlx5_flow_counter *cnt)
6245 {
6246         struct mlx5_age_info *age_info;
6247         struct mlx5_age_param *age_param;
6248         struct mlx5_priv *priv = dev->data->dev_private;
6249         uint16_t expected = AGE_CANDIDATE;
6250
6251         age_info = GET_PORT_AGE_INFO(priv);
6252         age_param = flow_dv_counter_idx_get_age(dev, counter);
6253         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
6254                                          AGE_FREE, false, __ATOMIC_RELAXED,
6255                                          __ATOMIC_RELAXED)) {
6256                 /**
6257                  * We need the lock even it is age timeout,
6258                  * since counter may still in process.
6259                  */
6260                 rte_spinlock_lock(&age_info->aged_sl);
6261                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
6262                 rte_spinlock_unlock(&age_info->aged_sl);
6263                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
6264         }
6265 }
6266
6267 /**
6268  * Release a flow counter.
6269  *
6270  * @param[in] dev
6271  *   Pointer to the Ethernet device structure.
6272  * @param[in] counter
6273  *   Index to the counter handler.
6274  */
6275 static void
6276 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter)
6277 {
6278         struct mlx5_priv *priv = dev->data->dev_private;
6279         struct mlx5_flow_counter_pool *pool = NULL;
6280         struct mlx5_flow_counter *cnt;
6281         enum mlx5_counter_type cnt_type;
6282
6283         if (!counter)
6284                 return;
6285         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
6286         MLX5_ASSERT(pool);
6287         if (pool->is_aged) {
6288                 flow_dv_counter_remove_from_age(dev, counter, cnt);
6289         } else {
6290                 /*
6291                  * If the counter action is shared by indirect action API,
6292                  * the atomic function reduces its references counter.
6293                  * If after the reduction the action is still referenced, the
6294                  * function returns here and does not release it.
6295                  * When the counter action is not shared by
6296                  * indirect action API, shared info is 1 before the reduction,
6297                  * so this condition is failed and function doesn't return here.
6298                  */
6299                 if (__atomic_sub_fetch(&cnt->shared_info.refcnt, 1,
6300                                        __ATOMIC_RELAXED))
6301                         return;
6302         }
6303         cnt->pool = pool;
6304         /*
6305          * Put the counter back to list to be updated in none fallback mode.
6306          * Currently, we are using two list alternately, while one is in query,
6307          * add the freed counter to the other list based on the pool query_gen
6308          * value. After query finishes, add counter the list to the global
6309          * container counter list. The list changes while query starts. In
6310          * this case, lock will not be needed as query callback and release
6311          * function both operate with the different list.
6312          */
6313         if (!priv->sh->cmng.counter_fallback) {
6314                 rte_spinlock_lock(&pool->csl);
6315                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
6316                 rte_spinlock_unlock(&pool->csl);
6317         } else {
6318                 cnt->dcs_when_free = cnt->dcs_when_active;
6319                 cnt_type = pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
6320                                            MLX5_COUNTER_TYPE_ORIGIN;
6321                 rte_spinlock_lock(&priv->sh->cmng.csl[cnt_type]);
6322                 TAILQ_INSERT_TAIL(&priv->sh->cmng.counters[cnt_type],
6323                                   cnt, next);
6324                 rte_spinlock_unlock(&priv->sh->cmng.csl[cnt_type]);
6325         }
6326 }
6327
6328 /**
6329  * Resize a meter id container.
6330  *
6331  * @param[in] dev
6332  *   Pointer to the Ethernet device structure.
6333  *
6334  * @return
6335  *   0 on success, otherwise negative errno value and rte_errno is set.
6336  */
6337 static int
6338 flow_dv_mtr_container_resize(struct rte_eth_dev *dev)
6339 {
6340         struct mlx5_priv *priv = dev->data->dev_private;
6341         struct mlx5_aso_mtr_pools_mng *pools_mng =
6342                                 &priv->sh->mtrmng->pools_mng;
6343         void *old_pools = pools_mng->pools;
6344         uint32_t resize = pools_mng->n + MLX5_MTRS_CONTAINER_RESIZE;
6345         uint32_t mem_size = sizeof(struct mlx5_aso_mtr_pool *) * resize;
6346         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
6347
6348         if (!pools) {
6349                 rte_errno = ENOMEM;
6350                 return -ENOMEM;
6351         }
6352         if (!pools_mng->n)
6353                 if (mlx5_aso_queue_init(priv->sh, ASO_OPC_MOD_POLICER)) {
6354                         mlx5_free(pools);
6355                         return -ENOMEM;
6356                 }
6357         if (old_pools)
6358                 memcpy(pools, old_pools, pools_mng->n *
6359                                        sizeof(struct mlx5_aso_mtr_pool *));
6360         pools_mng->n = resize;
6361         pools_mng->pools = pools;
6362         if (old_pools)
6363                 mlx5_free(old_pools);
6364         return 0;
6365 }
6366
6367 /**
6368  * Prepare a new meter and/or a new meter pool.
6369  *
6370  * @param[in] dev
6371  *   Pointer to the Ethernet device structure.
6372  * @param[out] mtr_free
6373  *   Where to put the pointer of a new meter.g.
6374  *
6375  * @return
6376  *   The meter pool pointer and @mtr_free is set on success,
6377  *   NULL otherwise and rte_errno is set.
6378  */
6379 static struct mlx5_aso_mtr_pool *
6380 flow_dv_mtr_pool_create(struct rte_eth_dev *dev, struct mlx5_aso_mtr **mtr_free)
6381 {
6382         struct mlx5_priv *priv = dev->data->dev_private;
6383         struct mlx5_aso_mtr_pools_mng *pools_mng = &priv->sh->mtrmng->pools_mng;
6384         struct mlx5_aso_mtr_pool *pool = NULL;
6385         struct mlx5_devx_obj *dcs = NULL;
6386         uint32_t i;
6387         uint32_t log_obj_size;
6388
6389         log_obj_size = rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1);
6390         dcs = mlx5_devx_cmd_create_flow_meter_aso_obj(priv->sh->cdev->ctx,
6391                                                       priv->sh->cdev->pdn,
6392                                                       log_obj_size);
6393         if (!dcs) {
6394                 rte_errno = ENODATA;
6395                 return NULL;
6396         }
6397         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
6398         if (!pool) {
6399                 rte_errno = ENOMEM;
6400                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6401                 return NULL;
6402         }
6403         pool->devx_obj = dcs;
6404         pool->index = pools_mng->n_valid;
6405         if (pool->index == pools_mng->n && flow_dv_mtr_container_resize(dev)) {
6406                 mlx5_free(pool);
6407                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6408                 return NULL;
6409         }
6410         pools_mng->pools[pool->index] = pool;
6411         pools_mng->n_valid++;
6412         for (i = 1; i < MLX5_ASO_MTRS_PER_POOL; ++i) {
6413                 pool->mtrs[i].offset = i;
6414                 LIST_INSERT_HEAD(&pools_mng->meters, &pool->mtrs[i], next);
6415         }
6416         pool->mtrs[0].offset = 0;
6417         *mtr_free = &pool->mtrs[0];
6418         return pool;
6419 }
6420
6421 /**
6422  * Release a flow meter into pool.
6423  *
6424  * @param[in] dev
6425  *   Pointer to the Ethernet device structure.
6426  * @param[in] mtr_idx
6427  *   Index to aso flow meter.
6428  */
6429 static void
6430 flow_dv_aso_mtr_release_to_pool(struct rte_eth_dev *dev, uint32_t mtr_idx)
6431 {
6432         struct mlx5_priv *priv = dev->data->dev_private;
6433         struct mlx5_aso_mtr_pools_mng *pools_mng =
6434                                 &priv->sh->mtrmng->pools_mng;
6435         struct mlx5_aso_mtr *aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
6436
6437         MLX5_ASSERT(aso_mtr);
6438         rte_spinlock_lock(&pools_mng->mtrsl);
6439         memset(&aso_mtr->fm, 0, sizeof(struct mlx5_flow_meter_info));
6440         aso_mtr->state = ASO_METER_FREE;
6441         LIST_INSERT_HEAD(&pools_mng->meters, aso_mtr, next);
6442         rte_spinlock_unlock(&pools_mng->mtrsl);
6443 }
6444
6445 /**
6446  * Allocate a aso flow meter.
6447  *
6448  * @param[in] dev
6449  *   Pointer to the Ethernet device structure.
6450  *
6451  * @return
6452  *   Index to aso flow meter on success, 0 otherwise and rte_errno is set.
6453  */
6454 static uint32_t
6455 flow_dv_mtr_alloc(struct rte_eth_dev *dev)
6456 {
6457         struct mlx5_priv *priv = dev->data->dev_private;
6458         struct mlx5_aso_mtr *mtr_free = NULL;
6459         struct mlx5_aso_mtr_pools_mng *pools_mng =
6460                                 &priv->sh->mtrmng->pools_mng;
6461         struct mlx5_aso_mtr_pool *pool;
6462         uint32_t mtr_idx = 0;
6463
6464         if (!priv->sh->devx) {
6465                 rte_errno = ENOTSUP;
6466                 return 0;
6467         }
6468         /* Allocate the flow meter memory. */
6469         /* Get free meters from management. */
6470         rte_spinlock_lock(&pools_mng->mtrsl);
6471         mtr_free = LIST_FIRST(&pools_mng->meters);
6472         if (mtr_free)
6473                 LIST_REMOVE(mtr_free, next);
6474         if (!mtr_free && !flow_dv_mtr_pool_create(dev, &mtr_free)) {
6475                 rte_spinlock_unlock(&pools_mng->mtrsl);
6476                 return 0;
6477         }
6478         mtr_free->state = ASO_METER_WAIT;
6479         rte_spinlock_unlock(&pools_mng->mtrsl);
6480         pool = container_of(mtr_free,
6481                         struct mlx5_aso_mtr_pool,
6482                         mtrs[mtr_free->offset]);
6483         mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset);
6484         if (!mtr_free->fm.meter_action) {
6485 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
6486                 struct rte_flow_error error;
6487                 uint8_t reg_id;
6488
6489                 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &error);
6490                 mtr_free->fm.meter_action =
6491                         mlx5_glue->dv_create_flow_action_aso
6492                                                 (priv->sh->rx_domain,
6493                                                  pool->devx_obj->obj,
6494                                                  mtr_free->offset,
6495                                                  (1 << MLX5_FLOW_COLOR_GREEN),
6496                                                  reg_id - REG_C_0);
6497 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
6498                 if (!mtr_free->fm.meter_action) {
6499                         flow_dv_aso_mtr_release_to_pool(dev, mtr_idx);
6500                         return 0;
6501                 }
6502         }
6503         return mtr_idx;
6504 }
6505
6506 /**
6507  * Verify the @p attributes will be correctly understood by the NIC and store
6508  * them in the @p flow if everything is correct.
6509  *
6510  * @param[in] dev
6511  *   Pointer to dev struct.
6512  * @param[in] attributes
6513  *   Pointer to flow attributes
6514  * @param[in] external
6515  *   This flow rule is created by request external to PMD.
6516  * @param[out] error
6517  *   Pointer to error structure.
6518  *
6519  * @return
6520  *   - 0 on success and non root table.
6521  *   - 1 on success and root table.
6522  *   - a negative errno value otherwise and rte_errno is set.
6523  */
6524 static int
6525 flow_dv_validate_attributes(struct rte_eth_dev *dev,
6526                             const struct mlx5_flow_tunnel *tunnel,
6527                             const struct rte_flow_attr *attributes,
6528                             const struct flow_grp_info *grp_info,
6529                             struct rte_flow_error *error)
6530 {
6531         struct mlx5_priv *priv = dev->data->dev_private;
6532         uint32_t lowest_priority = mlx5_get_lowest_priority(dev, attributes);
6533         int ret = 0;
6534
6535 #ifndef HAVE_MLX5DV_DR
6536         RTE_SET_USED(tunnel);
6537         RTE_SET_USED(grp_info);
6538         if (attributes->group)
6539                 return rte_flow_error_set(error, ENOTSUP,
6540                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6541                                           NULL,
6542                                           "groups are not supported");
6543 #else
6544         uint32_t table = 0;
6545
6546         ret = mlx5_flow_group_to_table(dev, tunnel, attributes->group, &table,
6547                                        grp_info, error);
6548         if (ret)
6549                 return ret;
6550         if (!table)
6551                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
6552 #endif
6553         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
6554             attributes->priority > lowest_priority)
6555                 return rte_flow_error_set(error, ENOTSUP,
6556                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
6557                                           NULL,
6558                                           "priority out of range");
6559         if (attributes->transfer) {
6560                 if (!priv->config.dv_esw_en)
6561                         return rte_flow_error_set
6562                                 (error, ENOTSUP,
6563                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6564                                  "E-Switch dr is not supported");
6565                 if (!(priv->representor || priv->master))
6566                         return rte_flow_error_set
6567                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6568                                  NULL, "E-Switch configuration can only be"
6569                                  " done by a master or a representor device");
6570                 if (attributes->egress)
6571                         return rte_flow_error_set
6572                                 (error, ENOTSUP,
6573                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
6574                                  "egress is not supported");
6575         }
6576         if (!(attributes->egress ^ attributes->ingress))
6577                 return rte_flow_error_set(error, ENOTSUP,
6578                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
6579                                           "must specify exactly one of "
6580                                           "ingress or egress");
6581         return ret;
6582 }
6583
6584 static uint16_t
6585 mlx5_flow_locate_proto_l3(const struct rte_flow_item **head,
6586                           const struct rte_flow_item *end)
6587 {
6588         const struct rte_flow_item *item = *head;
6589         uint16_t l3_protocol;
6590
6591         for (; item != end; item++) {
6592                 switch (item->type) {
6593                 default:
6594                         break;
6595                 case RTE_FLOW_ITEM_TYPE_IPV4:
6596                         l3_protocol = RTE_ETHER_TYPE_IPV4;
6597                         goto l3_ok;
6598                 case RTE_FLOW_ITEM_TYPE_IPV6:
6599                         l3_protocol = RTE_ETHER_TYPE_IPV6;
6600                         goto l3_ok;
6601                 case RTE_FLOW_ITEM_TYPE_ETH:
6602                         if (item->mask && item->spec) {
6603                                 MLX5_ETHER_TYPE_FROM_HEADER(rte_flow_item_eth,
6604                                                             type, item,
6605                                                             l3_protocol);
6606                                 if (l3_protocol == RTE_ETHER_TYPE_IPV4 ||
6607                                     l3_protocol == RTE_ETHER_TYPE_IPV6)
6608                                         goto l3_ok;
6609                         }
6610                         break;
6611                 case RTE_FLOW_ITEM_TYPE_VLAN:
6612                         if (item->mask && item->spec) {
6613                                 MLX5_ETHER_TYPE_FROM_HEADER(rte_flow_item_vlan,
6614                                                             inner_type, item,
6615                                                             l3_protocol);
6616                                 if (l3_protocol == RTE_ETHER_TYPE_IPV4 ||
6617                                     l3_protocol == RTE_ETHER_TYPE_IPV6)
6618                                         goto l3_ok;
6619                         }
6620                         break;
6621                 }
6622         }
6623         return 0;
6624 l3_ok:
6625         *head = item;
6626         return l3_protocol;
6627 }
6628
6629 static uint8_t
6630 mlx5_flow_locate_proto_l4(const struct rte_flow_item **head,
6631                           const struct rte_flow_item *end)
6632 {
6633         const struct rte_flow_item *item = *head;
6634         uint8_t l4_protocol;
6635
6636         for (; item != end; item++) {
6637                 switch (item->type) {
6638                 default:
6639                         break;
6640                 case RTE_FLOW_ITEM_TYPE_TCP:
6641                         l4_protocol = IPPROTO_TCP;
6642                         goto l4_ok;
6643                 case RTE_FLOW_ITEM_TYPE_UDP:
6644                         l4_protocol = IPPROTO_UDP;
6645                         goto l4_ok;
6646                 case RTE_FLOW_ITEM_TYPE_IPV4:
6647                         if (item->mask && item->spec) {
6648                                 const struct rte_flow_item_ipv4 *mask, *spec;
6649
6650                                 mask = (typeof(mask))item->mask;
6651                                 spec = (typeof(spec))item->spec;
6652                                 l4_protocol = mask->hdr.next_proto_id &
6653                                               spec->hdr.next_proto_id;
6654                                 if (l4_protocol == IPPROTO_TCP ||
6655                                     l4_protocol == IPPROTO_UDP)
6656                                         goto l4_ok;
6657                         }
6658                         break;
6659                 case RTE_FLOW_ITEM_TYPE_IPV6:
6660                         if (item->mask && item->spec) {
6661                                 const struct rte_flow_item_ipv6 *mask, *spec;
6662                                 mask = (typeof(mask))item->mask;
6663                                 spec = (typeof(spec))item->spec;
6664                                 l4_protocol = mask->hdr.proto & spec->hdr.proto;
6665                                 if (l4_protocol == IPPROTO_TCP ||
6666                                     l4_protocol == IPPROTO_UDP)
6667                                         goto l4_ok;
6668                         }
6669                         break;
6670                 }
6671         }
6672         return 0;
6673 l4_ok:
6674         *head = item;
6675         return l4_protocol;
6676 }
6677
6678 static int
6679 flow_dv_validate_item_integrity(struct rte_eth_dev *dev,
6680                                 const struct rte_flow_item *rule_items,
6681                                 const struct rte_flow_item *integrity_item,
6682                                 struct rte_flow_error *error)
6683 {
6684         struct mlx5_priv *priv = dev->data->dev_private;
6685         const struct rte_flow_item *tunnel_item, *end_item, *item = rule_items;
6686         const struct rte_flow_item_integrity *mask = (typeof(mask))
6687                                                      integrity_item->mask;
6688         const struct rte_flow_item_integrity *spec = (typeof(spec))
6689                                                      integrity_item->spec;
6690         uint32_t protocol;
6691
6692         if (!priv->config.hca_attr.pkt_integrity_match)
6693                 return rte_flow_error_set(error, ENOTSUP,
6694                                           RTE_FLOW_ERROR_TYPE_ITEM,
6695                                           integrity_item,
6696                                           "packet integrity integrity_item not supported");
6697         if (!mask)
6698                 mask = &rte_flow_item_integrity_mask;
6699         if (!mlx5_validate_integrity_item(mask))
6700                 return rte_flow_error_set(error, ENOTSUP,
6701                                           RTE_FLOW_ERROR_TYPE_ITEM,
6702                                           integrity_item,
6703                                           "unsupported integrity filter");
6704         tunnel_item = mlx5_flow_find_tunnel_item(rule_items);
6705         if (spec->level > 1) {
6706                 if (!tunnel_item)
6707                         return rte_flow_error_set(error, ENOTSUP,
6708                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6709                                                   integrity_item,
6710                                                   "missing tunnel item");
6711                 item = tunnel_item;
6712                 end_item = mlx5_find_end_item(tunnel_item);
6713         } else {
6714                 end_item = tunnel_item ? tunnel_item :
6715                            mlx5_find_end_item(integrity_item);
6716         }
6717         if (mask->l3_ok || mask->ipv4_csum_ok) {
6718                 protocol = mlx5_flow_locate_proto_l3(&item, end_item);
6719                 if (!protocol)
6720                         return rte_flow_error_set(error, EINVAL,
6721                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6722                                                   integrity_item,
6723                                                   "missing L3 protocol");
6724         }
6725         if (mask->l4_ok || mask->l4_csum_ok) {
6726                 protocol = mlx5_flow_locate_proto_l4(&item, end_item);
6727                 if (!protocol)
6728                         return rte_flow_error_set(error, EINVAL,
6729                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6730                                                   integrity_item,
6731                                                   "missing L4 protocol");
6732         }
6733         return 0;
6734 }
6735
6736 /**
6737  * Internal validation function. For validating both actions and items.
6738  *
6739  * @param[in] dev
6740  *   Pointer to the rte_eth_dev structure.
6741  * @param[in] attr
6742  *   Pointer to the flow attributes.
6743  * @param[in] items
6744  *   Pointer to the list of items.
6745  * @param[in] actions
6746  *   Pointer to the list of actions.
6747  * @param[in] external
6748  *   This flow rule is created by request external to PMD.
6749  * @param[in] hairpin
6750  *   Number of hairpin TX actions, 0 means classic flow.
6751  * @param[out] error
6752  *   Pointer to the error structure.
6753  *
6754  * @return
6755  *   0 on success, a negative errno value otherwise and rte_errno is set.
6756  */
6757 static int
6758 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
6759                  const struct rte_flow_item items[],
6760                  const struct rte_flow_action actions[],
6761                  bool external, int hairpin, struct rte_flow_error *error)
6762 {
6763         int ret;
6764         uint64_t action_flags = 0;
6765         uint64_t item_flags = 0;
6766         uint64_t last_item = 0;
6767         uint8_t next_protocol = 0xff;
6768         uint16_t ether_type = 0;
6769         int actions_n = 0;
6770         uint8_t item_ipv6_proto = 0;
6771         int fdb_mirror_limit = 0;
6772         int modify_after_mirror = 0;
6773         const struct rte_flow_item *geneve_item = NULL;
6774         const struct rte_flow_item *gre_item = NULL;
6775         const struct rte_flow_item *gtp_item = NULL;
6776         const struct rte_flow_action_raw_decap *decap;
6777         const struct rte_flow_action_raw_encap *encap;
6778         const struct rte_flow_action_rss *rss = NULL;
6779         const struct rte_flow_action_rss *sample_rss = NULL;
6780         const struct rte_flow_action_count *sample_count = NULL;
6781         const struct rte_flow_item_tcp nic_tcp_mask = {
6782                 .hdr = {
6783                         .tcp_flags = 0xFF,
6784                         .src_port = RTE_BE16(UINT16_MAX),
6785                         .dst_port = RTE_BE16(UINT16_MAX),
6786                 }
6787         };
6788         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
6789                 .hdr = {
6790                         .src_addr =
6791                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6792                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6793                         .dst_addr =
6794                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6795                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6796                         .vtc_flow = RTE_BE32(0xffffffff),
6797                         .proto = 0xff,
6798                         .hop_limits = 0xff,
6799                 },
6800                 .has_frag_ext = 1,
6801         };
6802         const struct rte_flow_item_ecpri nic_ecpri_mask = {
6803                 .hdr = {
6804                         .common = {
6805                                 .u32 =
6806                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
6807                                         .type = 0xFF,
6808                                         }).u32),
6809                         },
6810                         .dummy[0] = 0xffffffff,
6811                 },
6812         };
6813         struct mlx5_priv *priv = dev->data->dev_private;
6814         struct mlx5_dev_config *dev_conf = &priv->config;
6815         uint16_t queue_index = 0xFFFF;
6816         const struct rte_flow_item_vlan *vlan_m = NULL;
6817         uint32_t rw_act_num = 0;
6818         uint64_t is_root;
6819         const struct mlx5_flow_tunnel *tunnel;
6820         enum mlx5_tof_rule_type tof_rule_type;
6821         struct flow_grp_info grp_info = {
6822                 .external = !!external,
6823                 .transfer = !!attr->transfer,
6824                 .fdb_def_rule = !!priv->fdb_def_rule,
6825                 .std_tbl_fix = true,
6826         };
6827         const struct rte_eth_hairpin_conf *conf;
6828         const struct rte_flow_item *rule_items = items;
6829         const struct rte_flow_item *port_id_item = NULL;
6830         bool def_policy = false;
6831         uint16_t udp_dport = 0;
6832
6833         if (items == NULL)
6834                 return -1;
6835         tunnel = is_tunnel_offload_active(dev) ?
6836                  mlx5_get_tof(items, actions, &tof_rule_type) : NULL;
6837         if (tunnel) {
6838                 if (priv->representor)
6839                         return rte_flow_error_set
6840                                 (error, ENOTSUP,
6841                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6842                                  NULL, "decap not supported for VF representor");
6843                 if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_SET_RULE)
6844                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
6845                 else if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_MATCH_RULE)
6846                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_MATCH |
6847                                         MLX5_FLOW_ACTION_DECAP;
6848                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
6849                                         (dev, attr, tunnel, tof_rule_type);
6850         }
6851         ret = flow_dv_validate_attributes(dev, tunnel, attr, &grp_info, error);
6852         if (ret < 0)
6853                 return ret;
6854         is_root = (uint64_t)ret;
6855         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
6856                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
6857                 int type = items->type;
6858
6859                 if (!mlx5_flow_os_item_supported(type))
6860                         return rte_flow_error_set(error, ENOTSUP,
6861                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6862                                                   NULL, "item not supported");
6863                 switch (type) {
6864                 case RTE_FLOW_ITEM_TYPE_VOID:
6865                         break;
6866                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
6867                         ret = flow_dv_validate_item_port_id
6868                                         (dev, items, attr, item_flags, error);
6869                         if (ret < 0)
6870                                 return ret;
6871                         last_item = MLX5_FLOW_ITEM_PORT_ID;
6872                         port_id_item = items;
6873                         break;
6874                 case RTE_FLOW_ITEM_TYPE_ETH:
6875                         ret = mlx5_flow_validate_item_eth(items, item_flags,
6876                                                           true, error);
6877                         if (ret < 0)
6878                                 return ret;
6879                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
6880                                              MLX5_FLOW_LAYER_OUTER_L2;
6881                         if (items->mask != NULL && items->spec != NULL) {
6882                                 ether_type =
6883                                         ((const struct rte_flow_item_eth *)
6884                                          items->spec)->type;
6885                                 ether_type &=
6886                                         ((const struct rte_flow_item_eth *)
6887                                          items->mask)->type;
6888                                 ether_type = rte_be_to_cpu_16(ether_type);
6889                         } else {
6890                                 ether_type = 0;
6891                         }
6892                         break;
6893                 case RTE_FLOW_ITEM_TYPE_VLAN:
6894                         ret = flow_dv_validate_item_vlan(items, item_flags,
6895                                                          dev, error);
6896                         if (ret < 0)
6897                                 return ret;
6898                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
6899                                              MLX5_FLOW_LAYER_OUTER_VLAN;
6900                         if (items->mask != NULL && items->spec != NULL) {
6901                                 ether_type =
6902                                         ((const struct rte_flow_item_vlan *)
6903                                          items->spec)->inner_type;
6904                                 ether_type &=
6905                                         ((const struct rte_flow_item_vlan *)
6906                                          items->mask)->inner_type;
6907                                 ether_type = rte_be_to_cpu_16(ether_type);
6908                         } else {
6909                                 ether_type = 0;
6910                         }
6911                         /* Store outer VLAN mask for of_push_vlan action. */
6912                         if (!tunnel)
6913                                 vlan_m = items->mask;
6914                         break;
6915                 case RTE_FLOW_ITEM_TYPE_IPV4:
6916                         mlx5_flow_tunnel_ip_check(items, next_protocol,
6917                                                   &item_flags, &tunnel);
6918                         ret = flow_dv_validate_item_ipv4(dev, items, item_flags,
6919                                                          last_item, ether_type,
6920                                                          error);
6921                         if (ret < 0)
6922                                 return ret;
6923                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
6924                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
6925                         if (items->mask != NULL &&
6926                             ((const struct rte_flow_item_ipv4 *)
6927                              items->mask)->hdr.next_proto_id) {
6928                                 next_protocol =
6929                                         ((const struct rte_flow_item_ipv4 *)
6930                                          (items->spec))->hdr.next_proto_id;
6931                                 next_protocol &=
6932                                         ((const struct rte_flow_item_ipv4 *)
6933                                          (items->mask))->hdr.next_proto_id;
6934                         } else {
6935                                 /* Reset for inner layer. */
6936                                 next_protocol = 0xff;
6937                         }
6938                         break;
6939                 case RTE_FLOW_ITEM_TYPE_IPV6:
6940                         mlx5_flow_tunnel_ip_check(items, next_protocol,
6941                                                   &item_flags, &tunnel);
6942                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
6943                                                            last_item,
6944                                                            ether_type,
6945                                                            &nic_ipv6_mask,
6946                                                            error);
6947                         if (ret < 0)
6948                                 return ret;
6949                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
6950                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
6951                         if (items->mask != NULL &&
6952                             ((const struct rte_flow_item_ipv6 *)
6953                              items->mask)->hdr.proto) {
6954                                 item_ipv6_proto =
6955                                         ((const struct rte_flow_item_ipv6 *)
6956                                          items->spec)->hdr.proto;
6957                                 next_protocol =
6958                                         ((const struct rte_flow_item_ipv6 *)
6959                                          items->spec)->hdr.proto;
6960                                 next_protocol &=
6961                                         ((const struct rte_flow_item_ipv6 *)
6962                                          items->mask)->hdr.proto;
6963                         } else {
6964                                 /* Reset for inner layer. */
6965                                 next_protocol = 0xff;
6966                         }
6967                         break;
6968                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
6969                         ret = flow_dv_validate_item_ipv6_frag_ext(items,
6970                                                                   item_flags,
6971                                                                   error);
6972                         if (ret < 0)
6973                                 return ret;
6974                         last_item = tunnel ?
6975                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
6976                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
6977                         if (items->mask != NULL &&
6978                             ((const struct rte_flow_item_ipv6_frag_ext *)
6979                              items->mask)->hdr.next_header) {
6980                                 next_protocol =
6981                                 ((const struct rte_flow_item_ipv6_frag_ext *)
6982                                  items->spec)->hdr.next_header;
6983                                 next_protocol &=
6984                                 ((const struct rte_flow_item_ipv6_frag_ext *)
6985                                  items->mask)->hdr.next_header;
6986                         } else {
6987                                 /* Reset for inner layer. */
6988                                 next_protocol = 0xff;
6989                         }
6990                         break;
6991                 case RTE_FLOW_ITEM_TYPE_TCP:
6992                         ret = mlx5_flow_validate_item_tcp
6993                                                 (items, item_flags,
6994                                                  next_protocol,
6995                                                  &nic_tcp_mask,
6996                                                  error);
6997                         if (ret < 0)
6998                                 return ret;
6999                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
7000                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
7001                         break;
7002                 case RTE_FLOW_ITEM_TYPE_UDP:
7003                         ret = mlx5_flow_validate_item_udp(items, item_flags,
7004                                                           next_protocol,
7005                                                           error);
7006                         const struct rte_flow_item_udp *spec = items->spec;
7007                         const struct rte_flow_item_udp *mask = items->mask;
7008                         if (!mask)
7009                                 mask = &rte_flow_item_udp_mask;
7010                         if (spec != NULL)
7011                                 udp_dport = rte_be_to_cpu_16
7012                                                 (spec->hdr.dst_port &
7013                                                  mask->hdr.dst_port);
7014                         if (ret < 0)
7015                                 return ret;
7016                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
7017                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
7018                         break;
7019                 case RTE_FLOW_ITEM_TYPE_GRE:
7020                         ret = mlx5_flow_validate_item_gre(items, item_flags,
7021                                                           next_protocol, error);
7022                         if (ret < 0)
7023                                 return ret;
7024                         gre_item = items;
7025                         last_item = MLX5_FLOW_LAYER_GRE;
7026                         break;
7027                 case RTE_FLOW_ITEM_TYPE_NVGRE:
7028                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
7029                                                             next_protocol,
7030                                                             error);
7031                         if (ret < 0)
7032                                 return ret;
7033                         last_item = MLX5_FLOW_LAYER_NVGRE;
7034                         break;
7035                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
7036                         ret = mlx5_flow_validate_item_gre_key
7037                                 (items, item_flags, gre_item, error);
7038                         if (ret < 0)
7039                                 return ret;
7040                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
7041                         break;
7042                 case RTE_FLOW_ITEM_TYPE_VXLAN:
7043                         ret = mlx5_flow_validate_item_vxlan(dev, udp_dport,
7044                                                             items, item_flags,
7045                                                             attr, error);
7046                         if (ret < 0)
7047                                 return ret;
7048                         last_item = MLX5_FLOW_LAYER_VXLAN;
7049                         break;
7050                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
7051                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
7052                                                                 item_flags, dev,
7053                                                                 error);
7054                         if (ret < 0)
7055                                 return ret;
7056                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
7057                         break;
7058                 case RTE_FLOW_ITEM_TYPE_GENEVE:
7059                         ret = mlx5_flow_validate_item_geneve(items,
7060                                                              item_flags, dev,
7061                                                              error);
7062                         if (ret < 0)
7063                                 return ret;
7064                         geneve_item = items;
7065                         last_item = MLX5_FLOW_LAYER_GENEVE;
7066                         break;
7067                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
7068                         ret = mlx5_flow_validate_item_geneve_opt(items,
7069                                                                  last_item,
7070                                                                  geneve_item,
7071                                                                  dev,
7072                                                                  error);
7073                         if (ret < 0)
7074                                 return ret;
7075                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
7076                         break;
7077                 case RTE_FLOW_ITEM_TYPE_MPLS:
7078                         ret = mlx5_flow_validate_item_mpls(dev, items,
7079                                                            item_flags,
7080                                                            last_item, error);
7081                         if (ret < 0)
7082                                 return ret;
7083                         last_item = MLX5_FLOW_LAYER_MPLS;
7084                         break;
7085
7086                 case RTE_FLOW_ITEM_TYPE_MARK:
7087                         ret = flow_dv_validate_item_mark(dev, items, attr,
7088                                                          error);
7089                         if (ret < 0)
7090                                 return ret;
7091                         last_item = MLX5_FLOW_ITEM_MARK;
7092                         break;
7093                 case RTE_FLOW_ITEM_TYPE_META:
7094                         ret = flow_dv_validate_item_meta(dev, items, attr,
7095                                                          error);
7096                         if (ret < 0)
7097                                 return ret;
7098                         last_item = MLX5_FLOW_ITEM_METADATA;
7099                         break;
7100                 case RTE_FLOW_ITEM_TYPE_ICMP:
7101                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
7102                                                            next_protocol,
7103                                                            error);
7104                         if (ret < 0)
7105                                 return ret;
7106                         last_item = MLX5_FLOW_LAYER_ICMP;
7107                         break;
7108                 case RTE_FLOW_ITEM_TYPE_ICMP6:
7109                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
7110                                                             next_protocol,
7111                                                             error);
7112                         if (ret < 0)
7113                                 return ret;
7114                         item_ipv6_proto = IPPROTO_ICMPV6;
7115                         last_item = MLX5_FLOW_LAYER_ICMP6;
7116                         break;
7117                 case RTE_FLOW_ITEM_TYPE_TAG:
7118                         ret = flow_dv_validate_item_tag(dev, items,
7119                                                         attr, error);
7120                         if (ret < 0)
7121                                 return ret;
7122                         last_item = MLX5_FLOW_ITEM_TAG;
7123                         break;
7124                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
7125                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
7126                         break;
7127                 case RTE_FLOW_ITEM_TYPE_GTP:
7128                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
7129                                                         error);
7130                         if (ret < 0)
7131                                 return ret;
7132                         gtp_item = items;
7133                         last_item = MLX5_FLOW_LAYER_GTP;
7134                         break;
7135                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
7136                         ret = flow_dv_validate_item_gtp_psc(items, last_item,
7137                                                             gtp_item, attr,
7138                                                             error);
7139                         if (ret < 0)
7140                                 return ret;
7141                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
7142                         break;
7143                 case RTE_FLOW_ITEM_TYPE_ECPRI:
7144                         /* Capacity will be checked in the translate stage. */
7145                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
7146                                                             last_item,
7147                                                             ether_type,
7148                                                             &nic_ecpri_mask,
7149                                                             error);
7150                         if (ret < 0)
7151                                 return ret;
7152                         last_item = MLX5_FLOW_LAYER_ECPRI;
7153                         break;
7154                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
7155                         if (item_flags & MLX5_FLOW_ITEM_INTEGRITY)
7156                                 return rte_flow_error_set
7157                                         (error, ENOTSUP,
7158                                          RTE_FLOW_ERROR_TYPE_ITEM,
7159                                          NULL, "multiple integrity items not supported");
7160                         ret = flow_dv_validate_item_integrity(dev, rule_items,
7161                                                               items, error);
7162                         if (ret < 0)
7163                                 return ret;
7164                         last_item = MLX5_FLOW_ITEM_INTEGRITY;
7165                         break;
7166                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
7167                         ret = flow_dv_validate_item_aso_ct(dev, items,
7168                                                            &item_flags, error);
7169                         if (ret < 0)
7170                                 return ret;
7171                         break;
7172                 case MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL:
7173                         /* tunnel offload item was processed before
7174                          * list it here as a supported type
7175                          */
7176                         break;
7177                 default:
7178                         return rte_flow_error_set(error, ENOTSUP,
7179                                                   RTE_FLOW_ERROR_TYPE_ITEM,
7180                                                   NULL, "item not supported");
7181                 }
7182                 item_flags |= last_item;
7183         }
7184         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
7185                 int type = actions->type;
7186                 bool shared_count = false;
7187
7188                 if (!mlx5_flow_os_action_supported(type))
7189                         return rte_flow_error_set(error, ENOTSUP,
7190                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7191                                                   actions,
7192                                                   "action not supported");
7193                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
7194                         return rte_flow_error_set(error, ENOTSUP,
7195                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7196                                                   actions, "too many actions");
7197                 if (action_flags &
7198                         MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
7199                         return rte_flow_error_set(error, ENOTSUP,
7200                                 RTE_FLOW_ERROR_TYPE_ACTION,
7201                                 NULL, "meter action with policy "
7202                                 "must be the last action");
7203                 switch (type) {
7204                 case RTE_FLOW_ACTION_TYPE_VOID:
7205                         break;
7206                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
7207                 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
7208                         ret = flow_dv_validate_action_port_id(dev,
7209                                                               action_flags,
7210                                                               actions,
7211                                                               attr,
7212                                                               error);
7213                         if (ret)
7214                                 return ret;
7215                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
7216                         ++actions_n;
7217                         break;
7218                 case RTE_FLOW_ACTION_TYPE_FLAG:
7219                         ret = flow_dv_validate_action_flag(dev, action_flags,
7220                                                            attr, error);
7221                         if (ret < 0)
7222                                 return ret;
7223                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7224                                 /* Count all modify-header actions as one. */
7225                                 if (!(action_flags &
7226                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7227                                         ++actions_n;
7228                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
7229                                                 MLX5_FLOW_ACTION_MARK_EXT;
7230                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7231                                         modify_after_mirror = 1;
7232
7233                         } else {
7234                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
7235                                 ++actions_n;
7236                         }
7237                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7238                         break;
7239                 case RTE_FLOW_ACTION_TYPE_MARK:
7240                         ret = flow_dv_validate_action_mark(dev, actions,
7241                                                            action_flags,
7242                                                            attr, error);
7243                         if (ret < 0)
7244                                 return ret;
7245                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7246                                 /* Count all modify-header actions as one. */
7247                                 if (!(action_flags &
7248                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7249                                         ++actions_n;
7250                                 action_flags |= MLX5_FLOW_ACTION_MARK |
7251                                                 MLX5_FLOW_ACTION_MARK_EXT;
7252                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7253                                         modify_after_mirror = 1;
7254                         } else {
7255                                 action_flags |= MLX5_FLOW_ACTION_MARK;
7256                                 ++actions_n;
7257                         }
7258                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7259                         break;
7260                 case RTE_FLOW_ACTION_TYPE_SET_META:
7261                         ret = flow_dv_validate_action_set_meta(dev, actions,
7262                                                                action_flags,
7263                                                                attr, error);
7264                         if (ret < 0)
7265                                 return ret;
7266                         /* Count all modify-header actions as one action. */
7267                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7268                                 ++actions_n;
7269                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7270                                 modify_after_mirror = 1;
7271                         action_flags |= MLX5_FLOW_ACTION_SET_META;
7272                         rw_act_num += MLX5_ACT_NUM_SET_META;
7273                         break;
7274                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
7275                         ret = flow_dv_validate_action_set_tag(dev, actions,
7276                                                               action_flags,
7277                                                               attr, error);
7278                         if (ret < 0)
7279                                 return ret;
7280                         /* Count all modify-header actions as one action. */
7281                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7282                                 ++actions_n;
7283                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7284                                 modify_after_mirror = 1;
7285                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
7286                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7287                         break;
7288                 case RTE_FLOW_ACTION_TYPE_DROP:
7289                         ret = mlx5_flow_validate_action_drop(action_flags,
7290                                                              attr, error);
7291                         if (ret < 0)
7292                                 return ret;
7293                         action_flags |= MLX5_FLOW_ACTION_DROP;
7294                         ++actions_n;
7295                         break;
7296                 case RTE_FLOW_ACTION_TYPE_QUEUE:
7297                         ret = mlx5_flow_validate_action_queue(actions,
7298                                                               action_flags, dev,
7299                                                               attr, error);
7300                         if (ret < 0)
7301                                 return ret;
7302                         queue_index = ((const struct rte_flow_action_queue *)
7303                                                         (actions->conf))->index;
7304                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
7305                         ++actions_n;
7306                         break;
7307                 case RTE_FLOW_ACTION_TYPE_RSS:
7308                         rss = actions->conf;
7309                         ret = mlx5_flow_validate_action_rss(actions,
7310                                                             action_flags, dev,
7311                                                             attr, item_flags,
7312                                                             error);
7313                         if (ret < 0)
7314                                 return ret;
7315                         if (rss && sample_rss &&
7316                             (sample_rss->level != rss->level ||
7317                             sample_rss->types != rss->types))
7318                                 return rte_flow_error_set(error, ENOTSUP,
7319                                         RTE_FLOW_ERROR_TYPE_ACTION,
7320                                         NULL,
7321                                         "Can't use the different RSS types "
7322                                         "or level in the same flow");
7323                         if (rss != NULL && rss->queue_num)
7324                                 queue_index = rss->queue[0];
7325                         action_flags |= MLX5_FLOW_ACTION_RSS;
7326                         ++actions_n;
7327                         break;
7328                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
7329                         ret =
7330                         mlx5_flow_validate_action_default_miss(action_flags,
7331                                         attr, error);
7332                         if (ret < 0)
7333                                 return ret;
7334                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
7335                         ++actions_n;
7336                         break;
7337                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
7338                         shared_count = true;
7339                         /* fall-through. */
7340                 case RTE_FLOW_ACTION_TYPE_COUNT:
7341                         ret = flow_dv_validate_action_count(dev, shared_count,
7342                                                             action_flags,
7343                                                             error);
7344                         if (ret < 0)
7345                                 return ret;
7346                         action_flags |= MLX5_FLOW_ACTION_COUNT;
7347                         ++actions_n;
7348                         break;
7349                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
7350                         if (flow_dv_validate_action_pop_vlan(dev,
7351                                                              action_flags,
7352                                                              actions,
7353                                                              item_flags, attr,
7354                                                              error))
7355                                 return -rte_errno;
7356                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7357                                 modify_after_mirror = 1;
7358                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
7359                         ++actions_n;
7360                         break;
7361                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
7362                         ret = flow_dv_validate_action_push_vlan(dev,
7363                                                                 action_flags,
7364                                                                 vlan_m,
7365                                                                 actions, attr,
7366                                                                 error);
7367                         if (ret < 0)
7368                                 return ret;
7369                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7370                                 modify_after_mirror = 1;
7371                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
7372                         ++actions_n;
7373                         break;
7374                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
7375                         ret = flow_dv_validate_action_set_vlan_pcp
7376                                                 (action_flags, actions, error);
7377                         if (ret < 0)
7378                                 return ret;
7379                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7380                                 modify_after_mirror = 1;
7381                         /* Count PCP with push_vlan command. */
7382                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
7383                         break;
7384                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
7385                         ret = flow_dv_validate_action_set_vlan_vid
7386                                                 (item_flags, action_flags,
7387                                                  actions, error);
7388                         if (ret < 0)
7389                                 return ret;
7390                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7391                                 modify_after_mirror = 1;
7392                         /* Count VID with push_vlan command. */
7393                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
7394                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
7395                         break;
7396                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
7397                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
7398                         ret = flow_dv_validate_action_l2_encap(dev,
7399                                                                action_flags,
7400                                                                actions, attr,
7401                                                                error);
7402                         if (ret < 0)
7403                                 return ret;
7404                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
7405                         ++actions_n;
7406                         break;
7407                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
7408                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
7409                         ret = flow_dv_validate_action_decap(dev, action_flags,
7410                                                             actions, item_flags,
7411                                                             attr, error);
7412                         if (ret < 0)
7413                                 return ret;
7414                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7415                                 modify_after_mirror = 1;
7416                         action_flags |= MLX5_FLOW_ACTION_DECAP;
7417                         ++actions_n;
7418                         break;
7419                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
7420                         ret = flow_dv_validate_action_raw_encap_decap
7421                                 (dev, NULL, actions->conf, attr, &action_flags,
7422                                  &actions_n, actions, item_flags, error);
7423                         if (ret < 0)
7424                                 return ret;
7425                         break;
7426                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
7427                         decap = actions->conf;
7428                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
7429                                 ;
7430                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
7431                                 encap = NULL;
7432                                 actions--;
7433                         } else {
7434                                 encap = actions->conf;
7435                         }
7436                         ret = flow_dv_validate_action_raw_encap_decap
7437                                            (dev,
7438                                             decap ? decap : &empty_decap, encap,
7439                                             attr, &action_flags, &actions_n,
7440                                             actions, item_flags, error);
7441                         if (ret < 0)
7442                                 return ret;
7443                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7444                             (action_flags & MLX5_FLOW_ACTION_DECAP))
7445                                 modify_after_mirror = 1;
7446                         break;
7447                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
7448                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
7449                         ret = flow_dv_validate_action_modify_mac(action_flags,
7450                                                                  actions,
7451                                                                  item_flags,
7452                                                                  error);
7453                         if (ret < 0)
7454                                 return ret;
7455                         /* Count all modify-header actions as one action. */
7456                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7457                                 ++actions_n;
7458                         action_flags |= actions->type ==
7459                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
7460                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
7461                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
7462                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7463                                 modify_after_mirror = 1;
7464                         /*
7465                          * Even if the source and destination MAC addresses have
7466                          * overlap in the header with 4B alignment, the convert
7467                          * function will handle them separately and 4 SW actions
7468                          * will be created. And 2 actions will be added each
7469                          * time no matter how many bytes of address will be set.
7470                          */
7471                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
7472                         break;
7473                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
7474                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
7475                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
7476                                                                   actions,
7477                                                                   item_flags,
7478                                                                   error);
7479                         if (ret < 0)
7480                                 return ret;
7481                         /* Count all modify-header actions as one action. */
7482                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7483                                 ++actions_n;
7484                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7485                                 modify_after_mirror = 1;
7486                         action_flags |= actions->type ==
7487                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
7488                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
7489                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
7490                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
7491                         break;
7492                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
7493                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
7494                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
7495                                                                   actions,
7496                                                                   item_flags,
7497                                                                   error);
7498                         if (ret < 0)
7499                                 return ret;
7500                         if (item_ipv6_proto == IPPROTO_ICMPV6)
7501                                 return rte_flow_error_set(error, ENOTSUP,
7502                                         RTE_FLOW_ERROR_TYPE_ACTION,
7503                                         actions,
7504                                         "Can't change header "
7505                                         "with ICMPv6 proto");
7506                         /* Count all modify-header actions as one action. */
7507                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7508                                 ++actions_n;
7509                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7510                                 modify_after_mirror = 1;
7511                         action_flags |= actions->type ==
7512                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
7513                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
7514                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
7515                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
7516                         break;
7517                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
7518                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
7519                         ret = flow_dv_validate_action_modify_tp(action_flags,
7520                                                                 actions,
7521                                                                 item_flags,
7522                                                                 error);
7523                         if (ret < 0)
7524                                 return ret;
7525                         /* Count all modify-header actions as one action. */
7526                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7527                                 ++actions_n;
7528                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7529                                 modify_after_mirror = 1;
7530                         action_flags |= actions->type ==
7531                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
7532                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
7533                                                 MLX5_FLOW_ACTION_SET_TP_DST;
7534                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
7535                         break;
7536                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
7537                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
7538                         ret = flow_dv_validate_action_modify_ttl(action_flags,
7539                                                                  actions,
7540                                                                  item_flags,
7541                                                                  error);
7542                         if (ret < 0)
7543                                 return ret;
7544                         /* Count all modify-header actions as one action. */
7545                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7546                                 ++actions_n;
7547                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7548                                 modify_after_mirror = 1;
7549                         action_flags |= actions->type ==
7550                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
7551                                                 MLX5_FLOW_ACTION_SET_TTL :
7552                                                 MLX5_FLOW_ACTION_DEC_TTL;
7553                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
7554                         break;
7555                 case RTE_FLOW_ACTION_TYPE_JUMP:
7556                         ret = flow_dv_validate_action_jump(dev, tunnel, actions,
7557                                                            action_flags,
7558                                                            attr, external,
7559                                                            error);
7560                         if (ret)
7561                                 return ret;
7562                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7563                             fdb_mirror_limit)
7564                                 return rte_flow_error_set(error, EINVAL,
7565                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7566                                                   NULL,
7567                                                   "sample and jump action combination is not supported");
7568                         ++actions_n;
7569                         action_flags |= MLX5_FLOW_ACTION_JUMP;
7570                         break;
7571                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
7572                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
7573                         ret = flow_dv_validate_action_modify_tcp_seq
7574                                                                 (action_flags,
7575                                                                  actions,
7576                                                                  item_flags,
7577                                                                  error);
7578                         if (ret < 0)
7579                                 return ret;
7580                         /* Count all modify-header actions as one action. */
7581                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7582                                 ++actions_n;
7583                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7584                                 modify_after_mirror = 1;
7585                         action_flags |= actions->type ==
7586                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
7587                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
7588                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
7589                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
7590                         break;
7591                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
7592                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
7593                         ret = flow_dv_validate_action_modify_tcp_ack
7594                                                                 (action_flags,
7595                                                                  actions,
7596                                                                  item_flags,
7597                                                                  error);
7598                         if (ret < 0)
7599                                 return ret;
7600                         /* Count all modify-header actions as one action. */
7601                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7602                                 ++actions_n;
7603                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7604                                 modify_after_mirror = 1;
7605                         action_flags |= actions->type ==
7606                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
7607                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
7608                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
7609                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
7610                         break;
7611                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
7612                         break;
7613                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
7614                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
7615                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7616                         break;
7617                 case RTE_FLOW_ACTION_TYPE_METER:
7618                         ret = mlx5_flow_validate_action_meter(dev,
7619                                                               action_flags,
7620                                                               actions, attr,
7621                                                               port_id_item,
7622                                                               &def_policy,
7623                                                               error);
7624                         if (ret < 0)
7625                                 return ret;
7626                         action_flags |= MLX5_FLOW_ACTION_METER;
7627                         if (!def_policy)
7628                                 action_flags |=
7629                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
7630                         ++actions_n;
7631                         /* Meter action will add one more TAG action. */
7632                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7633                         break;
7634                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
7635                         if (!attr->transfer && !attr->group)
7636                                 return rte_flow_error_set(error, ENOTSUP,
7637                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7638                                                                            NULL,
7639                           "Shared ASO age action is not supported for group 0");
7640                         if (action_flags & MLX5_FLOW_ACTION_AGE)
7641                                 return rte_flow_error_set
7642                                                   (error, EINVAL,
7643                                                    RTE_FLOW_ERROR_TYPE_ACTION,
7644                                                    NULL,
7645                                                    "duplicate age actions set");
7646                         action_flags |= MLX5_FLOW_ACTION_AGE;
7647                         ++actions_n;
7648                         break;
7649                 case RTE_FLOW_ACTION_TYPE_AGE:
7650                         ret = flow_dv_validate_action_age(action_flags,
7651                                                           actions, dev,
7652                                                           error);
7653                         if (ret < 0)
7654                                 return ret;
7655                         /*
7656                          * Validate the regular AGE action (using counter)
7657                          * mutual exclusion with share counter actions.
7658                          */
7659                         if (!priv->sh->flow_hit_aso_en) {
7660                                 if (shared_count)
7661                                         return rte_flow_error_set
7662                                                 (error, EINVAL,
7663                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7664                                                 NULL,
7665                                                 "old age and shared count combination is not supported");
7666                                 if (sample_count)
7667                                         return rte_flow_error_set
7668                                                 (error, EINVAL,
7669                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7670                                                 NULL,
7671                                                 "old age action and count must be in the same sub flow");
7672                         }
7673                         action_flags |= MLX5_FLOW_ACTION_AGE;
7674                         ++actions_n;
7675                         break;
7676                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
7677                         ret = flow_dv_validate_action_modify_ipv4_dscp
7678                                                          (action_flags,
7679                                                           actions,
7680                                                           item_flags,
7681                                                           error);
7682                         if (ret < 0)
7683                                 return ret;
7684                         /* Count all modify-header actions as one action. */
7685                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7686                                 ++actions_n;
7687                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7688                                 modify_after_mirror = 1;
7689                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
7690                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7691                         break;
7692                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
7693                         ret = flow_dv_validate_action_modify_ipv6_dscp
7694                                                                 (action_flags,
7695                                                                  actions,
7696                                                                  item_flags,
7697                                                                  error);
7698                         if (ret < 0)
7699                                 return ret;
7700                         /* Count all modify-header actions as one action. */
7701                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7702                                 ++actions_n;
7703                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7704                                 modify_after_mirror = 1;
7705                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
7706                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7707                         break;
7708                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
7709                         ret = flow_dv_validate_action_sample(&action_flags,
7710                                                              actions, dev,
7711                                                              attr, item_flags,
7712                                                              rss, &sample_rss,
7713                                                              &sample_count,
7714                                                              &fdb_mirror_limit,
7715                                                              error);
7716                         if (ret < 0)
7717                                 return ret;
7718                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
7719                         ++actions_n;
7720                         break;
7721                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
7722                         ret = flow_dv_validate_action_modify_field(dev,
7723                                                                    action_flags,
7724                                                                    actions,
7725                                                                    attr,
7726                                                                    error);
7727                         if (ret < 0)
7728                                 return ret;
7729                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7730                                 modify_after_mirror = 1;
7731                         /* Count all modify-header actions as one action. */
7732                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7733                                 ++actions_n;
7734                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
7735                         rw_act_num += ret;
7736                         break;
7737                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
7738                         ret = flow_dv_validate_action_aso_ct(dev, action_flags,
7739                                                              item_flags, attr,
7740                                                              error);
7741                         if (ret < 0)
7742                                 return ret;
7743                         action_flags |= MLX5_FLOW_ACTION_CT;
7744                         break;
7745                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
7746                         /* tunnel offload action was processed before
7747                          * list it here as a supported type
7748                          */
7749                         break;
7750                 default:
7751                         return rte_flow_error_set(error, ENOTSUP,
7752                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7753                                                   actions,
7754                                                   "action not supported");
7755                 }
7756         }
7757         /*
7758          * Validate actions in flow rules
7759          * - Explicit decap action is prohibited by the tunnel offload API.
7760          * - Drop action in tunnel steer rule is prohibited by the API.
7761          * - Application cannot use MARK action because it's value can mask
7762          *   tunnel default miss nitification.
7763          * - JUMP in tunnel match rule has no support in current PMD
7764          *   implementation.
7765          * - TAG & META are reserved for future uses.
7766          */
7767         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_SET) {
7768                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_DECAP    |
7769                                             MLX5_FLOW_ACTION_MARK     |
7770                                             MLX5_FLOW_ACTION_SET_TAG  |
7771                                             MLX5_FLOW_ACTION_SET_META |
7772                                             MLX5_FLOW_ACTION_DROP;
7773
7774                 if (action_flags & bad_actions_mask)
7775                         return rte_flow_error_set
7776                                         (error, EINVAL,
7777                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7778                                         "Invalid RTE action in tunnel "
7779                                         "set decap rule");
7780                 if (!(action_flags & MLX5_FLOW_ACTION_JUMP))
7781                         return rte_flow_error_set
7782                                         (error, EINVAL,
7783                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7784                                         "tunnel set decap rule must terminate "
7785                                         "with JUMP");
7786                 if (!attr->ingress)
7787                         return rte_flow_error_set
7788                                         (error, EINVAL,
7789                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7790                                         "tunnel flows for ingress traffic only");
7791         }
7792         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_MATCH) {
7793                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_JUMP    |
7794                                             MLX5_FLOW_ACTION_MARK    |
7795                                             MLX5_FLOW_ACTION_SET_TAG |
7796                                             MLX5_FLOW_ACTION_SET_META;
7797
7798                 if (action_flags & bad_actions_mask)
7799                         return rte_flow_error_set
7800                                         (error, EINVAL,
7801                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7802                                         "Invalid RTE action in tunnel "
7803                                         "set match rule");
7804         }
7805         /*
7806          * Validate the drop action mutual exclusion with other actions.
7807          * Drop action is mutually-exclusive with any other action, except for
7808          * Count action.
7809          * Drop action compatibility with tunnel offload was already validated.
7810          */
7811         if (action_flags & (MLX5_FLOW_ACTION_TUNNEL_MATCH |
7812                             MLX5_FLOW_ACTION_TUNNEL_MATCH));
7813         else if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
7814             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
7815                 return rte_flow_error_set(error, EINVAL,
7816                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7817                                           "Drop action is mutually-exclusive "
7818                                           "with any other action, except for "
7819                                           "Count action");
7820         /* Eswitch has few restrictions on using items and actions */
7821         if (attr->transfer) {
7822                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7823                     action_flags & MLX5_FLOW_ACTION_FLAG)
7824                         return rte_flow_error_set(error, ENOTSUP,
7825                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7826                                                   NULL,
7827                                                   "unsupported action FLAG");
7828                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7829                     action_flags & MLX5_FLOW_ACTION_MARK)
7830                         return rte_flow_error_set(error, ENOTSUP,
7831                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7832                                                   NULL,
7833                                                   "unsupported action MARK");
7834                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
7835                         return rte_flow_error_set(error, ENOTSUP,
7836                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7837                                                   NULL,
7838                                                   "unsupported action QUEUE");
7839                 if (action_flags & MLX5_FLOW_ACTION_RSS)
7840                         return rte_flow_error_set(error, ENOTSUP,
7841                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7842                                                   NULL,
7843                                                   "unsupported action RSS");
7844                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
7845                         return rte_flow_error_set(error, EINVAL,
7846                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7847                                                   actions,
7848                                                   "no fate action is found");
7849         } else {
7850                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
7851                         return rte_flow_error_set(error, EINVAL,
7852                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7853                                                   actions,
7854                                                   "no fate action is found");
7855         }
7856         /*
7857          * Continue validation for Xcap and VLAN actions.
7858          * If hairpin is working in explicit TX rule mode, there is no actions
7859          * splitting and the validation of hairpin ingress flow should be the
7860          * same as other standard flows.
7861          */
7862         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
7863                              MLX5_FLOW_VLAN_ACTIONS)) &&
7864             (queue_index == 0xFFFF ||
7865              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN ||
7866              ((conf = mlx5_rxq_get_hairpin_conf(dev, queue_index)) != NULL &&
7867              conf->tx_explicit != 0))) {
7868                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
7869                     MLX5_FLOW_XCAP_ACTIONS)
7870                         return rte_flow_error_set(error, ENOTSUP,
7871                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7872                                                   NULL, "encap and decap "
7873                                                   "combination aren't supported");
7874                 if (!attr->transfer && attr->ingress) {
7875                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
7876                                 return rte_flow_error_set
7877                                                 (error, ENOTSUP,
7878                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7879                                                  NULL, "encap is not supported"
7880                                                  " for ingress traffic");
7881                         else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
7882                                 return rte_flow_error_set
7883                                                 (error, ENOTSUP,
7884                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7885                                                  NULL, "push VLAN action not "
7886                                                  "supported for ingress");
7887                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
7888                                         MLX5_FLOW_VLAN_ACTIONS)
7889                                 return rte_flow_error_set
7890                                                 (error, ENOTSUP,
7891                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7892                                                  NULL, "no support for "
7893                                                  "multiple VLAN actions");
7894                 }
7895         }
7896         if (action_flags & MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY) {
7897                 if ((action_flags & (MLX5_FLOW_FATE_ACTIONS &
7898                         ~MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)) &&
7899                         attr->ingress)
7900                         return rte_flow_error_set
7901                                 (error, ENOTSUP,
7902                                 RTE_FLOW_ERROR_TYPE_ACTION,
7903                                 NULL, "fate action not supported for "
7904                                 "meter with policy");
7905                 if (attr->egress) {
7906                         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
7907                                 return rte_flow_error_set
7908                                         (error, ENOTSUP,
7909                                         RTE_FLOW_ERROR_TYPE_ACTION,
7910                                         NULL, "modify header action in egress "
7911                                         "cannot be done before meter action");
7912                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
7913                                 return rte_flow_error_set
7914                                         (error, ENOTSUP,
7915                                         RTE_FLOW_ERROR_TYPE_ACTION,
7916                                         NULL, "encap action in egress "
7917                                         "cannot be done before meter action");
7918                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
7919                                 return rte_flow_error_set
7920                                         (error, ENOTSUP,
7921                                         RTE_FLOW_ERROR_TYPE_ACTION,
7922                                         NULL, "push vlan action in egress "
7923                                         "cannot be done before meter action");
7924                 }
7925         }
7926         /*
7927          * Hairpin flow will add one more TAG action in TX implicit mode.
7928          * In TX explicit mode, there will be no hairpin flow ID.
7929          */
7930         if (hairpin > 0)
7931                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
7932         /* extra metadata enabled: one more TAG action will be add. */
7933         if (dev_conf->dv_flow_en &&
7934             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
7935             mlx5_flow_ext_mreg_supported(dev))
7936                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
7937         if (rw_act_num >
7938                         flow_dv_modify_hdr_action_max(dev, is_root)) {
7939                 return rte_flow_error_set(error, ENOTSUP,
7940                                           RTE_FLOW_ERROR_TYPE_ACTION,
7941                                           NULL, "too many header modify"
7942                                           " actions to support");
7943         }
7944         /* Eswitch egress mirror and modify flow has limitation on CX5 */
7945         if (fdb_mirror_limit && modify_after_mirror)
7946                 return rte_flow_error_set(error, EINVAL,
7947                                 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7948                                 "sample before modify action is not supported");
7949         return 0;
7950 }
7951
7952 /**
7953  * Internal preparation function. Allocates the DV flow size,
7954  * this size is constant.
7955  *
7956  * @param[in] dev
7957  *   Pointer to the rte_eth_dev structure.
7958  * @param[in] attr
7959  *   Pointer to the flow attributes.
7960  * @param[in] items
7961  *   Pointer to the list of items.
7962  * @param[in] actions
7963  *   Pointer to the list of actions.
7964  * @param[out] error
7965  *   Pointer to the error structure.
7966  *
7967  * @return
7968  *   Pointer to mlx5_flow object on success,
7969  *   otherwise NULL and rte_errno is set.
7970  */
7971 static struct mlx5_flow *
7972 flow_dv_prepare(struct rte_eth_dev *dev,
7973                 const struct rte_flow_attr *attr __rte_unused,
7974                 const struct rte_flow_item items[] __rte_unused,
7975                 const struct rte_flow_action actions[] __rte_unused,
7976                 struct rte_flow_error *error)
7977 {
7978         uint32_t handle_idx = 0;
7979         struct mlx5_flow *dev_flow;
7980         struct mlx5_flow_handle *dev_handle;
7981         struct mlx5_priv *priv = dev->data->dev_private;
7982         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
7983
7984         MLX5_ASSERT(wks);
7985         wks->skip_matcher_reg = 0;
7986         wks->policy = NULL;
7987         wks->final_policy = NULL;
7988         /* In case of corrupting the memory. */
7989         if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
7990                 rte_flow_error_set(error, ENOSPC,
7991                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7992                                    "not free temporary device flow");
7993                 return NULL;
7994         }
7995         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
7996                                    &handle_idx);
7997         if (!dev_handle) {
7998                 rte_flow_error_set(error, ENOMEM,
7999                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8000                                    "not enough memory to create flow handle");
8001                 return NULL;
8002         }
8003         MLX5_ASSERT(wks->flow_idx < RTE_DIM(wks->flows));
8004         dev_flow = &wks->flows[wks->flow_idx++];
8005         memset(dev_flow, 0, sizeof(*dev_flow));
8006         dev_flow->handle = dev_handle;
8007         dev_flow->handle_idx = handle_idx;
8008         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
8009         dev_flow->ingress = attr->ingress;
8010         dev_flow->dv.transfer = attr->transfer;
8011         return dev_flow;
8012 }
8013
8014 #ifdef RTE_LIBRTE_MLX5_DEBUG
8015 /**
8016  * Sanity check for match mask and value. Similar to check_valid_spec() in
8017  * kernel driver. If unmasked bit is present in value, it returns failure.
8018  *
8019  * @param match_mask
8020  *   pointer to match mask buffer.
8021  * @param match_value
8022  *   pointer to match value buffer.
8023  *
8024  * @return
8025  *   0 if valid, -EINVAL otherwise.
8026  */
8027 static int
8028 flow_dv_check_valid_spec(void *match_mask, void *match_value)
8029 {
8030         uint8_t *m = match_mask;
8031         uint8_t *v = match_value;
8032         unsigned int i;
8033
8034         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
8035                 if (v[i] & ~m[i]) {
8036                         DRV_LOG(ERR,
8037                                 "match_value differs from match_criteria"
8038                                 " %p[%u] != %p[%u]",
8039                                 match_value, i, match_mask, i);
8040                         return -EINVAL;
8041                 }
8042         }
8043         return 0;
8044 }
8045 #endif
8046
8047 /**
8048  * Add match of ip_version.
8049  *
8050  * @param[in] group
8051  *   Flow group.
8052  * @param[in] headers_v
8053  *   Values header pointer.
8054  * @param[in] headers_m
8055  *   Masks header pointer.
8056  * @param[in] ip_version
8057  *   The IP version to set.
8058  */
8059 static inline void
8060 flow_dv_set_match_ip_version(uint32_t group,
8061                              void *headers_v,
8062                              void *headers_m,
8063                              uint8_t ip_version)
8064 {
8065         if (group == 0)
8066                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
8067         else
8068                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
8069                          ip_version);
8070         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
8071         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
8072         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
8073 }
8074
8075 /**
8076  * Add Ethernet item to matcher and to the value.
8077  *
8078  * @param[in, out] matcher
8079  *   Flow matcher.
8080  * @param[in, out] key
8081  *   Flow matcher value.
8082  * @param[in] item
8083  *   Flow pattern to translate.
8084  * @param[in] inner
8085  *   Item is inner pattern.
8086  */
8087 static void
8088 flow_dv_translate_item_eth(void *matcher, void *key,
8089                            const struct rte_flow_item *item, int inner,
8090                            uint32_t group)
8091 {
8092         const struct rte_flow_item_eth *eth_m = item->mask;
8093         const struct rte_flow_item_eth *eth_v = item->spec;
8094         const struct rte_flow_item_eth nic_mask = {
8095                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
8096                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
8097                 .type = RTE_BE16(0xffff),
8098                 .has_vlan = 0,
8099         };
8100         void *hdrs_m;
8101         void *hdrs_v;
8102         char *l24_v;
8103         unsigned int i;
8104
8105         if (!eth_v)
8106                 return;
8107         if (!eth_m)
8108                 eth_m = &nic_mask;
8109         if (inner) {
8110                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8111                                          inner_headers);
8112                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8113         } else {
8114                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8115                                          outer_headers);
8116                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8117         }
8118         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, dmac_47_16),
8119                &eth_m->dst, sizeof(eth_m->dst));
8120         /* The value must be in the range of the mask. */
8121         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, dmac_47_16);
8122         for (i = 0; i < sizeof(eth_m->dst); ++i)
8123                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
8124         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, smac_47_16),
8125                &eth_m->src, sizeof(eth_m->src));
8126         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, smac_47_16);
8127         /* The value must be in the range of the mask. */
8128         for (i = 0; i < sizeof(eth_m->dst); ++i)
8129                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
8130         /*
8131          * HW supports match on one Ethertype, the Ethertype following the last
8132          * VLAN tag of the packet (see PRM).
8133          * Set match on ethertype only if ETH header is not followed by VLAN.
8134          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8135          * ethertype, and use ip_version field instead.
8136          * eCPRI over Ether layer will use type value 0xAEFE.
8137          */
8138         if (eth_m->type == 0xFFFF) {
8139                 /* Set cvlan_tag mask for any single\multi\un-tagged case. */
8140                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8141                 switch (eth_v->type) {
8142                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8143                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8144                         return;
8145                 case RTE_BE16(RTE_ETHER_TYPE_QINQ):
8146                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8147                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8148                         return;
8149                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8150                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8151                         return;
8152                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8153                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8154                         return;
8155                 default:
8156                         break;
8157                 }
8158         }
8159         if (eth_m->has_vlan) {
8160                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8161                 if (eth_v->has_vlan) {
8162                         /*
8163                          * Here, when also has_more_vlan field in VLAN item is
8164                          * not set, only single-tagged packets will be matched.
8165                          */
8166                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8167                         return;
8168                 }
8169         }
8170         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8171                  rte_be_to_cpu_16(eth_m->type));
8172         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype);
8173         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
8174 }
8175
8176 /**
8177  * Add VLAN item to matcher and to the value.
8178  *
8179  * @param[in, out] dev_flow
8180  *   Flow descriptor.
8181  * @param[in, out] matcher
8182  *   Flow matcher.
8183  * @param[in, out] key
8184  *   Flow matcher value.
8185  * @param[in] item
8186  *   Flow pattern to translate.
8187  * @param[in] inner
8188  *   Item is inner pattern.
8189  */
8190 static void
8191 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
8192                             void *matcher, void *key,
8193                             const struct rte_flow_item *item,
8194                             int inner, uint32_t group)
8195 {
8196         const struct rte_flow_item_vlan *vlan_m = item->mask;
8197         const struct rte_flow_item_vlan *vlan_v = item->spec;
8198         void *hdrs_m;
8199         void *hdrs_v;
8200         uint16_t tci_m;
8201         uint16_t tci_v;
8202
8203         if (inner) {
8204                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8205                                          inner_headers);
8206                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8207         } else {
8208                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8209                                          outer_headers);
8210                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8211                 /*
8212                  * This is workaround, masks are not supported,
8213                  * and pre-validated.
8214                  */
8215                 if (vlan_v)
8216                         dev_flow->handle->vf_vlan.tag =
8217                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
8218         }
8219         /*
8220          * When VLAN item exists in flow, mark packet as tagged,
8221          * even if TCI is not specified.
8222          */
8223         if (!MLX5_GET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag)) {
8224                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8225                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8226         }
8227         if (!vlan_v)
8228                 return;
8229         if (!vlan_m)
8230                 vlan_m = &rte_flow_item_vlan_mask;
8231         tci_m = rte_be_to_cpu_16(vlan_m->tci);
8232         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
8233         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_vid, tci_m);
8234         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_vid, tci_v);
8235         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_cfi, tci_m >> 12);
8236         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_cfi, tci_v >> 12);
8237         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_prio, tci_m >> 13);
8238         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_prio, tci_v >> 13);
8239         /*
8240          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8241          * ethertype, and use ip_version field instead.
8242          */
8243         if (vlan_m->inner_type == 0xFFFF) {
8244                 switch (vlan_v->inner_type) {
8245                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8246                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8247                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8248                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8249                         return;
8250                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8251                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8252                         return;
8253                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8254                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8255                         return;
8256                 default:
8257                         break;
8258                 }
8259         }
8260         if (vlan_m->has_more_vlan && vlan_v->has_more_vlan) {
8261                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8262                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8263                 /* Only one vlan_tag bit can be set. */
8264                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8265                 return;
8266         }
8267         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8268                  rte_be_to_cpu_16(vlan_m->inner_type));
8269         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, ethertype,
8270                  rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
8271 }
8272
8273 /**
8274  * Add IPV4 item to matcher and to the value.
8275  *
8276  * @param[in, out] matcher
8277  *   Flow matcher.
8278  * @param[in, out] key
8279  *   Flow matcher value.
8280  * @param[in] item
8281  *   Flow pattern to translate.
8282  * @param[in] inner
8283  *   Item is inner pattern.
8284  * @param[in] group
8285  *   The group to insert the rule.
8286  */
8287 static void
8288 flow_dv_translate_item_ipv4(void *matcher, void *key,
8289                             const struct rte_flow_item *item,
8290                             int inner, uint32_t group)
8291 {
8292         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
8293         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
8294         const struct rte_flow_item_ipv4 nic_mask = {
8295                 .hdr = {
8296                         .src_addr = RTE_BE32(0xffffffff),
8297                         .dst_addr = RTE_BE32(0xffffffff),
8298                         .type_of_service = 0xff,
8299                         .next_proto_id = 0xff,
8300                         .time_to_live = 0xff,
8301                 },
8302         };
8303         void *headers_m;
8304         void *headers_v;
8305         char *l24_m;
8306         char *l24_v;
8307         uint8_t tos, ihl_m, ihl_v;
8308
8309         if (inner) {
8310                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8311                                          inner_headers);
8312                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8313         } else {
8314                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8315                                          outer_headers);
8316                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8317         }
8318         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
8319         if (!ipv4_v)
8320                 return;
8321         if (!ipv4_m)
8322                 ipv4_m = &nic_mask;
8323         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8324                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8325         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8326                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8327         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
8328         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
8329         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8330                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8331         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8332                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8333         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
8334         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
8335         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
8336         ihl_m = ipv4_m->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK;
8337         ihl_v = ipv4_v->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK;
8338         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_ihl, ihl_m);
8339         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_ihl, ihl_m & ihl_v);
8340         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
8341                  ipv4_m->hdr.type_of_service);
8342         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
8343         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
8344                  ipv4_m->hdr.type_of_service >> 2);
8345         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
8346         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8347                  ipv4_m->hdr.next_proto_id);
8348         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8349                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
8350         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8351                  ipv4_m->hdr.time_to_live);
8352         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8353                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
8354         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8355                  !!(ipv4_m->hdr.fragment_offset));
8356         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8357                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
8358 }
8359
8360 /**
8361  * Add IPV6 item to matcher and to the value.
8362  *
8363  * @param[in, out] matcher
8364  *   Flow matcher.
8365  * @param[in, out] key
8366  *   Flow matcher value.
8367  * @param[in] item
8368  *   Flow pattern to translate.
8369  * @param[in] inner
8370  *   Item is inner pattern.
8371  * @param[in] group
8372  *   The group to insert the rule.
8373  */
8374 static void
8375 flow_dv_translate_item_ipv6(void *matcher, void *key,
8376                             const struct rte_flow_item *item,
8377                             int inner, uint32_t group)
8378 {
8379         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
8380         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
8381         const struct rte_flow_item_ipv6 nic_mask = {
8382                 .hdr = {
8383                         .src_addr =
8384                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8385                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8386                         .dst_addr =
8387                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8388                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8389                         .vtc_flow = RTE_BE32(0xffffffff),
8390                         .proto = 0xff,
8391                         .hop_limits = 0xff,
8392                 },
8393         };
8394         void *headers_m;
8395         void *headers_v;
8396         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8397         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8398         char *l24_m;
8399         char *l24_v;
8400         uint32_t vtc_m;
8401         uint32_t vtc_v;
8402         int i;
8403         int size;
8404
8405         if (inner) {
8406                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8407                                          inner_headers);
8408                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8409         } else {
8410                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8411                                          outer_headers);
8412                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8413         }
8414         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
8415         if (!ipv6_v)
8416                 return;
8417         if (!ipv6_m)
8418                 ipv6_m = &nic_mask;
8419         size = sizeof(ipv6_m->hdr.dst_addr);
8420         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8421                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8422         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8423                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8424         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
8425         for (i = 0; i < size; ++i)
8426                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
8427         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8428                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8429         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8430                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8431         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
8432         for (i = 0; i < size; ++i)
8433                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
8434         /* TOS. */
8435         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
8436         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
8437         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
8438         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
8439         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
8440         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
8441         /* Label. */
8442         if (inner) {
8443                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
8444                          vtc_m);
8445                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
8446                          vtc_v);
8447         } else {
8448                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
8449                          vtc_m);
8450                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
8451                          vtc_v);
8452         }
8453         /* Protocol. */
8454         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8455                  ipv6_m->hdr.proto);
8456         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8457                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
8458         /* Hop limit. */
8459         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8460                  ipv6_m->hdr.hop_limits);
8461         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8462                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
8463         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8464                  !!(ipv6_m->has_frag_ext));
8465         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8466                  !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext));
8467 }
8468
8469 /**
8470  * Add IPV6 fragment extension item to matcher and to the value.
8471  *
8472  * @param[in, out] matcher
8473  *   Flow matcher.
8474  * @param[in, out] key
8475  *   Flow matcher value.
8476  * @param[in] item
8477  *   Flow pattern to translate.
8478  * @param[in] inner
8479  *   Item is inner pattern.
8480  */
8481 static void
8482 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key,
8483                                      const struct rte_flow_item *item,
8484                                      int inner)
8485 {
8486         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask;
8487         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec;
8488         const struct rte_flow_item_ipv6_frag_ext nic_mask = {
8489                 .hdr = {
8490                         .next_header = 0xff,
8491                         .frag_data = RTE_BE16(0xffff),
8492                 },
8493         };
8494         void *headers_m;
8495         void *headers_v;
8496
8497         if (inner) {
8498                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8499                                          inner_headers);
8500                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8501         } else {
8502                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8503                                          outer_headers);
8504                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8505         }
8506         /* IPv6 fragment extension item exists, so packet is IP fragment. */
8507         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
8508         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1);
8509         if (!ipv6_frag_ext_v)
8510                 return;
8511         if (!ipv6_frag_ext_m)
8512                 ipv6_frag_ext_m = &nic_mask;
8513         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8514                  ipv6_frag_ext_m->hdr.next_header);
8515         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8516                  ipv6_frag_ext_v->hdr.next_header &
8517                  ipv6_frag_ext_m->hdr.next_header);
8518 }
8519
8520 /**
8521  * Add TCP item to matcher and to the value.
8522  *
8523  * @param[in, out] matcher
8524  *   Flow matcher.
8525  * @param[in, out] key
8526  *   Flow matcher value.
8527  * @param[in] item
8528  *   Flow pattern to translate.
8529  * @param[in] inner
8530  *   Item is inner pattern.
8531  */
8532 static void
8533 flow_dv_translate_item_tcp(void *matcher, void *key,
8534                            const struct rte_flow_item *item,
8535                            int inner)
8536 {
8537         const struct rte_flow_item_tcp *tcp_m = item->mask;
8538         const struct rte_flow_item_tcp *tcp_v = item->spec;
8539         void *headers_m;
8540         void *headers_v;
8541
8542         if (inner) {
8543                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8544                                          inner_headers);
8545                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8546         } else {
8547                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8548                                          outer_headers);
8549                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8550         }
8551         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8552         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
8553         if (!tcp_v)
8554                 return;
8555         if (!tcp_m)
8556                 tcp_m = &rte_flow_item_tcp_mask;
8557         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
8558                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
8559         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
8560                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
8561         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
8562                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
8563         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
8564                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
8565         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
8566                  tcp_m->hdr.tcp_flags);
8567         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
8568                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
8569 }
8570
8571 /**
8572  * Add UDP item to matcher and to the value.
8573  *
8574  * @param[in, out] matcher
8575  *   Flow matcher.
8576  * @param[in, out] key
8577  *   Flow matcher value.
8578  * @param[in] item
8579  *   Flow pattern to translate.
8580  * @param[in] inner
8581  *   Item is inner pattern.
8582  */
8583 static void
8584 flow_dv_translate_item_udp(void *matcher, void *key,
8585                            const struct rte_flow_item *item,
8586                            int inner)
8587 {
8588         const struct rte_flow_item_udp *udp_m = item->mask;
8589         const struct rte_flow_item_udp *udp_v = item->spec;
8590         void *headers_m;
8591         void *headers_v;
8592
8593         if (inner) {
8594                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8595                                          inner_headers);
8596                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8597         } else {
8598                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8599                                          outer_headers);
8600                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8601         }
8602         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8603         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
8604         if (!udp_v)
8605                 return;
8606         if (!udp_m)
8607                 udp_m = &rte_flow_item_udp_mask;
8608         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
8609                  rte_be_to_cpu_16(udp_m->hdr.src_port));
8610         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
8611                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
8612         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
8613                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
8614         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
8615                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
8616 }
8617
8618 /**
8619  * Add GRE optional Key item to matcher and to the value.
8620  *
8621  * @param[in, out] matcher
8622  *   Flow matcher.
8623  * @param[in, out] key
8624  *   Flow matcher value.
8625  * @param[in] item
8626  *   Flow pattern to translate.
8627  * @param[in] inner
8628  *   Item is inner pattern.
8629  */
8630 static void
8631 flow_dv_translate_item_gre_key(void *matcher, void *key,
8632                                    const struct rte_flow_item *item)
8633 {
8634         const rte_be32_t *key_m = item->mask;
8635         const rte_be32_t *key_v = item->spec;
8636         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8637         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8638         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
8639
8640         /* GRE K bit must be on and should already be validated */
8641         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
8642         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
8643         if (!key_v)
8644                 return;
8645         if (!key_m)
8646                 key_m = &gre_key_default_mask;
8647         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
8648                  rte_be_to_cpu_32(*key_m) >> 8);
8649         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
8650                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
8651         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
8652                  rte_be_to_cpu_32(*key_m) & 0xFF);
8653         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
8654                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
8655 }
8656
8657 /**
8658  * Add GRE item to matcher and to the value.
8659  *
8660  * @param[in, out] matcher
8661  *   Flow matcher.
8662  * @param[in, out] key
8663  *   Flow matcher value.
8664  * @param[in] item
8665  *   Flow pattern to translate.
8666  * @param[in] inner
8667  *   Item is inner pattern.
8668  */
8669 static void
8670 flow_dv_translate_item_gre(void *matcher, void *key,
8671                            const struct rte_flow_item *item,
8672                            int inner)
8673 {
8674         const struct rte_flow_item_gre *gre_m = item->mask;
8675         const struct rte_flow_item_gre *gre_v = item->spec;
8676         void *headers_m;
8677         void *headers_v;
8678         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8679         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8680         struct {
8681                 union {
8682                         __extension__
8683                         struct {
8684                                 uint16_t version:3;
8685                                 uint16_t rsvd0:9;
8686                                 uint16_t s_present:1;
8687                                 uint16_t k_present:1;
8688                                 uint16_t rsvd_bit1:1;
8689                                 uint16_t c_present:1;
8690                         };
8691                         uint16_t value;
8692                 };
8693         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
8694
8695         if (inner) {
8696                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8697                                          inner_headers);
8698                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8699         } else {
8700                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8701                                          outer_headers);
8702                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8703         }
8704         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8705         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
8706         if (!gre_v)
8707                 return;
8708         if (!gre_m)
8709                 gre_m = &rte_flow_item_gre_mask;
8710         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
8711                  rte_be_to_cpu_16(gre_m->protocol));
8712         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
8713                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
8714         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
8715         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
8716         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
8717                  gre_crks_rsvd0_ver_m.c_present);
8718         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
8719                  gre_crks_rsvd0_ver_v.c_present &
8720                  gre_crks_rsvd0_ver_m.c_present);
8721         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
8722                  gre_crks_rsvd0_ver_m.k_present);
8723         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
8724                  gre_crks_rsvd0_ver_v.k_present &
8725                  gre_crks_rsvd0_ver_m.k_present);
8726         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
8727                  gre_crks_rsvd0_ver_m.s_present);
8728         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
8729                  gre_crks_rsvd0_ver_v.s_present &
8730                  gre_crks_rsvd0_ver_m.s_present);
8731 }
8732
8733 /**
8734  * Add NVGRE item to matcher and to the value.
8735  *
8736  * @param[in, out] matcher
8737  *   Flow matcher.
8738  * @param[in, out] key
8739  *   Flow matcher value.
8740  * @param[in] item
8741  *   Flow pattern to translate.
8742  * @param[in] inner
8743  *   Item is inner pattern.
8744  */
8745 static void
8746 flow_dv_translate_item_nvgre(void *matcher, void *key,
8747                              const struct rte_flow_item *item,
8748                              int inner)
8749 {
8750         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
8751         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
8752         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8753         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8754         const char *tni_flow_id_m;
8755         const char *tni_flow_id_v;
8756         char *gre_key_m;
8757         char *gre_key_v;
8758         int size;
8759         int i;
8760
8761         /* For NVGRE, GRE header fields must be set with defined values. */
8762         const struct rte_flow_item_gre gre_spec = {
8763                 .c_rsvd0_ver = RTE_BE16(0x2000),
8764                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
8765         };
8766         const struct rte_flow_item_gre gre_mask = {
8767                 .c_rsvd0_ver = RTE_BE16(0xB000),
8768                 .protocol = RTE_BE16(UINT16_MAX),
8769         };
8770         const struct rte_flow_item gre_item = {
8771                 .spec = &gre_spec,
8772                 .mask = &gre_mask,
8773                 .last = NULL,
8774         };
8775         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
8776         if (!nvgre_v)
8777                 return;
8778         if (!nvgre_m)
8779                 nvgre_m = &rte_flow_item_nvgre_mask;
8780         tni_flow_id_m = (const char *)nvgre_m->tni;
8781         tni_flow_id_v = (const char *)nvgre_v->tni;
8782         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
8783         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
8784         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
8785         memcpy(gre_key_m, tni_flow_id_m, size);
8786         for (i = 0; i < size; ++i)
8787                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
8788 }
8789
8790 /**
8791  * Add VXLAN item to matcher and to the value.
8792  *
8793  * @param[in] dev
8794  *   Pointer to the Ethernet device structure.
8795  * @param[in] attr
8796  *   Flow rule attributes.
8797  * @param[in, out] matcher
8798  *   Flow matcher.
8799  * @param[in, out] key
8800  *   Flow matcher value.
8801  * @param[in] item
8802  *   Flow pattern to translate.
8803  * @param[in] inner
8804  *   Item is inner pattern.
8805  */
8806 static void
8807 flow_dv_translate_item_vxlan(struct rte_eth_dev *dev,
8808                              const struct rte_flow_attr *attr,
8809                              void *matcher, void *key,
8810                              const struct rte_flow_item *item,
8811                              int inner)
8812 {
8813         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
8814         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
8815         void *headers_m;
8816         void *headers_v;
8817         void *misc5_m;
8818         void *misc5_v;
8819         uint32_t *tunnel_header_v;
8820         uint32_t *tunnel_header_m;
8821         uint16_t dport;
8822         struct mlx5_priv *priv = dev->data->dev_private;
8823         const struct rte_flow_item_vxlan nic_mask = {
8824                 .vni = "\xff\xff\xff",
8825                 .rsvd1 = 0xff,
8826         };
8827
8828         if (inner) {
8829                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8830                                          inner_headers);
8831                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8832         } else {
8833                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8834                                          outer_headers);
8835                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8836         }
8837         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
8838                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
8839         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
8840                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
8841                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
8842         }
8843         dport = MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport);
8844         if (!vxlan_v)
8845                 return;
8846         if (!vxlan_m) {
8847                 if ((!attr->group && !priv->sh->tunnel_header_0_1) ||
8848                     (attr->group && !priv->sh->misc5_cap))
8849                         vxlan_m = &rte_flow_item_vxlan_mask;
8850                 else
8851                         vxlan_m = &nic_mask;
8852         }
8853         if ((priv->sh->steering_format_version ==
8854             MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 &&
8855             dport != MLX5_UDP_PORT_VXLAN) ||
8856             (!attr->group && !attr->transfer && !priv->sh->tunnel_header_0_1) ||
8857             ((attr->group || attr->transfer) && !priv->sh->misc5_cap)) {
8858                 void *misc_m;
8859                 void *misc_v;
8860                 char *vni_m;
8861                 char *vni_v;
8862                 int size;
8863                 int i;
8864                 misc_m = MLX5_ADDR_OF(fte_match_param,
8865                                       matcher, misc_parameters);
8866                 misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8867                 size = sizeof(vxlan_m->vni);
8868                 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
8869                 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
8870                 memcpy(vni_m, vxlan_m->vni, size);
8871                 for (i = 0; i < size; ++i)
8872                         vni_v[i] = vni_m[i] & vxlan_v->vni[i];
8873                 return;
8874         }
8875         misc5_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_5);
8876         misc5_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_5);
8877         tunnel_header_v = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5,
8878                                                    misc5_v,
8879                                                    tunnel_header_1);
8880         tunnel_header_m = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5,
8881                                                    misc5_m,
8882                                                    tunnel_header_1);
8883         *tunnel_header_v = (vxlan_v->vni[0] & vxlan_m->vni[0]) |
8884                            (vxlan_v->vni[1] & vxlan_m->vni[1]) << 8 |
8885                            (vxlan_v->vni[2] & vxlan_m->vni[2]) << 16;
8886         if (*tunnel_header_v)
8887                 *tunnel_header_m = vxlan_m->vni[0] |
8888                         vxlan_m->vni[1] << 8 |
8889                         vxlan_m->vni[2] << 16;
8890         else
8891                 *tunnel_header_m = 0x0;
8892         *tunnel_header_v |= (vxlan_v->rsvd1 & vxlan_m->rsvd1) << 24;
8893         if (vxlan_v->rsvd1 & vxlan_m->rsvd1)
8894                 *tunnel_header_m |= vxlan_m->rsvd1 << 24;
8895 }
8896
8897 /**
8898  * Add VXLAN-GPE item to matcher and to the value.
8899  *
8900  * @param[in, out] matcher
8901  *   Flow matcher.
8902  * @param[in, out] key
8903  *   Flow matcher value.
8904  * @param[in] item
8905  *   Flow pattern to translate.
8906  * @param[in] inner
8907  *   Item is inner pattern.
8908  */
8909
8910 static void
8911 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
8912                                  const struct rte_flow_item *item, int inner)
8913 {
8914         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
8915         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
8916         void *headers_m;
8917         void *headers_v;
8918         void *misc_m =
8919                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
8920         void *misc_v =
8921                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
8922         char *vni_m;
8923         char *vni_v;
8924         uint16_t dport;
8925         int size;
8926         int i;
8927         uint8_t flags_m = 0xff;
8928         uint8_t flags_v = 0xc;
8929
8930         if (inner) {
8931                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8932                                          inner_headers);
8933                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8934         } else {
8935                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8936                                          outer_headers);
8937                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8938         }
8939         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
8940                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
8941         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
8942                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
8943                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
8944         }
8945         if (!vxlan_v)
8946                 return;
8947         if (!vxlan_m)
8948                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
8949         size = sizeof(vxlan_m->vni);
8950         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
8951         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
8952         memcpy(vni_m, vxlan_m->vni, size);
8953         for (i = 0; i < size; ++i)
8954                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
8955         if (vxlan_m->flags) {
8956                 flags_m = vxlan_m->flags;
8957                 flags_v = vxlan_v->flags;
8958         }
8959         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
8960         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
8961         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
8962                  vxlan_m->protocol);
8963         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
8964                  vxlan_v->protocol);
8965 }
8966
8967 /**
8968  * Add Geneve item to matcher and to the value.
8969  *
8970  * @param[in, out] matcher
8971  *   Flow matcher.
8972  * @param[in, out] key
8973  *   Flow matcher value.
8974  * @param[in] item
8975  *   Flow pattern to translate.
8976  * @param[in] inner
8977  *   Item is inner pattern.
8978  */
8979
8980 static void
8981 flow_dv_translate_item_geneve(void *matcher, void *key,
8982                               const struct rte_flow_item *item, int inner)
8983 {
8984         const struct rte_flow_item_geneve *geneve_m = item->mask;
8985         const struct rte_flow_item_geneve *geneve_v = item->spec;
8986         void *headers_m;
8987         void *headers_v;
8988         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8989         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8990         uint16_t dport;
8991         uint16_t gbhdr_m;
8992         uint16_t gbhdr_v;
8993         char *vni_m;
8994         char *vni_v;
8995         size_t size, i;
8996
8997         if (inner) {
8998                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8999                                          inner_headers);
9000                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9001         } else {
9002                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9003                                          outer_headers);
9004                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9005         }
9006         dport = MLX5_UDP_PORT_GENEVE;
9007         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9008                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9009                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
9010         }
9011         if (!geneve_v)
9012                 return;
9013         if (!geneve_m)
9014                 geneve_m = &rte_flow_item_geneve_mask;
9015         size = sizeof(geneve_m->vni);
9016         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
9017         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
9018         memcpy(vni_m, geneve_m->vni, size);
9019         for (i = 0; i < size; ++i)
9020                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
9021         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
9022                  rte_be_to_cpu_16(geneve_m->protocol));
9023         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
9024                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
9025         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
9026         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
9027         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
9028                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
9029         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
9030                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
9031         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
9032                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
9033         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
9034                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
9035                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
9036 }
9037
9038 /**
9039  * Create Geneve TLV option resource.
9040  *
9041  * @param dev[in, out]
9042  *   Pointer to rte_eth_dev structure.
9043  * @param[in, out] tag_be24
9044  *   Tag value in big endian then R-shift 8.
9045  * @parm[in, out] dev_flow
9046  *   Pointer to the dev_flow.
9047  * @param[out] error
9048  *   pointer to error structure.
9049  *
9050  * @return
9051  *   0 on success otherwise -errno and errno is set.
9052  */
9053
9054 int
9055 flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
9056                                              const struct rte_flow_item *item,
9057                                              struct rte_flow_error *error)
9058 {
9059         struct mlx5_priv *priv = dev->data->dev_private;
9060         struct mlx5_dev_ctx_shared *sh = priv->sh;
9061         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
9062                         sh->geneve_tlv_option_resource;
9063         struct mlx5_devx_obj *obj;
9064         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
9065         int ret = 0;
9066
9067         if (!geneve_opt_v)
9068                 return -1;
9069         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
9070         if (geneve_opt_resource != NULL) {
9071                 if (geneve_opt_resource->option_class ==
9072                         geneve_opt_v->option_class &&
9073                         geneve_opt_resource->option_type ==
9074                         geneve_opt_v->option_type &&
9075                         geneve_opt_resource->length ==
9076                         geneve_opt_v->option_len) {
9077                         /* We already have GENVE TLV option obj allocated. */
9078                         __atomic_fetch_add(&geneve_opt_resource->refcnt, 1,
9079                                            __ATOMIC_RELAXED);
9080                 } else {
9081                         ret = rte_flow_error_set(error, ENOMEM,
9082                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9083                                 "Only one GENEVE TLV option supported");
9084                         goto exit;
9085                 }
9086         } else {
9087                 /* Create a GENEVE TLV object and resource. */
9088                 obj = mlx5_devx_cmd_create_geneve_tlv_option(sh->cdev->ctx,
9089                                 geneve_opt_v->option_class,
9090                                 geneve_opt_v->option_type,
9091                                 geneve_opt_v->option_len);
9092                 if (!obj) {
9093                         ret = rte_flow_error_set(error, ENODATA,
9094                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9095                                 "Failed to create GENEVE TLV Devx object");
9096                         goto exit;
9097                 }
9098                 sh->geneve_tlv_option_resource =
9099                                 mlx5_malloc(MLX5_MEM_ZERO,
9100                                                 sizeof(*geneve_opt_resource),
9101                                                 0, SOCKET_ID_ANY);
9102                 if (!sh->geneve_tlv_option_resource) {
9103                         claim_zero(mlx5_devx_cmd_destroy(obj));
9104                         ret = rte_flow_error_set(error, ENOMEM,
9105                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9106                                 "GENEVE TLV object memory allocation failed");
9107                         goto exit;
9108                 }
9109                 geneve_opt_resource = sh->geneve_tlv_option_resource;
9110                 geneve_opt_resource->obj = obj;
9111                 geneve_opt_resource->option_class = geneve_opt_v->option_class;
9112                 geneve_opt_resource->option_type = geneve_opt_v->option_type;
9113                 geneve_opt_resource->length = geneve_opt_v->option_len;
9114                 __atomic_store_n(&geneve_opt_resource->refcnt, 1,
9115                                 __ATOMIC_RELAXED);
9116         }
9117 exit:
9118         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
9119         return ret;
9120 }
9121
9122 /**
9123  * Add Geneve TLV option item to matcher.
9124  *
9125  * @param[in, out] dev
9126  *   Pointer to rte_eth_dev structure.
9127  * @param[in, out] matcher
9128  *   Flow matcher.
9129  * @param[in, out] key
9130  *   Flow matcher value.
9131  * @param[in] item
9132  *   Flow pattern to translate.
9133  * @param[out] error
9134  *   Pointer to error structure.
9135  */
9136 static int
9137 flow_dv_translate_item_geneve_opt(struct rte_eth_dev *dev, void *matcher,
9138                                   void *key, const struct rte_flow_item *item,
9139                                   struct rte_flow_error *error)
9140 {
9141         const struct rte_flow_item_geneve_opt *geneve_opt_m = item->mask;
9142         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
9143         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9144         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9145         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9146                         misc_parameters_3);
9147         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9148         rte_be32_t opt_data_key = 0, opt_data_mask = 0;
9149         int ret = 0;
9150
9151         if (!geneve_opt_v)
9152                 return -1;
9153         if (!geneve_opt_m)
9154                 geneve_opt_m = &rte_flow_item_geneve_opt_mask;
9155         ret = flow_dev_geneve_tlv_option_resource_register(dev, item,
9156                                                            error);
9157         if (ret) {
9158                 DRV_LOG(ERR, "Failed to create geneve_tlv_obj");
9159                 return ret;
9160         }
9161         /*
9162          * Set the option length in GENEVE header if not requested.
9163          * The GENEVE TLV option length is expressed by the option length field
9164          * in the GENEVE header.
9165          * If the option length was not requested but the GENEVE TLV option item
9166          * is present we set the option length field implicitly.
9167          */
9168         if (!MLX5_GET16(fte_match_set_misc, misc_m, geneve_opt_len)) {
9169                 MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
9170                          MLX5_GENEVE_OPTLEN_MASK);
9171                 MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
9172                          geneve_opt_v->option_len + 1);
9173         }
9174         MLX5_SET(fte_match_set_misc, misc_m, geneve_tlv_option_0_exist, 1);
9175         MLX5_SET(fte_match_set_misc, misc_v, geneve_tlv_option_0_exist, 1);
9176         /* Set the data. */
9177         if (geneve_opt_v->data) {
9178                 memcpy(&opt_data_key, geneve_opt_v->data,
9179                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
9180                                 sizeof(opt_data_key)));
9181                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
9182                                 sizeof(opt_data_key));
9183                 memcpy(&opt_data_mask, geneve_opt_m->data,
9184                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
9185                                 sizeof(opt_data_mask)));
9186                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
9187                                 sizeof(opt_data_mask));
9188                 MLX5_SET(fte_match_set_misc3, misc3_m,
9189                                 geneve_tlv_option_0_data,
9190                                 rte_be_to_cpu_32(opt_data_mask));
9191                 MLX5_SET(fte_match_set_misc3, misc3_v,
9192                                 geneve_tlv_option_0_data,
9193                         rte_be_to_cpu_32(opt_data_key & opt_data_mask));
9194         }
9195         return ret;
9196 }
9197
9198 /**
9199  * Add MPLS item to matcher and to the value.
9200  *
9201  * @param[in, out] matcher
9202  *   Flow matcher.
9203  * @param[in, out] key
9204  *   Flow matcher value.
9205  * @param[in] item
9206  *   Flow pattern to translate.
9207  * @param[in] prev_layer
9208  *   The protocol layer indicated in previous item.
9209  * @param[in] inner
9210  *   Item is inner pattern.
9211  */
9212 static void
9213 flow_dv_translate_item_mpls(void *matcher, void *key,
9214                             const struct rte_flow_item *item,
9215                             uint64_t prev_layer,
9216                             int inner)
9217 {
9218         const uint32_t *in_mpls_m = item->mask;
9219         const uint32_t *in_mpls_v = item->spec;
9220         uint32_t *out_mpls_m = 0;
9221         uint32_t *out_mpls_v = 0;
9222         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9223         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9224         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
9225                                      misc_parameters_2);
9226         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9227         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
9228         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9229
9230         switch (prev_layer) {
9231         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9232                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
9233                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
9234                          MLX5_UDP_PORT_MPLS);
9235                 break;
9236         case MLX5_FLOW_LAYER_GRE:
9237                 /* Fall-through. */
9238         case MLX5_FLOW_LAYER_GRE_KEY:
9239                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
9240                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
9241                          RTE_ETHER_TYPE_MPLS);
9242                 break;
9243         default:
9244                 break;
9245         }
9246         if (!in_mpls_v)
9247                 return;
9248         if (!in_mpls_m)
9249                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
9250         switch (prev_layer) {
9251         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9252                 out_mpls_m =
9253                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9254                                                  outer_first_mpls_over_udp);
9255                 out_mpls_v =
9256                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9257                                                  outer_first_mpls_over_udp);
9258                 break;
9259         case MLX5_FLOW_LAYER_GRE:
9260                 out_mpls_m =
9261                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9262                                                  outer_first_mpls_over_gre);
9263                 out_mpls_v =
9264                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9265                                                  outer_first_mpls_over_gre);
9266                 break;
9267         default:
9268                 /* Inner MPLS not over GRE is not supported. */
9269                 if (!inner) {
9270                         out_mpls_m =
9271                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9272                                                          misc2_m,
9273                                                          outer_first_mpls);
9274                         out_mpls_v =
9275                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9276                                                          misc2_v,
9277                                                          outer_first_mpls);
9278                 }
9279                 break;
9280         }
9281         if (out_mpls_m && out_mpls_v) {
9282                 *out_mpls_m = *in_mpls_m;
9283                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
9284         }
9285 }
9286
9287 /**
9288  * Add metadata register item to matcher
9289  *
9290  * @param[in, out] matcher
9291  *   Flow matcher.
9292  * @param[in, out] key
9293  *   Flow matcher value.
9294  * @param[in] reg_type
9295  *   Type of device metadata register
9296  * @param[in] value
9297  *   Register value
9298  * @param[in] mask
9299  *   Register mask
9300  */
9301 static void
9302 flow_dv_match_meta_reg(void *matcher, void *key,
9303                        enum modify_reg reg_type,
9304                        uint32_t data, uint32_t mask)
9305 {
9306         void *misc2_m =
9307                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
9308         void *misc2_v =
9309                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9310         uint32_t temp;
9311
9312         data &= mask;
9313         switch (reg_type) {
9314         case REG_A:
9315                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
9316                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
9317                 break;
9318         case REG_B:
9319                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
9320                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
9321                 break;
9322         case REG_C_0:
9323                 /*
9324                  * The metadata register C0 field might be divided into
9325                  * source vport index and META item value, we should set
9326                  * this field according to specified mask, not as whole one.
9327                  */
9328                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
9329                 temp |= mask;
9330                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
9331                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
9332                 temp &= ~mask;
9333                 temp |= data;
9334                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
9335                 break;
9336         case REG_C_1:
9337                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
9338                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
9339                 break;
9340         case REG_C_2:
9341                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
9342                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
9343                 break;
9344         case REG_C_3:
9345                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
9346                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
9347                 break;
9348         case REG_C_4:
9349                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
9350                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
9351                 break;
9352         case REG_C_5:
9353                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
9354                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
9355                 break;
9356         case REG_C_6:
9357                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
9358                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
9359                 break;
9360         case REG_C_7:
9361                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
9362                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
9363                 break;
9364         default:
9365                 MLX5_ASSERT(false);
9366                 break;
9367         }
9368 }
9369
9370 /**
9371  * Add MARK item to matcher
9372  *
9373  * @param[in] dev
9374  *   The device to configure through.
9375  * @param[in, out] matcher
9376  *   Flow matcher.
9377  * @param[in, out] key
9378  *   Flow matcher value.
9379  * @param[in] item
9380  *   Flow pattern to translate.
9381  */
9382 static void
9383 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
9384                             void *matcher, void *key,
9385                             const struct rte_flow_item *item)
9386 {
9387         struct mlx5_priv *priv = dev->data->dev_private;
9388         const struct rte_flow_item_mark *mark;
9389         uint32_t value;
9390         uint32_t mask;
9391
9392         mark = item->mask ? (const void *)item->mask :
9393                             &rte_flow_item_mark_mask;
9394         mask = mark->id & priv->sh->dv_mark_mask;
9395         mark = (const void *)item->spec;
9396         MLX5_ASSERT(mark);
9397         value = mark->id & priv->sh->dv_mark_mask & mask;
9398         if (mask) {
9399                 enum modify_reg reg;
9400
9401                 /* Get the metadata register index for the mark. */
9402                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
9403                 MLX5_ASSERT(reg > 0);
9404                 if (reg == REG_C_0) {
9405                         struct mlx5_priv *priv = dev->data->dev_private;
9406                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9407                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9408
9409                         mask &= msk_c0;
9410                         mask <<= shl_c0;
9411                         value <<= shl_c0;
9412                 }
9413                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9414         }
9415 }
9416
9417 /**
9418  * Add META item to matcher
9419  *
9420  * @param[in] dev
9421  *   The devich to configure through.
9422  * @param[in, out] matcher
9423  *   Flow matcher.
9424  * @param[in, out] key
9425  *   Flow matcher value.
9426  * @param[in] attr
9427  *   Attributes of flow that includes this item.
9428  * @param[in] item
9429  *   Flow pattern to translate.
9430  */
9431 static void
9432 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
9433                             void *matcher, void *key,
9434                             const struct rte_flow_attr *attr,
9435                             const struct rte_flow_item *item)
9436 {
9437         const struct rte_flow_item_meta *meta_m;
9438         const struct rte_flow_item_meta *meta_v;
9439
9440         meta_m = (const void *)item->mask;
9441         if (!meta_m)
9442                 meta_m = &rte_flow_item_meta_mask;
9443         meta_v = (const void *)item->spec;
9444         if (meta_v) {
9445                 int reg;
9446                 uint32_t value = meta_v->data;
9447                 uint32_t mask = meta_m->data;
9448
9449                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
9450                 if (reg < 0)
9451                         return;
9452                 MLX5_ASSERT(reg != REG_NON);
9453                 if (reg == REG_C_0) {
9454                         struct mlx5_priv *priv = dev->data->dev_private;
9455                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9456                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9457
9458                         mask &= msk_c0;
9459                         mask <<= shl_c0;
9460                         value <<= shl_c0;
9461                 }
9462                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9463         }
9464 }
9465
9466 /**
9467  * Add vport metadata Reg C0 item to matcher
9468  *
9469  * @param[in, out] matcher
9470  *   Flow matcher.
9471  * @param[in, out] key
9472  *   Flow matcher value.
9473  * @param[in] reg
9474  *   Flow pattern to translate.
9475  */
9476 static void
9477 flow_dv_translate_item_meta_vport(void *matcher, void *key,
9478                                   uint32_t value, uint32_t mask)
9479 {
9480         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
9481 }
9482
9483 /**
9484  * Add tag item to matcher
9485  *
9486  * @param[in] dev
9487  *   The devich to configure through.
9488  * @param[in, out] matcher
9489  *   Flow matcher.
9490  * @param[in, out] key
9491  *   Flow matcher value.
9492  * @param[in] item
9493  *   Flow pattern to translate.
9494  */
9495 static void
9496 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
9497                                 void *matcher, void *key,
9498                                 const struct rte_flow_item *item)
9499 {
9500         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
9501         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
9502         uint32_t mask, value;
9503
9504         MLX5_ASSERT(tag_v);
9505         value = tag_v->data;
9506         mask = tag_m ? tag_m->data : UINT32_MAX;
9507         if (tag_v->id == REG_C_0) {
9508                 struct mlx5_priv *priv = dev->data->dev_private;
9509                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9510                 uint32_t shl_c0 = rte_bsf32(msk_c0);
9511
9512                 mask &= msk_c0;
9513                 mask <<= shl_c0;
9514                 value <<= shl_c0;
9515         }
9516         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
9517 }
9518
9519 /**
9520  * Add TAG item to matcher
9521  *
9522  * @param[in] dev
9523  *   The devich to configure through.
9524  * @param[in, out] matcher
9525  *   Flow matcher.
9526  * @param[in, out] key
9527  *   Flow matcher value.
9528  * @param[in] item
9529  *   Flow pattern to translate.
9530  */
9531 static void
9532 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
9533                            void *matcher, void *key,
9534                            const struct rte_flow_item *item)
9535 {
9536         const struct rte_flow_item_tag *tag_v = item->spec;
9537         const struct rte_flow_item_tag *tag_m = item->mask;
9538         enum modify_reg reg;
9539
9540         MLX5_ASSERT(tag_v);
9541         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
9542         /* Get the metadata register index for the tag. */
9543         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
9544         MLX5_ASSERT(reg > 0);
9545         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
9546 }
9547
9548 /**
9549  * Add source vport match to the specified matcher.
9550  *
9551  * @param[in, out] matcher
9552  *   Flow matcher.
9553  * @param[in, out] key
9554  *   Flow matcher value.
9555  * @param[in] port
9556  *   Source vport value to match
9557  * @param[in] mask
9558  *   Mask
9559  */
9560 static void
9561 flow_dv_translate_item_source_vport(void *matcher, void *key,
9562                                     int16_t port, uint16_t mask)
9563 {
9564         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9565         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9566
9567         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
9568         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
9569 }
9570
9571 /**
9572  * Translate port-id item to eswitch match on  port-id.
9573  *
9574  * @param[in] dev
9575  *   The devich to configure through.
9576  * @param[in, out] matcher
9577  *   Flow matcher.
9578  * @param[in, out] key
9579  *   Flow matcher value.
9580  * @param[in] item
9581  *   Flow pattern to translate.
9582  * @param[in]
9583  *   Flow attributes.
9584  *
9585  * @return
9586  *   0 on success, a negative errno value otherwise.
9587  */
9588 static int
9589 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
9590                                void *key, const struct rte_flow_item *item,
9591                                const struct rte_flow_attr *attr)
9592 {
9593         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
9594         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
9595         struct mlx5_priv *priv;
9596         uint16_t mask, id;
9597
9598         if (pid_v && pid_v->id == MLX5_PORT_ESW_MGR) {
9599                 flow_dv_translate_item_source_vport(matcher, key,
9600                         flow_dv_get_esw_manager_vport_id(dev), 0xffff);
9601                 return 0;
9602         }
9603         mask = pid_m ? pid_m->id : 0xffff;
9604         id = pid_v ? pid_v->id : dev->data->port_id;
9605         priv = mlx5_port_to_eswitch_info(id, item == NULL);
9606         if (!priv)
9607                 return -rte_errno;
9608         /*
9609          * Translate to vport field or to metadata, depending on mode.
9610          * Kernel can use either misc.source_port or half of C0 metadata
9611          * register.
9612          */
9613         if (priv->vport_meta_mask) {
9614                 /*
9615                  * Provide the hint for SW steering library
9616                  * to insert the flow into ingress domain and
9617                  * save the extra vport match.
9618                  */
9619                 if (mask == 0xffff && priv->vport_id == 0xffff &&
9620                     priv->pf_bond < 0 && attr->transfer)
9621                         flow_dv_translate_item_source_vport
9622                                 (matcher, key, priv->vport_id, mask);
9623                 /*
9624                  * We should always set the vport metadata register,
9625                  * otherwise the SW steering library can drop
9626                  * the rule if wire vport metadata value is not zero,
9627                  * it depends on kernel configuration.
9628                  */
9629                 flow_dv_translate_item_meta_vport(matcher, key,
9630                                                   priv->vport_meta_tag,
9631                                                   priv->vport_meta_mask);
9632         } else {
9633                 flow_dv_translate_item_source_vport(matcher, key,
9634                                                     priv->vport_id, mask);
9635         }
9636         return 0;
9637 }
9638
9639 /**
9640  * Add ICMP6 item to matcher and to the value.
9641  *
9642  * @param[in, out] matcher
9643  *   Flow matcher.
9644  * @param[in, out] key
9645  *   Flow matcher value.
9646  * @param[in] item
9647  *   Flow pattern to translate.
9648  * @param[in] inner
9649  *   Item is inner pattern.
9650  */
9651 static void
9652 flow_dv_translate_item_icmp6(void *matcher, void *key,
9653                               const struct rte_flow_item *item,
9654                               int inner)
9655 {
9656         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
9657         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
9658         void *headers_m;
9659         void *headers_v;
9660         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9661                                      misc_parameters_3);
9662         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9663         if (inner) {
9664                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9665                                          inner_headers);
9666                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9667         } else {
9668                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9669                                          outer_headers);
9670                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9671         }
9672         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
9673         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
9674         if (!icmp6_v)
9675                 return;
9676         if (!icmp6_m)
9677                 icmp6_m = &rte_flow_item_icmp6_mask;
9678         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
9679         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
9680                  icmp6_v->type & icmp6_m->type);
9681         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
9682         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
9683                  icmp6_v->code & icmp6_m->code);
9684 }
9685
9686 /**
9687  * Add ICMP item to matcher and to the value.
9688  *
9689  * @param[in, out] matcher
9690  *   Flow matcher.
9691  * @param[in, out] key
9692  *   Flow matcher value.
9693  * @param[in] item
9694  *   Flow pattern to translate.
9695  * @param[in] inner
9696  *   Item is inner pattern.
9697  */
9698 static void
9699 flow_dv_translate_item_icmp(void *matcher, void *key,
9700                             const struct rte_flow_item *item,
9701                             int inner)
9702 {
9703         const struct rte_flow_item_icmp *icmp_m = item->mask;
9704         const struct rte_flow_item_icmp *icmp_v = item->spec;
9705         uint32_t icmp_header_data_m = 0;
9706         uint32_t icmp_header_data_v = 0;
9707         void *headers_m;
9708         void *headers_v;
9709         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9710                                      misc_parameters_3);
9711         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9712         if (inner) {
9713                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9714                                          inner_headers);
9715                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9716         } else {
9717                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9718                                          outer_headers);
9719                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9720         }
9721         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
9722         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
9723         if (!icmp_v)
9724                 return;
9725         if (!icmp_m)
9726                 icmp_m = &rte_flow_item_icmp_mask;
9727         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
9728                  icmp_m->hdr.icmp_type);
9729         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
9730                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
9731         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
9732                  icmp_m->hdr.icmp_code);
9733         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
9734                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
9735         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
9736         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
9737         if (icmp_header_data_m) {
9738                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
9739                 icmp_header_data_v |=
9740                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
9741                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
9742                          icmp_header_data_m);
9743                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
9744                          icmp_header_data_v & icmp_header_data_m);
9745         }
9746 }
9747
9748 /**
9749  * Add GTP item to matcher and to the value.
9750  *
9751  * @param[in, out] matcher
9752  *   Flow matcher.
9753  * @param[in, out] key
9754  *   Flow matcher value.
9755  * @param[in] item
9756  *   Flow pattern to translate.
9757  * @param[in] inner
9758  *   Item is inner pattern.
9759  */
9760 static void
9761 flow_dv_translate_item_gtp(void *matcher, void *key,
9762                            const struct rte_flow_item *item, int inner)
9763 {
9764         const struct rte_flow_item_gtp *gtp_m = item->mask;
9765         const struct rte_flow_item_gtp *gtp_v = item->spec;
9766         void *headers_m;
9767         void *headers_v;
9768         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9769                                      misc_parameters_3);
9770         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9771         uint16_t dport = RTE_GTPU_UDP_PORT;
9772
9773         if (inner) {
9774                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9775                                          inner_headers);
9776                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9777         } else {
9778                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9779                                          outer_headers);
9780                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9781         }
9782         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9783                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9784                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
9785         }
9786         if (!gtp_v)
9787                 return;
9788         if (!gtp_m)
9789                 gtp_m = &rte_flow_item_gtp_mask;
9790         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
9791                  gtp_m->v_pt_rsv_flags);
9792         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
9793                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
9794         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
9795         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
9796                  gtp_v->msg_type & gtp_m->msg_type);
9797         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
9798                  rte_be_to_cpu_32(gtp_m->teid));
9799         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
9800                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
9801 }
9802
9803 /**
9804  * Add GTP PSC item to matcher.
9805  *
9806  * @param[in, out] matcher
9807  *   Flow matcher.
9808  * @param[in, out] key
9809  *   Flow matcher value.
9810  * @param[in] item
9811  *   Flow pattern to translate.
9812  */
9813 static int
9814 flow_dv_translate_item_gtp_psc(void *matcher, void *key,
9815                                const struct rte_flow_item *item)
9816 {
9817         const struct rte_flow_item_gtp_psc *gtp_psc_m = item->mask;
9818         const struct rte_flow_item_gtp_psc *gtp_psc_v = item->spec;
9819         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9820                         misc_parameters_3);
9821         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9822         union {
9823                 uint32_t w32;
9824                 struct {
9825                         uint16_t seq_num;
9826                         uint8_t npdu_num;
9827                         uint8_t next_ext_header_type;
9828                 };
9829         } dw_2;
9830         uint8_t gtp_flags;
9831
9832         /* Always set E-flag match on one, regardless of GTP item settings. */
9833         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_m, gtpu_msg_flags);
9834         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
9835         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags, gtp_flags);
9836         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_v, gtpu_msg_flags);
9837         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
9838         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags, gtp_flags);
9839         /*Set next extension header type. */
9840         dw_2.seq_num = 0;
9841         dw_2.npdu_num = 0;
9842         dw_2.next_ext_header_type = 0xff;
9843         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_dw_2,
9844                  rte_cpu_to_be_32(dw_2.w32));
9845         dw_2.seq_num = 0;
9846         dw_2.npdu_num = 0;
9847         dw_2.next_ext_header_type = 0x85;
9848         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_dw_2,
9849                  rte_cpu_to_be_32(dw_2.w32));
9850         if (gtp_psc_v) {
9851                 union {
9852                         uint32_t w32;
9853                         struct {
9854                                 uint8_t len;
9855                                 uint8_t type_flags;
9856                                 uint8_t qfi;
9857                                 uint8_t reserved;
9858                         };
9859                 } dw_0;
9860
9861                 /*Set extension header PDU type and Qos. */
9862                 if (!gtp_psc_m)
9863                         gtp_psc_m = &rte_flow_item_gtp_psc_mask;
9864                 dw_0.w32 = 0;
9865                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_m->hdr.type);
9866                 dw_0.qfi = gtp_psc_m->hdr.qfi;
9867                 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_first_ext_dw_0,
9868                          rte_cpu_to_be_32(dw_0.w32));
9869                 dw_0.w32 = 0;
9870                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_v->hdr.type &
9871                                                         gtp_psc_m->hdr.type);
9872                 dw_0.qfi = gtp_psc_v->hdr.qfi & gtp_psc_m->hdr.qfi;
9873                 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_first_ext_dw_0,
9874                          rte_cpu_to_be_32(dw_0.w32));
9875         }
9876         return 0;
9877 }
9878
9879 /**
9880  * Add eCPRI item to matcher and to the value.
9881  *
9882  * @param[in] dev
9883  *   The devich to configure through.
9884  * @param[in, out] matcher
9885  *   Flow matcher.
9886  * @param[in, out] key
9887  *   Flow matcher value.
9888  * @param[in] item
9889  *   Flow pattern to translate.
9890  * @param[in] last_item
9891  *   Last item flags.
9892  */
9893 static void
9894 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
9895                              void *key, const struct rte_flow_item *item,
9896                              uint64_t last_item)
9897 {
9898         struct mlx5_priv *priv = dev->data->dev_private;
9899         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
9900         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
9901         struct rte_ecpri_common_hdr common;
9902         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
9903                                      misc_parameters_4);
9904         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
9905         uint32_t *samples;
9906         void *dw_m;
9907         void *dw_v;
9908
9909         /*
9910          * In case of eCPRI over Ethernet, if EtherType is not specified,
9911          * match on eCPRI EtherType implicitly.
9912          */
9913         if (last_item & MLX5_FLOW_LAYER_OUTER_L2) {
9914                 void *hdrs_m, *hdrs_v, *l2m, *l2v;
9915
9916                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
9917                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9918                 l2m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, ethertype);
9919                 l2v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype);
9920                 if (*(uint16_t *)l2m == 0 && *(uint16_t *)l2v == 0) {
9921                         *(uint16_t *)l2m = UINT16_MAX;
9922                         *(uint16_t *)l2v = RTE_BE16(RTE_ETHER_TYPE_ECPRI);
9923                 }
9924         }
9925         if (!ecpri_v)
9926                 return;
9927         if (!ecpri_m)
9928                 ecpri_m = &rte_flow_item_ecpri_mask;
9929         /*
9930          * Maximal four DW samples are supported in a single matching now.
9931          * Two are used now for a eCPRI matching:
9932          * 1. Type: one byte, mask should be 0x00ff0000 in network order
9933          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
9934          *    if any.
9935          */
9936         if (!ecpri_m->hdr.common.u32)
9937                 return;
9938         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
9939         /* Need to take the whole DW as the mask to fill the entry. */
9940         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
9941                             prog_sample_field_value_0);
9942         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
9943                             prog_sample_field_value_0);
9944         /* Already big endian (network order) in the header. */
9945         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
9946         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32 & ecpri_m->hdr.common.u32;
9947         /* Sample#0, used for matching type, offset 0. */
9948         MLX5_SET(fte_match_set_misc4, misc4_m,
9949                  prog_sample_field_id_0, samples[0]);
9950         /* It makes no sense to set the sample ID in the mask field. */
9951         MLX5_SET(fte_match_set_misc4, misc4_v,
9952                  prog_sample_field_id_0, samples[0]);
9953         /*
9954          * Checking if message body part needs to be matched.
9955          * Some wildcard rules only matching type field should be supported.
9956          */
9957         if (ecpri_m->hdr.dummy[0]) {
9958                 common.u32 = rte_be_to_cpu_32(ecpri_v->hdr.common.u32);
9959                 switch (common.type) {
9960                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
9961                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
9962                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
9963                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
9964                                             prog_sample_field_value_1);
9965                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
9966                                             prog_sample_field_value_1);
9967                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
9968                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0] &
9969                                             ecpri_m->hdr.dummy[0];
9970                         /* Sample#1, to match message body, offset 4. */
9971                         MLX5_SET(fte_match_set_misc4, misc4_m,
9972                                  prog_sample_field_id_1, samples[1]);
9973                         MLX5_SET(fte_match_set_misc4, misc4_v,
9974                                  prog_sample_field_id_1, samples[1]);
9975                         break;
9976                 default:
9977                         /* Others, do not match any sample ID. */
9978                         break;
9979                 }
9980         }
9981 }
9982
9983 /*
9984  * Add connection tracking status item to matcher
9985  *
9986  * @param[in] dev
9987  *   The devich to configure through.
9988  * @param[in, out] matcher
9989  *   Flow matcher.
9990  * @param[in, out] key
9991  *   Flow matcher value.
9992  * @param[in] item
9993  *   Flow pattern to translate.
9994  */
9995 static void
9996 flow_dv_translate_item_aso_ct(struct rte_eth_dev *dev,
9997                               void *matcher, void *key,
9998                               const struct rte_flow_item *item)
9999 {
10000         uint32_t reg_value = 0;
10001         int reg_id;
10002         /* 8LSB 0b 11/0000/11, middle 4 bits are reserved. */
10003         uint32_t reg_mask = 0;
10004         const struct rte_flow_item_conntrack *spec = item->spec;
10005         const struct rte_flow_item_conntrack *mask = item->mask;
10006         uint32_t flags;
10007         struct rte_flow_error error;
10008
10009         if (!mask)
10010                 mask = &rte_flow_item_conntrack_mask;
10011         if (!spec || !mask->flags)
10012                 return;
10013         flags = spec->flags & mask->flags;
10014         /* The conflict should be checked in the validation. */
10015         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID)
10016                 reg_value |= MLX5_CT_SYNDROME_VALID;
10017         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
10018                 reg_value |= MLX5_CT_SYNDROME_STATE_CHANGE;
10019         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID)
10020                 reg_value |= MLX5_CT_SYNDROME_INVALID;
10021         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)
10022                 reg_value |= MLX5_CT_SYNDROME_TRAP;
10023         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
10024                 reg_value |= MLX5_CT_SYNDROME_BAD_PACKET;
10025         if (mask->flags & (RTE_FLOW_CONNTRACK_PKT_STATE_VALID |
10026                            RTE_FLOW_CONNTRACK_PKT_STATE_INVALID |
10027                            RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED))
10028                 reg_mask |= 0xc0;
10029         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
10030                 reg_mask |= MLX5_CT_SYNDROME_STATE_CHANGE;
10031         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
10032                 reg_mask |= MLX5_CT_SYNDROME_BAD_PACKET;
10033         /* The REG_C_x value could be saved during startup. */
10034         reg_id = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, &error);
10035         if (reg_id == REG_NON)
10036                 return;
10037         flow_dv_match_meta_reg(matcher, key, (enum modify_reg)reg_id,
10038                                reg_value, reg_mask);
10039 }
10040
10041 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
10042
10043 #define HEADER_IS_ZERO(match_criteria, headers)                              \
10044         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
10045                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
10046
10047 /**
10048  * Calculate flow matcher enable bitmap.
10049  *
10050  * @param match_criteria
10051  *   Pointer to flow matcher criteria.
10052  *
10053  * @return
10054  *   Bitmap of enabled fields.
10055  */
10056 static uint8_t
10057 flow_dv_matcher_enable(uint32_t *match_criteria)
10058 {
10059         uint8_t match_criteria_enable;
10060
10061         match_criteria_enable =
10062                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
10063                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
10064         match_criteria_enable |=
10065                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
10066                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
10067         match_criteria_enable |=
10068                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
10069                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
10070         match_criteria_enable |=
10071                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
10072                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
10073         match_criteria_enable |=
10074                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
10075                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
10076         match_criteria_enable |=
10077                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
10078                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
10079         match_criteria_enable |=
10080                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_5)) <<
10081                 MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT;
10082         return match_criteria_enable;
10083 }
10084
10085 static void
10086 __flow_dv_adjust_buf_size(size_t *size, uint8_t match_criteria)
10087 {
10088         /*
10089          * Check flow matching criteria first, subtract misc5/4 length if flow
10090          * doesn't own misc5/4 parameters. In some old rdma-core releases,
10091          * misc5/4 are not supported, and matcher creation failure is expected
10092          * w/o subtration. If misc5 is provided, misc4 must be counted in since
10093          * misc5 is right after misc4.
10094          */
10095         if (!(match_criteria & (1 << MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT))) {
10096                 *size = MLX5_ST_SZ_BYTES(fte_match_param) -
10097                         MLX5_ST_SZ_BYTES(fte_match_set_misc5);
10098                 if (!(match_criteria & (1 <<
10099                         MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT))) {
10100                         *size -= MLX5_ST_SZ_BYTES(fte_match_set_misc4);
10101                 }
10102         }
10103 }
10104
10105 static struct mlx5_list_entry *
10106 flow_dv_matcher_clone_cb(void *tool_ctx __rte_unused,
10107                          struct mlx5_list_entry *entry, void *cb_ctx)
10108 {
10109         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10110         struct mlx5_flow_dv_matcher *ref = ctx->data;
10111         struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
10112                                                             typeof(*tbl), tbl);
10113         struct mlx5_flow_dv_matcher *resource = mlx5_malloc(MLX5_MEM_ANY,
10114                                                             sizeof(*resource),
10115                                                             0, SOCKET_ID_ANY);
10116
10117         if (!resource) {
10118                 rte_flow_error_set(ctx->error, ENOMEM,
10119                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10120                                    "cannot create matcher");
10121                 return NULL;
10122         }
10123         memcpy(resource, entry, sizeof(*resource));
10124         resource->tbl = &tbl->tbl;
10125         return &resource->entry;
10126 }
10127
10128 static void
10129 flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused,
10130                              struct mlx5_list_entry *entry)
10131 {
10132         mlx5_free(entry);
10133 }
10134
10135 struct mlx5_list_entry *
10136 flow_dv_tbl_create_cb(void *tool_ctx, void *cb_ctx)
10137 {
10138         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10139         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10140         struct rte_eth_dev *dev = ctx->dev;
10141         struct mlx5_flow_tbl_data_entry *tbl_data;
10142         struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data2;
10143         struct rte_flow_error *error = ctx->error;
10144         union mlx5_flow_tbl_key key = { .v64 = *(uint64_t *)(ctx->data) };
10145         struct mlx5_flow_tbl_resource *tbl;
10146         void *domain;
10147         uint32_t idx = 0;
10148         int ret;
10149
10150         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
10151         if (!tbl_data) {
10152                 rte_flow_error_set(error, ENOMEM,
10153                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10154                                    NULL,
10155                                    "cannot allocate flow table data entry");
10156                 return NULL;
10157         }
10158         tbl_data->idx = idx;
10159         tbl_data->tunnel = tt_prm->tunnel;
10160         tbl_data->group_id = tt_prm->group_id;
10161         tbl_data->external = !!tt_prm->external;
10162         tbl_data->tunnel_offload = is_tunnel_offload_active(dev);
10163         tbl_data->is_egress = !!key.is_egress;
10164         tbl_data->is_transfer = !!key.is_fdb;
10165         tbl_data->dummy = !!key.dummy;
10166         tbl_data->level = key.level;
10167         tbl_data->id = key.id;
10168         tbl = &tbl_data->tbl;
10169         if (key.dummy)
10170                 return &tbl_data->entry;
10171         if (key.is_fdb)
10172                 domain = sh->fdb_domain;
10173         else if (key.is_egress)
10174                 domain = sh->tx_domain;
10175         else
10176                 domain = sh->rx_domain;
10177         ret = mlx5_flow_os_create_flow_tbl(domain, key.level, &tbl->obj);
10178         if (ret) {
10179                 rte_flow_error_set(error, ENOMEM,
10180                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10181                                    NULL, "cannot create flow table object");
10182                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10183                 return NULL;
10184         }
10185         if (key.level != 0) {
10186                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
10187                                         (tbl->obj, &tbl_data->jump.action);
10188                 if (ret) {
10189                         rte_flow_error_set(error, ENOMEM,
10190                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10191                                            NULL,
10192                                            "cannot create flow jump action");
10193                         mlx5_flow_os_destroy_flow_tbl(tbl->obj);
10194                         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10195                         return NULL;
10196                 }
10197         }
10198         MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
10199               key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
10200               key.level, key.id);
10201         tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
10202                                               flow_dv_matcher_create_cb,
10203                                               flow_dv_matcher_match_cb,
10204                                               flow_dv_matcher_remove_cb,
10205                                               flow_dv_matcher_clone_cb,
10206                                               flow_dv_matcher_clone_free_cb);
10207         if (!tbl_data->matchers) {
10208                 rte_flow_error_set(error, ENOMEM,
10209                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10210                                    NULL,
10211                                    "cannot create tbl matcher list");
10212                 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
10213                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
10214                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10215                 return NULL;
10216         }
10217         return &tbl_data->entry;
10218 }
10219
10220 int
10221 flow_dv_tbl_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
10222                      void *cb_ctx)
10223 {
10224         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10225         struct mlx5_flow_tbl_data_entry *tbl_data =
10226                 container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10227         union mlx5_flow_tbl_key key = { .v64 =  *(uint64_t *)(ctx->data) };
10228
10229         return tbl_data->level != key.level ||
10230                tbl_data->id != key.id ||
10231                tbl_data->dummy != key.dummy ||
10232                tbl_data->is_transfer != !!key.is_fdb ||
10233                tbl_data->is_egress != !!key.is_egress;
10234 }
10235
10236 struct mlx5_list_entry *
10237 flow_dv_tbl_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
10238                       void *cb_ctx)
10239 {
10240         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10241         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10242         struct mlx5_flow_tbl_data_entry *tbl_data;
10243         struct rte_flow_error *error = ctx->error;
10244         uint32_t idx = 0;
10245
10246         tbl_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
10247         if (!tbl_data) {
10248                 rte_flow_error_set(error, ENOMEM,
10249                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10250                                    NULL,
10251                                    "cannot allocate flow table data entry");
10252                 return NULL;
10253         }
10254         memcpy(tbl_data, oentry, sizeof(*tbl_data));
10255         tbl_data->idx = idx;
10256         return &tbl_data->entry;
10257 }
10258
10259 void
10260 flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10261 {
10262         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10263         struct mlx5_flow_tbl_data_entry *tbl_data =
10264                     container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10265
10266         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
10267 }
10268
10269 /**
10270  * Get a flow table.
10271  *
10272  * @param[in, out] dev
10273  *   Pointer to rte_eth_dev structure.
10274  * @param[in] table_level
10275  *   Table level to use.
10276  * @param[in] egress
10277  *   Direction of the table.
10278  * @param[in] transfer
10279  *   E-Switch or NIC flow.
10280  * @param[in] dummy
10281  *   Dummy entry for dv API.
10282  * @param[in] table_id
10283  *   Table id to use.
10284  * @param[out] error
10285  *   pointer to error structure.
10286  *
10287  * @return
10288  *   Returns tables resource based on the index, NULL in case of failed.
10289  */
10290 struct mlx5_flow_tbl_resource *
10291 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
10292                          uint32_t table_level, uint8_t egress,
10293                          uint8_t transfer,
10294                          bool external,
10295                          const struct mlx5_flow_tunnel *tunnel,
10296                          uint32_t group_id, uint8_t dummy,
10297                          uint32_t table_id,
10298                          struct rte_flow_error *error)
10299 {
10300         struct mlx5_priv *priv = dev->data->dev_private;
10301         union mlx5_flow_tbl_key table_key = {
10302                 {
10303                         .level = table_level,
10304                         .id = table_id,
10305                         .reserved = 0,
10306                         .dummy = !!dummy,
10307                         .is_fdb = !!transfer,
10308                         .is_egress = !!egress,
10309                 }
10310         };
10311         struct mlx5_flow_tbl_tunnel_prm tt_prm = {
10312                 .tunnel = tunnel,
10313                 .group_id = group_id,
10314                 .external = external,
10315         };
10316         struct mlx5_flow_cb_ctx ctx = {
10317                 .dev = dev,
10318                 .error = error,
10319                 .data = &table_key.v64,
10320                 .data2 = &tt_prm,
10321         };
10322         struct mlx5_list_entry *entry;
10323         struct mlx5_flow_tbl_data_entry *tbl_data;
10324
10325         entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx);
10326         if (!entry) {
10327                 rte_flow_error_set(error, ENOMEM,
10328                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10329                                    "cannot get table");
10330                 return NULL;
10331         }
10332         DRV_LOG(DEBUG, "table_level %u table_id %u "
10333                 "tunnel %u group %u registered.",
10334                 table_level, table_id,
10335                 tunnel ? tunnel->tunnel_id : 0, group_id);
10336         tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10337         return &tbl_data->tbl;
10338 }
10339
10340 void
10341 flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10342 {
10343         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10344         struct mlx5_flow_tbl_data_entry *tbl_data =
10345                     container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10346
10347         MLX5_ASSERT(entry && sh);
10348         if (tbl_data->jump.action)
10349                 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
10350         if (tbl_data->tbl.obj)
10351                 mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj);
10352         if (tbl_data->tunnel_offload && tbl_data->external) {
10353                 struct mlx5_list_entry *he;
10354                 struct mlx5_hlist *tunnel_grp_hash;
10355                 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
10356                 union tunnel_tbl_key tunnel_key = {
10357                         .tunnel_id = tbl_data->tunnel ?
10358                                         tbl_data->tunnel->tunnel_id : 0,
10359                         .group = tbl_data->group_id
10360                 };
10361                 uint32_t table_level = tbl_data->level;
10362                 struct mlx5_flow_cb_ctx ctx = {
10363                         .data = (void *)&tunnel_key.val,
10364                 };
10365
10366                 tunnel_grp_hash = tbl_data->tunnel ?
10367                                         tbl_data->tunnel->groups :
10368                                         thub->groups;
10369                 he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, &ctx);
10370                 if (he)
10371                         mlx5_hlist_unregister(tunnel_grp_hash, he);
10372                 DRV_LOG(DEBUG,
10373                         "table_level %u id %u tunnel %u group %u released.",
10374                         table_level,
10375                         tbl_data->id,
10376                         tbl_data->tunnel ?
10377                         tbl_data->tunnel->tunnel_id : 0,
10378                         tbl_data->group_id);
10379         }
10380         mlx5_list_destroy(tbl_data->matchers);
10381         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
10382 }
10383
10384 /**
10385  * Release a flow table.
10386  *
10387  * @param[in] sh
10388  *   Pointer to device shared structure.
10389  * @param[in] tbl
10390  *   Table resource to be released.
10391  *
10392  * @return
10393  *   Returns 0 if table was released, else return 1;
10394  */
10395 static int
10396 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
10397                              struct mlx5_flow_tbl_resource *tbl)
10398 {
10399         struct mlx5_flow_tbl_data_entry *tbl_data =
10400                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10401
10402         if (!tbl)
10403                 return 0;
10404         return mlx5_hlist_unregister(sh->flow_tbls, &tbl_data->entry);
10405 }
10406
10407 int
10408 flow_dv_matcher_match_cb(void *tool_ctx __rte_unused,
10409                          struct mlx5_list_entry *entry, void *cb_ctx)
10410 {
10411         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10412         struct mlx5_flow_dv_matcher *ref = ctx->data;
10413         struct mlx5_flow_dv_matcher *cur = container_of(entry, typeof(*cur),
10414                                                         entry);
10415
10416         return cur->crc != ref->crc ||
10417                cur->priority != ref->priority ||
10418                memcmp((const void *)cur->mask.buf,
10419                       (const void *)ref->mask.buf, ref->mask.size);
10420 }
10421
10422 struct mlx5_list_entry *
10423 flow_dv_matcher_create_cb(void *tool_ctx, void *cb_ctx)
10424 {
10425         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10426         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10427         struct mlx5_flow_dv_matcher *ref = ctx->data;
10428         struct mlx5_flow_dv_matcher *resource;
10429         struct mlx5dv_flow_matcher_attr dv_attr = {
10430                 .type = IBV_FLOW_ATTR_NORMAL,
10431                 .match_mask = (void *)&ref->mask,
10432         };
10433         struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
10434                                                             typeof(*tbl), tbl);
10435         int ret;
10436
10437         resource = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*resource), 0,
10438                                SOCKET_ID_ANY);
10439         if (!resource) {
10440                 rte_flow_error_set(ctx->error, ENOMEM,
10441                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10442                                    "cannot create matcher");
10443                 return NULL;
10444         }
10445         *resource = *ref;
10446         dv_attr.match_criteria_enable =
10447                 flow_dv_matcher_enable(resource->mask.buf);
10448         __flow_dv_adjust_buf_size(&ref->mask.size,
10449                                   dv_attr.match_criteria_enable);
10450         dv_attr.priority = ref->priority;
10451         if (tbl->is_egress)
10452                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
10453         ret = mlx5_flow_os_create_flow_matcher(sh->cdev->ctx, &dv_attr,
10454                                                tbl->tbl.obj,
10455                                                &resource->matcher_object);
10456         if (ret) {
10457                 mlx5_free(resource);
10458                 rte_flow_error_set(ctx->error, ENOMEM,
10459                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10460                                    "cannot create matcher");
10461                 return NULL;
10462         }
10463         return &resource->entry;
10464 }
10465
10466 /**
10467  * Register the flow matcher.
10468  *
10469  * @param[in, out] dev
10470  *   Pointer to rte_eth_dev structure.
10471  * @param[in, out] matcher
10472  *   Pointer to flow matcher.
10473  * @param[in, out] key
10474  *   Pointer to flow table key.
10475  * @parm[in, out] dev_flow
10476  *   Pointer to the dev_flow.
10477  * @param[out] error
10478  *   pointer to error structure.
10479  *
10480  * @return
10481  *   0 on success otherwise -errno and errno is set.
10482  */
10483 static int
10484 flow_dv_matcher_register(struct rte_eth_dev *dev,
10485                          struct mlx5_flow_dv_matcher *ref,
10486                          union mlx5_flow_tbl_key *key,
10487                          struct mlx5_flow *dev_flow,
10488                          const struct mlx5_flow_tunnel *tunnel,
10489                          uint32_t group_id,
10490                          struct rte_flow_error *error)
10491 {
10492         struct mlx5_list_entry *entry;
10493         struct mlx5_flow_dv_matcher *resource;
10494         struct mlx5_flow_tbl_resource *tbl;
10495         struct mlx5_flow_tbl_data_entry *tbl_data;
10496         struct mlx5_flow_cb_ctx ctx = {
10497                 .error = error,
10498                 .data = ref,
10499         };
10500         /**
10501          * tunnel offload API requires this registration for cases when
10502          * tunnel match rule was inserted before tunnel set rule.
10503          */
10504         tbl = flow_dv_tbl_resource_get(dev, key->level,
10505                                        key->is_egress, key->is_fdb,
10506                                        dev_flow->external, tunnel,
10507                                        group_id, 0, key->id, error);
10508         if (!tbl)
10509                 return -rte_errno;      /* No need to refill the error info */
10510         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10511         ref->tbl = tbl;
10512         entry = mlx5_list_register(tbl_data->matchers, &ctx);
10513         if (!entry) {
10514                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
10515                 return rte_flow_error_set(error, ENOMEM,
10516                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10517                                           "cannot allocate ref memory");
10518         }
10519         resource = container_of(entry, typeof(*resource), entry);
10520         dev_flow->handle->dvh.matcher = resource;
10521         return 0;
10522 }
10523
10524 struct mlx5_list_entry *
10525 flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx)
10526 {
10527         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10528         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10529         struct mlx5_flow_dv_tag_resource *entry;
10530         uint32_t idx = 0;
10531         int ret;
10532
10533         entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
10534         if (!entry) {
10535                 rte_flow_error_set(ctx->error, ENOMEM,
10536                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10537                                    "cannot allocate resource memory");
10538                 return NULL;
10539         }
10540         entry->idx = idx;
10541         entry->tag_id = *(uint32_t *)(ctx->data);
10542         ret = mlx5_flow_os_create_flow_action_tag(entry->tag_id,
10543                                                   &entry->action);
10544         if (ret) {
10545                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx);
10546                 rte_flow_error_set(ctx->error, ENOMEM,
10547                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10548                                    NULL, "cannot create action");
10549                 return NULL;
10550         }
10551         return &entry->entry;
10552 }
10553
10554 int
10555 flow_dv_tag_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
10556                      void *cb_ctx)
10557 {
10558         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10559         struct mlx5_flow_dv_tag_resource *tag =
10560                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10561
10562         return *(uint32_t *)(ctx->data) != tag->tag_id;
10563 }
10564
10565 struct mlx5_list_entry *
10566 flow_dv_tag_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
10567                      void *cb_ctx)
10568 {
10569         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10570         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10571         struct mlx5_flow_dv_tag_resource *entry;
10572         uint32_t idx = 0;
10573
10574         entry = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
10575         if (!entry) {
10576                 rte_flow_error_set(ctx->error, ENOMEM,
10577                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10578                                    "cannot allocate tag resource memory");
10579                 return NULL;
10580         }
10581         memcpy(entry, oentry, sizeof(*entry));
10582         entry->idx = idx;
10583         return &entry->entry;
10584 }
10585
10586 void
10587 flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10588 {
10589         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10590         struct mlx5_flow_dv_tag_resource *tag =
10591                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10592
10593         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
10594 }
10595
10596 /**
10597  * Find existing tag resource or create and register a new one.
10598  *
10599  * @param dev[in, out]
10600  *   Pointer to rte_eth_dev structure.
10601  * @param[in, out] tag_be24
10602  *   Tag value in big endian then R-shift 8.
10603  * @parm[in, out] dev_flow
10604  *   Pointer to the dev_flow.
10605  * @param[out] error
10606  *   pointer to error structure.
10607  *
10608  * @return
10609  *   0 on success otherwise -errno and errno is set.
10610  */
10611 static int
10612 flow_dv_tag_resource_register
10613                         (struct rte_eth_dev *dev,
10614                          uint32_t tag_be24,
10615                          struct mlx5_flow *dev_flow,
10616                          struct rte_flow_error *error)
10617 {
10618         struct mlx5_priv *priv = dev->data->dev_private;
10619         struct mlx5_flow_dv_tag_resource *resource;
10620         struct mlx5_list_entry *entry;
10621         struct mlx5_flow_cb_ctx ctx = {
10622                                         .error = error,
10623                                         .data = &tag_be24,
10624                                         };
10625         struct mlx5_hlist *tag_table;
10626
10627         tag_table = flow_dv_hlist_prepare(priv->sh, &priv->sh->tag_table,
10628                                       "tags",
10629                                       MLX5_TAGS_HLIST_ARRAY_SIZE,
10630                                       false, false, priv->sh,
10631                                       flow_dv_tag_create_cb,
10632                                       flow_dv_tag_match_cb,
10633                                       flow_dv_tag_remove_cb,
10634                                       flow_dv_tag_clone_cb,
10635                                       flow_dv_tag_clone_free_cb);
10636         if (unlikely(!tag_table))
10637                 return -rte_errno;
10638         entry = mlx5_hlist_register(tag_table, tag_be24, &ctx);
10639         if (entry) {
10640                 resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
10641                                         entry);
10642                 dev_flow->handle->dvh.rix_tag = resource->idx;
10643                 dev_flow->dv.tag_resource = resource;
10644                 return 0;
10645         }
10646         return -rte_errno;
10647 }
10648
10649 void
10650 flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10651 {
10652         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10653         struct mlx5_flow_dv_tag_resource *tag =
10654                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10655
10656         MLX5_ASSERT(tag && sh && tag->action);
10657         claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
10658         DRV_LOG(DEBUG, "Tag %p: removed.", (void *)tag);
10659         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
10660 }
10661
10662 /**
10663  * Release the tag.
10664  *
10665  * @param dev
10666  *   Pointer to Ethernet device.
10667  * @param tag_idx
10668  *   Tag index.
10669  *
10670  * @return
10671  *   1 while a reference on it exists, 0 when freed.
10672  */
10673 static int
10674 flow_dv_tag_release(struct rte_eth_dev *dev,
10675                     uint32_t tag_idx)
10676 {
10677         struct mlx5_priv *priv = dev->data->dev_private;
10678         struct mlx5_flow_dv_tag_resource *tag;
10679
10680         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
10681         if (!tag)
10682                 return 0;
10683         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
10684                 dev->data->port_id, (void *)tag, tag->entry.ref_cnt);
10685         return mlx5_hlist_unregister(priv->sh->tag_table, &tag->entry);
10686 }
10687
10688 /**
10689  * Translate action PORT_ID / REPRESENTED_PORT to vport.
10690  *
10691  * @param[in] dev
10692  *   Pointer to rte_eth_dev structure.
10693  * @param[in] action
10694  *   Pointer to action PORT_ID / REPRESENTED_PORT.
10695  * @param[out] dst_port_id
10696  *   The target port ID.
10697  * @param[out] error
10698  *   Pointer to the error structure.
10699  *
10700  * @return
10701  *   0 on success, a negative errno value otherwise and rte_errno is set.
10702  */
10703 static int
10704 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
10705                                  const struct rte_flow_action *action,
10706                                  uint32_t *dst_port_id,
10707                                  struct rte_flow_error *error)
10708 {
10709         uint32_t port;
10710         struct mlx5_priv *priv;
10711
10712         switch (action->type) {
10713         case RTE_FLOW_ACTION_TYPE_PORT_ID: {
10714                 const struct rte_flow_action_port_id *conf;
10715
10716                 conf = (const struct rte_flow_action_port_id *)action->conf;
10717                 port = conf->original ? dev->data->port_id : conf->id;
10718                 break;
10719         }
10720         case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT: {
10721                 const struct rte_flow_action_ethdev *ethdev;
10722
10723                 ethdev = (const struct rte_flow_action_ethdev *)action->conf;
10724                 port = ethdev->port_id;
10725                 break;
10726         }
10727         default:
10728                 MLX5_ASSERT(false);
10729                 return rte_flow_error_set(error, EINVAL,
10730                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
10731                                           "unknown E-Switch action");
10732         }
10733
10734         priv = mlx5_port_to_eswitch_info(port, false);
10735         if (!priv)
10736                 return rte_flow_error_set(error, -rte_errno,
10737                                           RTE_FLOW_ERROR_TYPE_ACTION,
10738                                           NULL,
10739                                           "No eswitch info was found for port");
10740 #ifdef HAVE_MLX5DV_DR_CREATE_DEST_IB_PORT
10741         /*
10742          * This parameter is transferred to
10743          * mlx5dv_dr_action_create_dest_ib_port().
10744          */
10745         *dst_port_id = priv->dev_port;
10746 #else
10747         /*
10748          * Legacy mode, no LAG configurations is supported.
10749          * This parameter is transferred to
10750          * mlx5dv_dr_action_create_dest_vport().
10751          */
10752         *dst_port_id = priv->vport_id;
10753 #endif
10754         return 0;
10755 }
10756
10757 /**
10758  * Create a counter with aging configuration.
10759  *
10760  * @param[in] dev
10761  *   Pointer to rte_eth_dev structure.
10762  * @param[in] dev_flow
10763  *   Pointer to the mlx5_flow.
10764  * @param[out] count
10765  *   Pointer to the counter action configuration.
10766  * @param[in] age
10767  *   Pointer to the aging action configuration.
10768  *
10769  * @return
10770  *   Index to flow counter on success, 0 otherwise.
10771  */
10772 static uint32_t
10773 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
10774                                 struct mlx5_flow *dev_flow,
10775                                 const struct rte_flow_action_count *count
10776                                         __rte_unused,
10777                                 const struct rte_flow_action_age *age)
10778 {
10779         uint32_t counter;
10780         struct mlx5_age_param *age_param;
10781
10782         counter = flow_dv_counter_alloc(dev, !!age);
10783         if (!counter || age == NULL)
10784                 return counter;
10785         age_param = flow_dv_counter_idx_get_age(dev, counter);
10786         age_param->context = age->context ? age->context :
10787                 (void *)(uintptr_t)(dev_flow->flow_idx);
10788         age_param->timeout = age->timeout;
10789         age_param->port_id = dev->data->port_id;
10790         __atomic_store_n(&age_param->sec_since_last_hit, 0, __ATOMIC_RELAXED);
10791         __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED);
10792         return counter;
10793 }
10794
10795 /**
10796  * Add Tx queue matcher
10797  *
10798  * @param[in] dev
10799  *   Pointer to the dev struct.
10800  * @param[in, out] matcher
10801  *   Flow matcher.
10802  * @param[in, out] key
10803  *   Flow matcher value.
10804  * @param[in] item
10805  *   Flow pattern to translate.
10806  * @param[in] inner
10807  *   Item is inner pattern.
10808  */
10809 static void
10810 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
10811                                 void *matcher, void *key,
10812                                 const struct rte_flow_item *item)
10813 {
10814         const struct mlx5_rte_flow_item_tx_queue *queue_m;
10815         const struct mlx5_rte_flow_item_tx_queue *queue_v;
10816         void *misc_m =
10817                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
10818         void *misc_v =
10819                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
10820         struct mlx5_txq_ctrl *txq;
10821         uint32_t queue, mask;
10822
10823         queue_m = (const void *)item->mask;
10824         queue_v = (const void *)item->spec;
10825         if (!queue_v)
10826                 return;
10827         txq = mlx5_txq_get(dev, queue_v->queue);
10828         if (!txq)
10829                 return;
10830         if (txq->type == MLX5_TXQ_TYPE_HAIRPIN)
10831                 queue = txq->obj->sq->id;
10832         else
10833                 queue = txq->obj->sq_obj.sq->id;
10834         mask = queue_m == NULL ? UINT32_MAX : queue_m->queue;
10835         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, mask);
10836         MLX5_SET(fte_match_set_misc, misc_v, source_sqn, queue & mask);
10837         mlx5_txq_release(dev, queue_v->queue);
10838 }
10839
10840 /**
10841  * Set the hash fields according to the @p flow information.
10842  *
10843  * @param[in] dev_flow
10844  *   Pointer to the mlx5_flow.
10845  * @param[in] rss_desc
10846  *   Pointer to the mlx5_flow_rss_desc.
10847  */
10848 static void
10849 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
10850                        struct mlx5_flow_rss_desc *rss_desc)
10851 {
10852         uint64_t items = dev_flow->handle->layers;
10853         int rss_inner = 0;
10854         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
10855
10856         dev_flow->hash_fields = 0;
10857 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
10858         if (rss_desc->level >= 2)
10859                 rss_inner = 1;
10860 #endif
10861         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
10862             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
10863                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
10864                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
10865                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
10866                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
10867                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
10868                         else
10869                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
10870                 }
10871         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
10872                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
10873                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
10874                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
10875                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
10876                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
10877                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
10878                         else
10879                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
10880                 }
10881         }
10882         if (dev_flow->hash_fields == 0)
10883                 /*
10884                  * There is no match between the RSS types and the
10885                  * L3 protocol (IPv4/IPv6) defined in the flow rule.
10886                  */
10887                 return;
10888         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
10889             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
10890                 if (rss_types & ETH_RSS_UDP) {
10891                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
10892                                 dev_flow->hash_fields |=
10893                                                 IBV_RX_HASH_SRC_PORT_UDP;
10894                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
10895                                 dev_flow->hash_fields |=
10896                                                 IBV_RX_HASH_DST_PORT_UDP;
10897                         else
10898                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
10899                 }
10900         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
10901                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
10902                 if (rss_types & ETH_RSS_TCP) {
10903                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
10904                                 dev_flow->hash_fields |=
10905                                                 IBV_RX_HASH_SRC_PORT_TCP;
10906                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
10907                                 dev_flow->hash_fields |=
10908                                                 IBV_RX_HASH_DST_PORT_TCP;
10909                         else
10910                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
10911                 }
10912         }
10913         if (rss_inner)
10914                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
10915 }
10916
10917 /**
10918  * Prepare an Rx Hash queue.
10919  *
10920  * @param dev
10921  *   Pointer to Ethernet device.
10922  * @param[in] dev_flow
10923  *   Pointer to the mlx5_flow.
10924  * @param[in] rss_desc
10925  *   Pointer to the mlx5_flow_rss_desc.
10926  * @param[out] hrxq_idx
10927  *   Hash Rx queue index.
10928  *
10929  * @return
10930  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
10931  */
10932 static struct mlx5_hrxq *
10933 flow_dv_hrxq_prepare(struct rte_eth_dev *dev,
10934                      struct mlx5_flow *dev_flow,
10935                      struct mlx5_flow_rss_desc *rss_desc,
10936                      uint32_t *hrxq_idx)
10937 {
10938         struct mlx5_priv *priv = dev->data->dev_private;
10939         struct mlx5_flow_handle *dh = dev_flow->handle;
10940         struct mlx5_hrxq *hrxq;
10941
10942         MLX5_ASSERT(rss_desc->queue_num);
10943         rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN;
10944         rss_desc->hash_fields = dev_flow->hash_fields;
10945         rss_desc->tunnel = !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL);
10946         rss_desc->shared_rss = 0;
10947         if (rss_desc->hash_fields == 0)
10948                 rss_desc->queue_num = 1;
10949         *hrxq_idx = mlx5_hrxq_get(dev, rss_desc);
10950         if (!*hrxq_idx)
10951                 return NULL;
10952         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
10953                               *hrxq_idx);
10954         return hrxq;
10955 }
10956
10957 /**
10958  * Release sample sub action resource.
10959  *
10960  * @param[in, out] dev
10961  *   Pointer to rte_eth_dev structure.
10962  * @param[in] act_res
10963  *   Pointer to sample sub action resource.
10964  */
10965 static void
10966 flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
10967                                    struct mlx5_flow_sub_actions_idx *act_res)
10968 {
10969         if (act_res->rix_hrxq) {
10970                 mlx5_hrxq_release(dev, act_res->rix_hrxq);
10971                 act_res->rix_hrxq = 0;
10972         }
10973         if (act_res->rix_encap_decap) {
10974                 flow_dv_encap_decap_resource_release(dev,
10975                                                      act_res->rix_encap_decap);
10976                 act_res->rix_encap_decap = 0;
10977         }
10978         if (act_res->rix_port_id_action) {
10979                 flow_dv_port_id_action_resource_release(dev,
10980                                                 act_res->rix_port_id_action);
10981                 act_res->rix_port_id_action = 0;
10982         }
10983         if (act_res->rix_tag) {
10984                 flow_dv_tag_release(dev, act_res->rix_tag);
10985                 act_res->rix_tag = 0;
10986         }
10987         if (act_res->rix_jump) {
10988                 flow_dv_jump_tbl_resource_release(dev, act_res->rix_jump);
10989                 act_res->rix_jump = 0;
10990         }
10991 }
10992
10993 int
10994 flow_dv_sample_match_cb(void *tool_ctx __rte_unused,
10995                         struct mlx5_list_entry *entry, void *cb_ctx)
10996 {
10997         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10998         struct rte_eth_dev *dev = ctx->dev;
10999         struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
11000         struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
11001                                                               typeof(*resource),
11002                                                               entry);
11003
11004         if (ctx_resource->ratio == resource->ratio &&
11005             ctx_resource->ft_type == resource->ft_type &&
11006             ctx_resource->ft_id == resource->ft_id &&
11007             ctx_resource->set_action == resource->set_action &&
11008             !memcmp((void *)&ctx_resource->sample_act,
11009                     (void *)&resource->sample_act,
11010                     sizeof(struct mlx5_flow_sub_actions_list))) {
11011                 /*
11012                  * Existing sample action should release the prepared
11013                  * sub-actions reference counter.
11014                  */
11015                 flow_dv_sample_sub_actions_release(dev,
11016                                                    &ctx_resource->sample_idx);
11017                 return 0;
11018         }
11019         return 1;
11020 }
11021
11022 struct mlx5_list_entry *
11023 flow_dv_sample_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
11024 {
11025         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11026         struct rte_eth_dev *dev = ctx->dev;
11027         struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
11028         void **sample_dv_actions = ctx_resource->sub_actions;
11029         struct mlx5_flow_dv_sample_resource *resource;
11030         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
11031         struct mlx5_priv *priv = dev->data->dev_private;
11032         struct mlx5_dev_ctx_shared *sh = priv->sh;
11033         struct mlx5_flow_tbl_resource *tbl;
11034         uint32_t idx = 0;
11035         const uint32_t next_ft_step = 1;
11036         uint32_t next_ft_id = ctx_resource->ft_id + next_ft_step;
11037         uint8_t is_egress = 0;
11038         uint8_t is_transfer = 0;
11039         struct rte_flow_error *error = ctx->error;
11040
11041         /* Register new sample resource. */
11042         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
11043         if (!resource) {
11044                 rte_flow_error_set(error, ENOMEM,
11045                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11046                                           NULL,
11047                                           "cannot allocate resource memory");
11048                 return NULL;
11049         }
11050         *resource = *ctx_resource;
11051         /* Create normal path table level */
11052         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
11053                 is_transfer = 1;
11054         else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
11055                 is_egress = 1;
11056         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
11057                                         is_egress, is_transfer,
11058                                         true, NULL, 0, 0, 0, error);
11059         if (!tbl) {
11060                 rte_flow_error_set(error, ENOMEM,
11061                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11062                                           NULL,
11063                                           "fail to create normal path table "
11064                                           "for sample");
11065                 goto error;
11066         }
11067         resource->normal_path_tbl = tbl;
11068         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
11069                 if (!sh->default_miss_action) {
11070                         rte_flow_error_set(error, ENOMEM,
11071                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11072                                                 NULL,
11073                                                 "default miss action was not "
11074                                                 "created");
11075                         goto error;
11076                 }
11077                 sample_dv_actions[ctx_resource->sample_act.actions_num++] =
11078                                                 sh->default_miss_action;
11079         }
11080         /* Create a DR sample action */
11081         sampler_attr.sample_ratio = resource->ratio;
11082         sampler_attr.default_next_table = tbl->obj;
11083         sampler_attr.num_sample_actions = ctx_resource->sample_act.actions_num;
11084         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
11085                                                         &sample_dv_actions[0];
11086         sampler_attr.action = resource->set_action;
11087         if (mlx5_os_flow_dr_create_flow_action_sampler
11088                         (&sampler_attr, &resource->verbs_action)) {
11089                 rte_flow_error_set(error, ENOMEM,
11090                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11091                                         NULL, "cannot create sample action");
11092                 goto error;
11093         }
11094         resource->idx = idx;
11095         resource->dev = dev;
11096         return &resource->entry;
11097 error:
11098         if (resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
11099                 flow_dv_sample_sub_actions_release(dev,
11100                                                    &resource->sample_idx);
11101         if (resource->normal_path_tbl)
11102                 flow_dv_tbl_resource_release(MLX5_SH(dev),
11103                                 resource->normal_path_tbl);
11104         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx);
11105         return NULL;
11106
11107 }
11108
11109 struct mlx5_list_entry *
11110 flow_dv_sample_clone_cb(void *tool_ctx __rte_unused,
11111                          struct mlx5_list_entry *entry __rte_unused,
11112                          void *cb_ctx)
11113 {
11114         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11115         struct rte_eth_dev *dev = ctx->dev;
11116         struct mlx5_flow_dv_sample_resource *resource;
11117         struct mlx5_priv *priv = dev->data->dev_private;
11118         struct mlx5_dev_ctx_shared *sh = priv->sh;
11119         uint32_t idx = 0;
11120
11121         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
11122         if (!resource) {
11123                 rte_flow_error_set(ctx->error, ENOMEM,
11124                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11125                                           NULL,
11126                                           "cannot allocate resource memory");
11127                 return NULL;
11128         }
11129         memcpy(resource, entry, sizeof(*resource));
11130         resource->idx = idx;
11131         resource->dev = dev;
11132         return &resource->entry;
11133 }
11134
11135 void
11136 flow_dv_sample_clone_free_cb(void *tool_ctx __rte_unused,
11137                              struct mlx5_list_entry *entry)
11138 {
11139         struct mlx5_flow_dv_sample_resource *resource =
11140                                   container_of(entry, typeof(*resource), entry);
11141         struct rte_eth_dev *dev = resource->dev;
11142         struct mlx5_priv *priv = dev->data->dev_private;
11143
11144         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
11145 }
11146
11147 /**
11148  * Find existing sample resource or create and register a new one.
11149  *
11150  * @param[in, out] dev
11151  *   Pointer to rte_eth_dev structure.
11152  * @param[in] ref
11153  *   Pointer to sample resource reference.
11154  * @parm[in, out] dev_flow
11155  *   Pointer to the dev_flow.
11156  * @param[out] error
11157  *   pointer to error structure.
11158  *
11159  * @return
11160  *   0 on success otherwise -errno and errno is set.
11161  */
11162 static int
11163 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
11164                          struct mlx5_flow_dv_sample_resource *ref,
11165                          struct mlx5_flow *dev_flow,
11166                          struct rte_flow_error *error)
11167 {
11168         struct mlx5_flow_dv_sample_resource *resource;
11169         struct mlx5_list_entry *entry;
11170         struct mlx5_priv *priv = dev->data->dev_private;
11171         struct mlx5_flow_cb_ctx ctx = {
11172                 .dev = dev,
11173                 .error = error,
11174                 .data = ref,
11175         };
11176
11177         entry = mlx5_list_register(priv->sh->sample_action_list, &ctx);
11178         if (!entry)
11179                 return -rte_errno;
11180         resource = container_of(entry, typeof(*resource), entry);
11181         dev_flow->handle->dvh.rix_sample = resource->idx;
11182         dev_flow->dv.sample_res = resource;
11183         return 0;
11184 }
11185
11186 int
11187 flow_dv_dest_array_match_cb(void *tool_ctx __rte_unused,
11188                             struct mlx5_list_entry *entry, void *cb_ctx)
11189 {
11190         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11191         struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
11192         struct rte_eth_dev *dev = ctx->dev;
11193         struct mlx5_flow_dv_dest_array_resource *resource =
11194                                   container_of(entry, typeof(*resource), entry);
11195         uint32_t idx = 0;
11196
11197         if (ctx_resource->num_of_dest == resource->num_of_dest &&
11198             ctx_resource->ft_type == resource->ft_type &&
11199             !memcmp((void *)resource->sample_act,
11200                     (void *)ctx_resource->sample_act,
11201                    (ctx_resource->num_of_dest *
11202                    sizeof(struct mlx5_flow_sub_actions_list)))) {
11203                 /*
11204                  * Existing sample action should release the prepared
11205                  * sub-actions reference counter.
11206                  */
11207                 for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
11208                         flow_dv_sample_sub_actions_release(dev,
11209                                         &ctx_resource->sample_idx[idx]);
11210                 return 0;
11211         }
11212         return 1;
11213 }
11214
11215 struct mlx5_list_entry *
11216 flow_dv_dest_array_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
11217 {
11218         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11219         struct rte_eth_dev *dev = ctx->dev;
11220         struct mlx5_flow_dv_dest_array_resource *resource;
11221         struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
11222         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
11223         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
11224         struct mlx5_priv *priv = dev->data->dev_private;
11225         struct mlx5_dev_ctx_shared *sh = priv->sh;
11226         struct mlx5_flow_sub_actions_list *sample_act;
11227         struct mlx5dv_dr_domain *domain;
11228         uint32_t idx = 0, res_idx = 0;
11229         struct rte_flow_error *error = ctx->error;
11230         uint64_t action_flags;
11231         int ret;
11232
11233         /* Register new destination array resource. */
11234         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
11235                                             &res_idx);
11236         if (!resource) {
11237                 rte_flow_error_set(error, ENOMEM,
11238                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11239                                           NULL,
11240                                           "cannot allocate resource memory");
11241                 return NULL;
11242         }
11243         *resource = *ctx_resource;
11244         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
11245                 domain = sh->fdb_domain;
11246         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
11247                 domain = sh->rx_domain;
11248         else
11249                 domain = sh->tx_domain;
11250         for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
11251                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
11252                                  mlx5_malloc(MLX5_MEM_ZERO,
11253                                  sizeof(struct mlx5dv_dr_action_dest_attr),
11254                                  0, SOCKET_ID_ANY);
11255                 if (!dest_attr[idx]) {
11256                         rte_flow_error_set(error, ENOMEM,
11257                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11258                                            NULL,
11259                                            "cannot allocate resource memory");
11260                         goto error;
11261                 }
11262                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
11263                 sample_act = &ctx_resource->sample_act[idx];
11264                 action_flags = sample_act->action_flags;
11265                 switch (action_flags) {
11266                 case MLX5_FLOW_ACTION_QUEUE:
11267                         dest_attr[idx]->dest = sample_act->dr_queue_action;
11268                         break;
11269                 case (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP):
11270                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
11271                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
11272                         dest_attr[idx]->dest_reformat->reformat =
11273                                         sample_act->dr_encap_action;
11274                         dest_attr[idx]->dest_reformat->dest =
11275                                         sample_act->dr_port_id_action;
11276                         break;
11277                 case MLX5_FLOW_ACTION_PORT_ID:
11278                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
11279                         break;
11280                 case MLX5_FLOW_ACTION_JUMP:
11281                         dest_attr[idx]->dest = sample_act->dr_jump_action;
11282                         break;
11283                 default:
11284                         rte_flow_error_set(error, EINVAL,
11285                                            RTE_FLOW_ERROR_TYPE_ACTION,
11286                                            NULL,
11287                                            "unsupported actions type");
11288                         goto error;
11289                 }
11290         }
11291         /* create a dest array actioin */
11292         ret = mlx5_os_flow_dr_create_flow_action_dest_array
11293                                                 (domain,
11294                                                  resource->num_of_dest,
11295                                                  dest_attr,
11296                                                  &resource->action);
11297         if (ret) {
11298                 rte_flow_error_set(error, ENOMEM,
11299                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11300                                    NULL,
11301                                    "cannot create destination array action");
11302                 goto error;
11303         }
11304         resource->idx = res_idx;
11305         resource->dev = dev;
11306         for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
11307                 mlx5_free(dest_attr[idx]);
11308         return &resource->entry;
11309 error:
11310         for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
11311                 flow_dv_sample_sub_actions_release(dev,
11312                                                    &resource->sample_idx[idx]);
11313                 if (dest_attr[idx])
11314                         mlx5_free(dest_attr[idx]);
11315         }
11316         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx);
11317         return NULL;
11318 }
11319
11320 struct mlx5_list_entry *
11321 flow_dv_dest_array_clone_cb(void *tool_ctx __rte_unused,
11322                             struct mlx5_list_entry *entry __rte_unused,
11323                             void *cb_ctx)
11324 {
11325         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11326         struct rte_eth_dev *dev = ctx->dev;
11327         struct mlx5_flow_dv_dest_array_resource *resource;
11328         struct mlx5_priv *priv = dev->data->dev_private;
11329         struct mlx5_dev_ctx_shared *sh = priv->sh;
11330         uint32_t res_idx = 0;
11331         struct rte_flow_error *error = ctx->error;
11332
11333         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
11334                                       &res_idx);
11335         if (!resource) {
11336                 rte_flow_error_set(error, ENOMEM,
11337                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11338                                           NULL,
11339                                           "cannot allocate dest-array memory");
11340                 return NULL;
11341         }
11342         memcpy(resource, entry, sizeof(*resource));
11343         resource->idx = res_idx;
11344         resource->dev = dev;
11345         return &resource->entry;
11346 }
11347
11348 void
11349 flow_dv_dest_array_clone_free_cb(void *tool_ctx __rte_unused,
11350                                  struct mlx5_list_entry *entry)
11351 {
11352         struct mlx5_flow_dv_dest_array_resource *resource =
11353                         container_of(entry, typeof(*resource), entry);
11354         struct rte_eth_dev *dev = resource->dev;
11355         struct mlx5_priv *priv = dev->data->dev_private;
11356
11357         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
11358 }
11359
11360 /**
11361  * Find existing destination array resource or create and register a new one.
11362  *
11363  * @param[in, out] dev
11364  *   Pointer to rte_eth_dev structure.
11365  * @param[in] ref
11366  *   Pointer to destination array resource reference.
11367  * @parm[in, out] dev_flow
11368  *   Pointer to the dev_flow.
11369  * @param[out] error
11370  *   pointer to error structure.
11371  *
11372  * @return
11373  *   0 on success otherwise -errno and errno is set.
11374  */
11375 static int
11376 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
11377                          struct mlx5_flow_dv_dest_array_resource *ref,
11378                          struct mlx5_flow *dev_flow,
11379                          struct rte_flow_error *error)
11380 {
11381         struct mlx5_flow_dv_dest_array_resource *resource;
11382         struct mlx5_priv *priv = dev->data->dev_private;
11383         struct mlx5_list_entry *entry;
11384         struct mlx5_flow_cb_ctx ctx = {
11385                 .dev = dev,
11386                 .error = error,
11387                 .data = ref,
11388         };
11389
11390         entry = mlx5_list_register(priv->sh->dest_array_list, &ctx);
11391         if (!entry)
11392                 return -rte_errno;
11393         resource = container_of(entry, typeof(*resource), entry);
11394         dev_flow->handle->dvh.rix_dest_array = resource->idx;
11395         dev_flow->dv.dest_array_res = resource;
11396         return 0;
11397 }
11398
11399 /**
11400  * Convert Sample action to DV specification.
11401  *
11402  * @param[in] dev
11403  *   Pointer to rte_eth_dev structure.
11404  * @param[in] action
11405  *   Pointer to sample action structure.
11406  * @param[in, out] dev_flow
11407  *   Pointer to the mlx5_flow.
11408  * @param[in] attr
11409  *   Pointer to the flow attributes.
11410  * @param[in, out] num_of_dest
11411  *   Pointer to the num of destination.
11412  * @param[in, out] sample_actions
11413  *   Pointer to sample actions list.
11414  * @param[in, out] res
11415  *   Pointer to sample resource.
11416  * @param[out] error
11417  *   Pointer to the error structure.
11418  *
11419  * @return
11420  *   0 on success, a negative errno value otherwise and rte_errno is set.
11421  */
11422 static int
11423 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
11424                                 const struct rte_flow_action_sample *action,
11425                                 struct mlx5_flow *dev_flow,
11426                                 const struct rte_flow_attr *attr,
11427                                 uint32_t *num_of_dest,
11428                                 void **sample_actions,
11429                                 struct mlx5_flow_dv_sample_resource *res,
11430                                 struct rte_flow_error *error)
11431 {
11432         struct mlx5_priv *priv = dev->data->dev_private;
11433         const struct rte_flow_action *sub_actions;
11434         struct mlx5_flow_sub_actions_list *sample_act;
11435         struct mlx5_flow_sub_actions_idx *sample_idx;
11436         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
11437         struct rte_flow *flow = dev_flow->flow;
11438         struct mlx5_flow_rss_desc *rss_desc;
11439         uint64_t action_flags = 0;
11440
11441         MLX5_ASSERT(wks);
11442         rss_desc = &wks->rss_desc;
11443         sample_act = &res->sample_act;
11444         sample_idx = &res->sample_idx;
11445         res->ratio = action->ratio;
11446         sub_actions = action->actions;
11447         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
11448                 int type = sub_actions->type;
11449                 uint32_t pre_rix = 0;
11450                 void *pre_r;
11451                 switch (type) {
11452                 case RTE_FLOW_ACTION_TYPE_QUEUE:
11453                 {
11454                         const struct rte_flow_action_queue *queue;
11455                         struct mlx5_hrxq *hrxq;
11456                         uint32_t hrxq_idx;
11457
11458                         queue = sub_actions->conf;
11459                         rss_desc->queue_num = 1;
11460                         rss_desc->queue[0] = queue->index;
11461                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11462                                                     rss_desc, &hrxq_idx);
11463                         if (!hrxq)
11464                                 return rte_flow_error_set
11465                                         (error, rte_errno,
11466                                          RTE_FLOW_ERROR_TYPE_ACTION,
11467                                          NULL,
11468                                          "cannot create fate queue");
11469                         sample_act->dr_queue_action = hrxq->action;
11470                         sample_idx->rix_hrxq = hrxq_idx;
11471                         sample_actions[sample_act->actions_num++] =
11472                                                 hrxq->action;
11473                         (*num_of_dest)++;
11474                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
11475                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11476                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11477                         dev_flow->handle->fate_action =
11478                                         MLX5_FLOW_FATE_QUEUE;
11479                         break;
11480                 }
11481                 case RTE_FLOW_ACTION_TYPE_RSS:
11482                 {
11483                         struct mlx5_hrxq *hrxq;
11484                         uint32_t hrxq_idx;
11485                         const struct rte_flow_action_rss *rss;
11486                         const uint8_t *rss_key;
11487
11488                         rss = sub_actions->conf;
11489                         memcpy(rss_desc->queue, rss->queue,
11490                                rss->queue_num * sizeof(uint16_t));
11491                         rss_desc->queue_num = rss->queue_num;
11492                         /* NULL RSS key indicates default RSS key. */
11493                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
11494                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
11495                         /*
11496                          * rss->level and rss.types should be set in advance
11497                          * when expanding items for RSS.
11498                          */
11499                         flow_dv_hashfields_set(dev_flow, rss_desc);
11500                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11501                                                     rss_desc, &hrxq_idx);
11502                         if (!hrxq)
11503                                 return rte_flow_error_set
11504                                         (error, rte_errno,
11505                                          RTE_FLOW_ERROR_TYPE_ACTION,
11506                                          NULL,
11507                                          "cannot create fate queue");
11508                         sample_act->dr_queue_action = hrxq->action;
11509                         sample_idx->rix_hrxq = hrxq_idx;
11510                         sample_actions[sample_act->actions_num++] =
11511                                                 hrxq->action;
11512                         (*num_of_dest)++;
11513                         action_flags |= MLX5_FLOW_ACTION_RSS;
11514                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11515                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11516                         dev_flow->handle->fate_action =
11517                                         MLX5_FLOW_FATE_QUEUE;
11518                         break;
11519                 }
11520                 case RTE_FLOW_ACTION_TYPE_MARK:
11521                 {
11522                         uint32_t tag_be = mlx5_flow_mark_set
11523                                 (((const struct rte_flow_action_mark *)
11524                                 (sub_actions->conf))->id);
11525
11526                         dev_flow->handle->mark = 1;
11527                         pre_rix = dev_flow->handle->dvh.rix_tag;
11528                         /* Save the mark resource before sample */
11529                         pre_r = dev_flow->dv.tag_resource;
11530                         if (flow_dv_tag_resource_register(dev, tag_be,
11531                                                   dev_flow, error))
11532                                 return -rte_errno;
11533                         MLX5_ASSERT(dev_flow->dv.tag_resource);
11534                         sample_act->dr_tag_action =
11535                                 dev_flow->dv.tag_resource->action;
11536                         sample_idx->rix_tag =
11537                                 dev_flow->handle->dvh.rix_tag;
11538                         sample_actions[sample_act->actions_num++] =
11539                                                 sample_act->dr_tag_action;
11540                         /* Recover the mark resource after sample */
11541                         dev_flow->dv.tag_resource = pre_r;
11542                         dev_flow->handle->dvh.rix_tag = pre_rix;
11543                         action_flags |= MLX5_FLOW_ACTION_MARK;
11544                         break;
11545                 }
11546                 case RTE_FLOW_ACTION_TYPE_COUNT:
11547                 {
11548                         if (!flow->counter) {
11549                                 flow->counter =
11550                                         flow_dv_translate_create_counter(dev,
11551                                                 dev_flow, sub_actions->conf,
11552                                                 0);
11553                                 if (!flow->counter)
11554                                         return rte_flow_error_set
11555                                                 (error, rte_errno,
11556                                                 RTE_FLOW_ERROR_TYPE_ACTION,
11557                                                 NULL,
11558                                                 "cannot create counter"
11559                                                 " object.");
11560                         }
11561                         sample_act->dr_cnt_action =
11562                                   (flow_dv_counter_get_by_idx(dev,
11563                                   flow->counter, NULL))->action;
11564                         sample_actions[sample_act->actions_num++] =
11565                                                 sample_act->dr_cnt_action;
11566                         action_flags |= MLX5_FLOW_ACTION_COUNT;
11567                         break;
11568                 }
11569                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
11570                 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
11571                 {
11572                         struct mlx5_flow_dv_port_id_action_resource
11573                                         port_id_resource;
11574                         uint32_t port_id = 0;
11575
11576                         memset(&port_id_resource, 0, sizeof(port_id_resource));
11577                         /* Save the port id resource before sample */
11578                         pre_rix = dev_flow->handle->rix_port_id_action;
11579                         pre_r = dev_flow->dv.port_id_action;
11580                         if (flow_dv_translate_action_port_id(dev, sub_actions,
11581                                                              &port_id, error))
11582                                 return -rte_errno;
11583                         port_id_resource.port_id = port_id;
11584                         if (flow_dv_port_id_action_resource_register
11585                             (dev, &port_id_resource, dev_flow, error))
11586                                 return -rte_errno;
11587                         sample_act->dr_port_id_action =
11588                                 dev_flow->dv.port_id_action->action;
11589                         sample_idx->rix_port_id_action =
11590                                 dev_flow->handle->rix_port_id_action;
11591                         sample_actions[sample_act->actions_num++] =
11592                                                 sample_act->dr_port_id_action;
11593                         /* Recover the port id resource after sample */
11594                         dev_flow->dv.port_id_action = pre_r;
11595                         dev_flow->handle->rix_port_id_action = pre_rix;
11596                         (*num_of_dest)++;
11597                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
11598                         break;
11599                 }
11600                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
11601                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
11602                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
11603                         /* Save the encap resource before sample */
11604                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
11605                         pre_r = dev_flow->dv.encap_decap;
11606                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
11607                                                            dev_flow,
11608                                                            attr->transfer,
11609                                                            error))
11610                                 return -rte_errno;
11611                         sample_act->dr_encap_action =
11612                                 dev_flow->dv.encap_decap->action;
11613                         sample_idx->rix_encap_decap =
11614                                 dev_flow->handle->dvh.rix_encap_decap;
11615                         sample_actions[sample_act->actions_num++] =
11616                                                 sample_act->dr_encap_action;
11617                         /* Recover the encap resource after sample */
11618                         dev_flow->dv.encap_decap = pre_r;
11619                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
11620                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
11621                         break;
11622                 default:
11623                         return rte_flow_error_set(error, EINVAL,
11624                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11625                                 NULL,
11626                                 "Not support for sampler action");
11627                 }
11628         }
11629         sample_act->action_flags = action_flags;
11630         res->ft_id = dev_flow->dv.group;
11631         if (attr->transfer) {
11632                 union {
11633                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
11634                         uint64_t set_action;
11635                 } action_ctx = { .set_action = 0 };
11636
11637                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
11638                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
11639                          MLX5_MODIFICATION_TYPE_SET);
11640                 MLX5_SET(set_action_in, action_ctx.action_in, field,
11641                          MLX5_MODI_META_REG_C_0);
11642                 MLX5_SET(set_action_in, action_ctx.action_in, data,
11643                          priv->vport_meta_tag);
11644                 res->set_action = action_ctx.set_action;
11645         } else if (attr->ingress) {
11646                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
11647         } else {
11648                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX;
11649         }
11650         return 0;
11651 }
11652
11653 /**
11654  * Convert Sample action to DV specification.
11655  *
11656  * @param[in] dev
11657  *   Pointer to rte_eth_dev structure.
11658  * @param[in, out] dev_flow
11659  *   Pointer to the mlx5_flow.
11660  * @param[in] num_of_dest
11661  *   The num of destination.
11662  * @param[in, out] res
11663  *   Pointer to sample resource.
11664  * @param[in, out] mdest_res
11665  *   Pointer to destination array resource.
11666  * @param[in] sample_actions
11667  *   Pointer to sample path actions list.
11668  * @param[in] action_flags
11669  *   Holds the actions detected until now.
11670  * @param[out] error
11671  *   Pointer to the error structure.
11672  *
11673  * @return
11674  *   0 on success, a negative errno value otherwise and rte_errno is set.
11675  */
11676 static int
11677 flow_dv_create_action_sample(struct rte_eth_dev *dev,
11678                              struct mlx5_flow *dev_flow,
11679                              uint32_t num_of_dest,
11680                              struct mlx5_flow_dv_sample_resource *res,
11681                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
11682                              void **sample_actions,
11683                              uint64_t action_flags,
11684                              struct rte_flow_error *error)
11685 {
11686         /* update normal path action resource into last index of array */
11687         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
11688         struct mlx5_flow_sub_actions_list *sample_act =
11689                                         &mdest_res->sample_act[dest_index];
11690         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
11691         struct mlx5_flow_rss_desc *rss_desc;
11692         uint32_t normal_idx = 0;
11693         struct mlx5_hrxq *hrxq;
11694         uint32_t hrxq_idx;
11695
11696         MLX5_ASSERT(wks);
11697         rss_desc = &wks->rss_desc;
11698         if (num_of_dest > 1) {
11699                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
11700                         /* Handle QP action for mirroring */
11701                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11702                                                     rss_desc, &hrxq_idx);
11703                         if (!hrxq)
11704                                 return rte_flow_error_set
11705                                      (error, rte_errno,
11706                                       RTE_FLOW_ERROR_TYPE_ACTION,
11707                                       NULL,
11708                                       "cannot create rx queue");
11709                         normal_idx++;
11710                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
11711                         sample_act->dr_queue_action = hrxq->action;
11712                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11713                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11714                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
11715                 }
11716                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
11717                         normal_idx++;
11718                         mdest_res->sample_idx[dest_index].rix_encap_decap =
11719                                 dev_flow->handle->dvh.rix_encap_decap;
11720                         sample_act->dr_encap_action =
11721                                 dev_flow->dv.encap_decap->action;
11722                         dev_flow->handle->dvh.rix_encap_decap = 0;
11723                 }
11724                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
11725                         normal_idx++;
11726                         mdest_res->sample_idx[dest_index].rix_port_id_action =
11727                                 dev_flow->handle->rix_port_id_action;
11728                         sample_act->dr_port_id_action =
11729                                 dev_flow->dv.port_id_action->action;
11730                         dev_flow->handle->rix_port_id_action = 0;
11731                 }
11732                 if (sample_act->action_flags & MLX5_FLOW_ACTION_JUMP) {
11733                         normal_idx++;
11734                         mdest_res->sample_idx[dest_index].rix_jump =
11735                                 dev_flow->handle->rix_jump;
11736                         sample_act->dr_jump_action =
11737                                 dev_flow->dv.jump->action;
11738                         dev_flow->handle->rix_jump = 0;
11739                 }
11740                 sample_act->actions_num = normal_idx;
11741                 /* update sample action resource into first index of array */
11742                 mdest_res->ft_type = res->ft_type;
11743                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
11744                                 sizeof(struct mlx5_flow_sub_actions_idx));
11745                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
11746                                 sizeof(struct mlx5_flow_sub_actions_list));
11747                 mdest_res->num_of_dest = num_of_dest;
11748                 if (flow_dv_dest_array_resource_register(dev, mdest_res,
11749                                                          dev_flow, error))
11750                         return rte_flow_error_set(error, EINVAL,
11751                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11752                                                   NULL, "can't create sample "
11753                                                   "action");
11754         } else {
11755                 res->sub_actions = sample_actions;
11756                 if (flow_dv_sample_resource_register(dev, res, dev_flow, error))
11757                         return rte_flow_error_set(error, EINVAL,
11758                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11759                                                   NULL,
11760                                                   "can't create sample action");
11761         }
11762         return 0;
11763 }
11764
11765 /**
11766  * Remove an ASO age action from age actions list.
11767  *
11768  * @param[in] dev
11769  *   Pointer to the Ethernet device structure.
11770  * @param[in] age
11771  *   Pointer to the aso age action handler.
11772  */
11773 static void
11774 flow_dv_aso_age_remove_from_age(struct rte_eth_dev *dev,
11775                                 struct mlx5_aso_age_action *age)
11776 {
11777         struct mlx5_age_info *age_info;
11778         struct mlx5_age_param *age_param = &age->age_params;
11779         struct mlx5_priv *priv = dev->data->dev_private;
11780         uint16_t expected = AGE_CANDIDATE;
11781
11782         age_info = GET_PORT_AGE_INFO(priv);
11783         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
11784                                          AGE_FREE, false, __ATOMIC_RELAXED,
11785                                          __ATOMIC_RELAXED)) {
11786                 /**
11787                  * We need the lock even it is age timeout,
11788                  * since age action may still in process.
11789                  */
11790                 rte_spinlock_lock(&age_info->aged_sl);
11791                 LIST_REMOVE(age, next);
11792                 rte_spinlock_unlock(&age_info->aged_sl);
11793                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
11794         }
11795 }
11796
11797 /**
11798  * Release an ASO age action.
11799  *
11800  * @param[in] dev
11801  *   Pointer to the Ethernet device structure.
11802  * @param[in] age_idx
11803  *   Index of ASO age action to release.
11804  * @param[in] flow
11805  *   True if the release operation is during flow destroy operation.
11806  *   False if the release operation is during action destroy operation.
11807  *
11808  * @return
11809  *   0 when age action was removed, otherwise the number of references.
11810  */
11811 static int
11812 flow_dv_aso_age_release(struct rte_eth_dev *dev, uint32_t age_idx)
11813 {
11814         struct mlx5_priv *priv = dev->data->dev_private;
11815         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11816         struct mlx5_aso_age_action *age = flow_aso_age_get_by_idx(dev, age_idx);
11817         uint32_t ret = __atomic_sub_fetch(&age->refcnt, 1, __ATOMIC_RELAXED);
11818
11819         if (!ret) {
11820                 flow_dv_aso_age_remove_from_age(dev, age);
11821                 rte_spinlock_lock(&mng->free_sl);
11822                 LIST_INSERT_HEAD(&mng->free, age, next);
11823                 rte_spinlock_unlock(&mng->free_sl);
11824         }
11825         return ret;
11826 }
11827
11828 /**
11829  * Resize the ASO age pools array by MLX5_CNT_CONTAINER_RESIZE pools.
11830  *
11831  * @param[in] dev
11832  *   Pointer to the Ethernet device structure.
11833  *
11834  * @return
11835  *   0 on success, otherwise negative errno value and rte_errno is set.
11836  */
11837 static int
11838 flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev)
11839 {
11840         struct mlx5_priv *priv = dev->data->dev_private;
11841         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11842         void *old_pools = mng->pools;
11843         uint32_t resize = mng->n + MLX5_CNT_CONTAINER_RESIZE;
11844         uint32_t mem_size = sizeof(struct mlx5_aso_age_pool *) * resize;
11845         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
11846
11847         if (!pools) {
11848                 rte_errno = ENOMEM;
11849                 return -ENOMEM;
11850         }
11851         if (old_pools) {
11852                 memcpy(pools, old_pools,
11853                        mng->n * sizeof(struct mlx5_flow_counter_pool *));
11854                 mlx5_free(old_pools);
11855         } else {
11856                 /* First ASO flow hit allocation - starting ASO data-path. */
11857                 int ret = mlx5_aso_flow_hit_queue_poll_start(priv->sh);
11858
11859                 if (ret) {
11860                         mlx5_free(pools);
11861                         return ret;
11862                 }
11863         }
11864         mng->n = resize;
11865         mng->pools = pools;
11866         return 0;
11867 }
11868
11869 /**
11870  * Create and initialize a new ASO aging pool.
11871  *
11872  * @param[in] dev
11873  *   Pointer to the Ethernet device structure.
11874  * @param[out] age_free
11875  *   Where to put the pointer of a new age action.
11876  *
11877  * @return
11878  *   The age actions pool pointer and @p age_free is set on success,
11879  *   NULL otherwise and rte_errno is set.
11880  */
11881 static struct mlx5_aso_age_pool *
11882 flow_dv_age_pool_create(struct rte_eth_dev *dev,
11883                         struct mlx5_aso_age_action **age_free)
11884 {
11885         struct mlx5_priv *priv = dev->data->dev_private;
11886         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11887         struct mlx5_aso_age_pool *pool = NULL;
11888         struct mlx5_devx_obj *obj = NULL;
11889         uint32_t i;
11890
11891         obj = mlx5_devx_cmd_create_flow_hit_aso_obj(priv->sh->cdev->ctx,
11892                                                     priv->sh->cdev->pdn);
11893         if (!obj) {
11894                 rte_errno = ENODATA;
11895                 DRV_LOG(ERR, "Failed to create flow_hit_aso_obj using DevX.");
11896                 return NULL;
11897         }
11898         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
11899         if (!pool) {
11900                 claim_zero(mlx5_devx_cmd_destroy(obj));
11901                 rte_errno = ENOMEM;
11902                 return NULL;
11903         }
11904         pool->flow_hit_aso_obj = obj;
11905         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
11906         rte_spinlock_lock(&mng->resize_sl);
11907         pool->index = mng->next;
11908         /* Resize pools array if there is no room for the new pool in it. */
11909         if (pool->index == mng->n && flow_dv_aso_age_pools_resize(dev)) {
11910                 claim_zero(mlx5_devx_cmd_destroy(obj));
11911                 mlx5_free(pool);
11912                 rte_spinlock_unlock(&mng->resize_sl);
11913                 return NULL;
11914         }
11915         mng->pools[pool->index] = pool;
11916         mng->next++;
11917         rte_spinlock_unlock(&mng->resize_sl);
11918         /* Assign the first action in the new pool, the rest go to free list. */
11919         *age_free = &pool->actions[0];
11920         for (i = 1; i < MLX5_ASO_AGE_ACTIONS_PER_POOL; i++) {
11921                 pool->actions[i].offset = i;
11922                 LIST_INSERT_HEAD(&mng->free, &pool->actions[i], next);
11923         }
11924         return pool;
11925 }
11926
11927 /**
11928  * Allocate a ASO aging bit.
11929  *
11930  * @param[in] dev
11931  *   Pointer to the Ethernet device structure.
11932  * @param[out] error
11933  *   Pointer to the error structure.
11934  *
11935  * @return
11936  *   Index to ASO age action on success, 0 otherwise and rte_errno is set.
11937  */
11938 static uint32_t
11939 flow_dv_aso_age_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
11940 {
11941         struct mlx5_priv *priv = dev->data->dev_private;
11942         const struct mlx5_aso_age_pool *pool;
11943         struct mlx5_aso_age_action *age_free = NULL;
11944         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11945
11946         MLX5_ASSERT(mng);
11947         /* Try to get the next free age action bit. */
11948         rte_spinlock_lock(&mng->free_sl);
11949         age_free = LIST_FIRST(&mng->free);
11950         if (age_free) {
11951                 LIST_REMOVE(age_free, next);
11952         } else if (!flow_dv_age_pool_create(dev, &age_free)) {
11953                 rte_spinlock_unlock(&mng->free_sl);
11954                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
11955                                    NULL, "failed to create ASO age pool");
11956                 return 0; /* 0 is an error. */
11957         }
11958         rte_spinlock_unlock(&mng->free_sl);
11959         pool = container_of
11960           ((const struct mlx5_aso_age_action (*)[MLX5_ASO_AGE_ACTIONS_PER_POOL])
11961                   (age_free - age_free->offset), const struct mlx5_aso_age_pool,
11962                                                                        actions);
11963         if (!age_free->dr_action) {
11964                 int reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_FLOW_HIT, 0,
11965                                                  error);
11966
11967                 if (reg_c < 0) {
11968                         rte_flow_error_set(error, rte_errno,
11969                                            RTE_FLOW_ERROR_TYPE_ACTION,
11970                                            NULL, "failed to get reg_c "
11971                                            "for ASO flow hit");
11972                         return 0; /* 0 is an error. */
11973                 }
11974 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
11975                 age_free->dr_action = mlx5_glue->dv_create_flow_action_aso
11976                                 (priv->sh->rx_domain,
11977                                  pool->flow_hit_aso_obj->obj, age_free->offset,
11978                                  MLX5DV_DR_ACTION_FLAGS_ASO_FIRST_HIT_SET,
11979                                  (reg_c - REG_C_0));
11980 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
11981                 if (!age_free->dr_action) {
11982                         rte_errno = errno;
11983                         rte_spinlock_lock(&mng->free_sl);
11984                         LIST_INSERT_HEAD(&mng->free, age_free, next);
11985                         rte_spinlock_unlock(&mng->free_sl);
11986                         rte_flow_error_set(error, rte_errno,
11987                                            RTE_FLOW_ERROR_TYPE_ACTION,
11988                                            NULL, "failed to create ASO "
11989                                            "flow hit action");
11990                         return 0; /* 0 is an error. */
11991                 }
11992         }
11993         __atomic_store_n(&age_free->refcnt, 1, __ATOMIC_RELAXED);
11994         return pool->index | ((age_free->offset + 1) << 16);
11995 }
11996
11997 /**
11998  * Initialize flow ASO age parameters.
11999  *
12000  * @param[in] dev
12001  *   Pointer to rte_eth_dev structure.
12002  * @param[in] age_idx
12003  *   Index of ASO age action.
12004  * @param[in] context
12005  *   Pointer to flow counter age context.
12006  * @param[in] timeout
12007  *   Aging timeout in seconds.
12008  *
12009  */
12010 static void
12011 flow_dv_aso_age_params_init(struct rte_eth_dev *dev,
12012                             uint32_t age_idx,
12013                             void *context,
12014                             uint32_t timeout)
12015 {
12016         struct mlx5_aso_age_action *aso_age;
12017
12018         aso_age = flow_aso_age_get_by_idx(dev, age_idx);
12019         MLX5_ASSERT(aso_age);
12020         aso_age->age_params.context = context;
12021         aso_age->age_params.timeout = timeout;
12022         aso_age->age_params.port_id = dev->data->port_id;
12023         __atomic_store_n(&aso_age->age_params.sec_since_last_hit, 0,
12024                          __ATOMIC_RELAXED);
12025         __atomic_store_n(&aso_age->age_params.state, AGE_CANDIDATE,
12026                          __ATOMIC_RELAXED);
12027 }
12028
12029 static void
12030 flow_dv_translate_integrity_l4(const struct rte_flow_item_integrity *mask,
12031                                const struct rte_flow_item_integrity *value,
12032                                void *headers_m, void *headers_v)
12033 {
12034         if (mask->l4_ok) {
12035                 /* application l4_ok filter aggregates all hardware l4 filters
12036                  * therefore hw l4_checksum_ok must be implicitly added here.
12037                  */
12038                 struct rte_flow_item_integrity local_item;
12039
12040                 local_item.l4_csum_ok = 1;
12041                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok,
12042                          local_item.l4_csum_ok);
12043                 if (value->l4_ok) {
12044                         /* application l4_ok = 1 matches sets both hw flags
12045                          * l4_ok and l4_checksum_ok flags to 1.
12046                          */
12047                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
12048                                  l4_checksum_ok, local_item.l4_csum_ok);
12049                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_ok,
12050                                  mask->l4_ok);
12051                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_ok,
12052                                  value->l4_ok);
12053                 } else {
12054                         /* application l4_ok = 0 matches on hw flag
12055                          * l4_checksum_ok = 0 only.
12056                          */
12057                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
12058                                  l4_checksum_ok, 0);
12059                 }
12060         } else if (mask->l4_csum_ok) {
12061                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok,
12062                          mask->l4_csum_ok);
12063                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_checksum_ok,
12064                          value->l4_csum_ok);
12065         }
12066 }
12067
12068 static void
12069 flow_dv_translate_integrity_l3(const struct rte_flow_item_integrity *mask,
12070                                const struct rte_flow_item_integrity *value,
12071                                void *headers_m, void *headers_v,
12072                                bool is_ipv4)
12073 {
12074         if (mask->l3_ok) {
12075                 /* application l3_ok filter aggregates all hardware l3 filters
12076                  * therefore hw ipv4_checksum_ok must be implicitly added here.
12077                  */
12078                 struct rte_flow_item_integrity local_item;
12079
12080                 local_item.ipv4_csum_ok = !!is_ipv4;
12081                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok,
12082                          local_item.ipv4_csum_ok);
12083                 if (value->l3_ok) {
12084                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
12085                                  ipv4_checksum_ok, local_item.ipv4_csum_ok);
12086                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l3_ok,
12087                                  mask->l3_ok);
12088                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l3_ok,
12089                                  value->l3_ok);
12090                 } else {
12091                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
12092                                  ipv4_checksum_ok, 0);
12093                 }
12094         } else if (mask->ipv4_csum_ok) {
12095                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok,
12096                          mask->ipv4_csum_ok);
12097                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_checksum_ok,
12098                          value->ipv4_csum_ok);
12099         }
12100 }
12101
12102 static void
12103 flow_dv_translate_item_integrity(void *matcher, void *key,
12104                                  const struct rte_flow_item *head_item,
12105                                  const struct rte_flow_item *integrity_item)
12106 {
12107         const struct rte_flow_item_integrity *mask = integrity_item->mask;
12108         const struct rte_flow_item_integrity *value = integrity_item->spec;
12109         const struct rte_flow_item *tunnel_item, *end_item, *item;
12110         void *headers_m;
12111         void *headers_v;
12112         uint32_t l3_protocol;
12113
12114         if (!value)
12115                 return;
12116         if (!mask)
12117                 mask = &rte_flow_item_integrity_mask;
12118         if (value->level > 1) {
12119                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
12120                                          inner_headers);
12121                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
12122         } else {
12123                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
12124                                          outer_headers);
12125                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
12126         }
12127         tunnel_item = mlx5_flow_find_tunnel_item(head_item);
12128         if (value->level > 1) {
12129                 /* tunnel item was verified during the item validation */
12130                 item = tunnel_item;
12131                 end_item = mlx5_find_end_item(tunnel_item);
12132         } else {
12133                 item = head_item;
12134                 end_item = tunnel_item ? tunnel_item :
12135                            mlx5_find_end_item(integrity_item);
12136         }
12137         l3_protocol = mask->l3_ok ?
12138                       mlx5_flow_locate_proto_l3(&item, end_item) : 0;
12139         flow_dv_translate_integrity_l3(mask, value, headers_m, headers_v,
12140                                        l3_protocol == RTE_ETHER_TYPE_IPV4);
12141         flow_dv_translate_integrity_l4(mask, value, headers_m, headers_v);
12142 }
12143
12144 /**
12145  * Prepares DV flow counter with aging configuration.
12146  * Gets it by index when exists, creates a new one when doesn't.
12147  *
12148  * @param[in] dev
12149  *   Pointer to rte_eth_dev structure.
12150  * @param[in] dev_flow
12151  *   Pointer to the mlx5_flow.
12152  * @param[in, out] flow
12153  *   Pointer to the sub flow.
12154  * @param[in] count
12155  *   Pointer to the counter action configuration.
12156  * @param[in] age
12157  *   Pointer to the aging action configuration.
12158  * @param[out] error
12159  *   Pointer to the error structure.
12160  *
12161  * @return
12162  *   Pointer to the counter, NULL otherwise.
12163  */
12164 static struct mlx5_flow_counter *
12165 flow_dv_prepare_counter(struct rte_eth_dev *dev,
12166                         struct mlx5_flow *dev_flow,
12167                         struct rte_flow *flow,
12168                         const struct rte_flow_action_count *count,
12169                         const struct rte_flow_action_age *age,
12170                         struct rte_flow_error *error)
12171 {
12172         if (!flow->counter) {
12173                 flow->counter = flow_dv_translate_create_counter(dev, dev_flow,
12174                                                                  count, age);
12175                 if (!flow->counter) {
12176                         rte_flow_error_set(error, rte_errno,
12177                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12178                                            "cannot create counter object.");
12179                         return NULL;
12180                 }
12181         }
12182         return flow_dv_counter_get_by_idx(dev, flow->counter, NULL);
12183 }
12184
12185 /*
12186  * Release an ASO CT action by its own device.
12187  *
12188  * @param[in] dev
12189  *   Pointer to the Ethernet device structure.
12190  * @param[in] idx
12191  *   Index of ASO CT action to release.
12192  *
12193  * @return
12194  *   0 when CT action was removed, otherwise the number of references.
12195  */
12196 static inline int
12197 flow_dv_aso_ct_dev_release(struct rte_eth_dev *dev, uint32_t idx)
12198 {
12199         struct mlx5_priv *priv = dev->data->dev_private;
12200         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12201         uint32_t ret;
12202         struct mlx5_aso_ct_action *ct = flow_aso_ct_get_by_dev_idx(dev, idx);
12203         enum mlx5_aso_ct_state state =
12204                         __atomic_load_n(&ct->state, __ATOMIC_RELAXED);
12205
12206         /* Cannot release when CT is in the ASO SQ. */
12207         if (state == ASO_CONNTRACK_WAIT || state == ASO_CONNTRACK_QUERY)
12208                 return -1;
12209         ret = __atomic_sub_fetch(&ct->refcnt, 1, __ATOMIC_RELAXED);
12210         if (!ret) {
12211                 if (ct->dr_action_orig) {
12212 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12213                         claim_zero(mlx5_glue->destroy_flow_action
12214                                         (ct->dr_action_orig));
12215 #endif
12216                         ct->dr_action_orig = NULL;
12217                 }
12218                 if (ct->dr_action_rply) {
12219 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12220                         claim_zero(mlx5_glue->destroy_flow_action
12221                                         (ct->dr_action_rply));
12222 #endif
12223                         ct->dr_action_rply = NULL;
12224                 }
12225                 /* Clear the state to free, no need in 1st allocation. */
12226                 MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_FREE);
12227                 rte_spinlock_lock(&mng->ct_sl);
12228                 LIST_INSERT_HEAD(&mng->free_cts, ct, next);
12229                 rte_spinlock_unlock(&mng->ct_sl);
12230         }
12231         return (int)ret;
12232 }
12233
12234 static inline int
12235 flow_dv_aso_ct_release(struct rte_eth_dev *dev, uint32_t own_idx,
12236                        struct rte_flow_error *error)
12237 {
12238         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(own_idx);
12239         uint32_t idx = MLX5_INDIRECT_ACT_CT_GET_IDX(own_idx);
12240         struct rte_eth_dev *owndev = &rte_eth_devices[owner];
12241         int ret;
12242
12243         MLX5_ASSERT(owner < RTE_MAX_ETHPORTS);
12244         if (dev->data->dev_started != 1)
12245                 return rte_flow_error_set(error, EAGAIN,
12246                                           RTE_FLOW_ERROR_TYPE_ACTION,
12247                                           NULL,
12248                                           "Indirect CT action cannot be destroyed when the port is stopped");
12249         ret = flow_dv_aso_ct_dev_release(owndev, idx);
12250         if (ret < 0)
12251                 return rte_flow_error_set(error, EAGAIN,
12252                                           RTE_FLOW_ERROR_TYPE_ACTION,
12253                                           NULL,
12254                                           "Current state prevents indirect CT action from being destroyed");
12255         return ret;
12256 }
12257
12258 /*
12259  * Resize the ASO CT pools array by 64 pools.
12260  *
12261  * @param[in] dev
12262  *   Pointer to the Ethernet device structure.
12263  *
12264  * @return
12265  *   0 on success, otherwise negative errno value and rte_errno is set.
12266  */
12267 static int
12268 flow_dv_aso_ct_pools_resize(struct rte_eth_dev *dev)
12269 {
12270         struct mlx5_priv *priv = dev->data->dev_private;
12271         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12272         void *old_pools = mng->pools;
12273         /* Magic number now, need a macro. */
12274         uint32_t resize = mng->n + 64;
12275         uint32_t mem_size = sizeof(struct mlx5_aso_ct_pool *) * resize;
12276         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
12277
12278         if (!pools) {
12279                 rte_errno = ENOMEM;
12280                 return -rte_errno;
12281         }
12282         rte_rwlock_write_lock(&mng->resize_rwl);
12283         /* ASO SQ/QP was already initialized in the startup. */
12284         if (old_pools) {
12285                 /* Realloc could be an alternative choice. */
12286                 rte_memcpy(pools, old_pools,
12287                            mng->n * sizeof(struct mlx5_aso_ct_pool *));
12288                 mlx5_free(old_pools);
12289         }
12290         mng->n = resize;
12291         mng->pools = pools;
12292         rte_rwlock_write_unlock(&mng->resize_rwl);
12293         return 0;
12294 }
12295
12296 /*
12297  * Create and initialize a new ASO CT pool.
12298  *
12299  * @param[in] dev
12300  *   Pointer to the Ethernet device structure.
12301  * @param[out] ct_free
12302  *   Where to put the pointer of a new CT action.
12303  *
12304  * @return
12305  *   The CT actions pool pointer and @p ct_free is set on success,
12306  *   NULL otherwise and rte_errno is set.
12307  */
12308 static struct mlx5_aso_ct_pool *
12309 flow_dv_ct_pool_create(struct rte_eth_dev *dev,
12310                        struct mlx5_aso_ct_action **ct_free)
12311 {
12312         struct mlx5_priv *priv = dev->data->dev_private;
12313         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12314         struct mlx5_aso_ct_pool *pool = NULL;
12315         struct mlx5_devx_obj *obj = NULL;
12316         uint32_t i;
12317         uint32_t log_obj_size = rte_log2_u32(MLX5_ASO_CT_ACTIONS_PER_POOL);
12318
12319         obj = mlx5_devx_cmd_create_conn_track_offload_obj(priv->sh->cdev->ctx,
12320                                                           priv->sh->cdev->pdn,
12321                                                           log_obj_size);
12322         if (!obj) {
12323                 rte_errno = ENODATA;
12324                 DRV_LOG(ERR, "Failed to create conn_track_offload_obj using DevX.");
12325                 return NULL;
12326         }
12327         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
12328         if (!pool) {
12329                 rte_errno = ENOMEM;
12330                 claim_zero(mlx5_devx_cmd_destroy(obj));
12331                 return NULL;
12332         }
12333         pool->devx_obj = obj;
12334         pool->index = mng->next;
12335         /* Resize pools array if there is no room for the new pool in it. */
12336         if (pool->index == mng->n && flow_dv_aso_ct_pools_resize(dev)) {
12337                 claim_zero(mlx5_devx_cmd_destroy(obj));
12338                 mlx5_free(pool);
12339                 return NULL;
12340         }
12341         mng->pools[pool->index] = pool;
12342         mng->next++;
12343         /* Assign the first action in the new pool, the rest go to free list. */
12344         *ct_free = &pool->actions[0];
12345         /* Lock outside, the list operation is safe here. */
12346         for (i = 1; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) {
12347                 /* refcnt is 0 when allocating the memory. */
12348                 pool->actions[i].offset = i;
12349                 LIST_INSERT_HEAD(&mng->free_cts, &pool->actions[i], next);
12350         }
12351         return pool;
12352 }
12353
12354 /*
12355  * Allocate a ASO CT action from free list.
12356  *
12357  * @param[in] dev
12358  *   Pointer to the Ethernet device structure.
12359  * @param[out] error
12360  *   Pointer to the error structure.
12361  *
12362  * @return
12363  *   Index to ASO CT action on success, 0 otherwise and rte_errno is set.
12364  */
12365 static uint32_t
12366 flow_dv_aso_ct_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
12367 {
12368         struct mlx5_priv *priv = dev->data->dev_private;
12369         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12370         struct mlx5_aso_ct_action *ct = NULL;
12371         struct mlx5_aso_ct_pool *pool;
12372         uint8_t reg_c;
12373         uint32_t ct_idx;
12374
12375         MLX5_ASSERT(mng);
12376         if (!priv->sh->devx) {
12377                 rte_errno = ENOTSUP;
12378                 return 0;
12379         }
12380         /* Get a free CT action, if no, a new pool will be created. */
12381         rte_spinlock_lock(&mng->ct_sl);
12382         ct = LIST_FIRST(&mng->free_cts);
12383         if (ct) {
12384                 LIST_REMOVE(ct, next);
12385         } else if (!flow_dv_ct_pool_create(dev, &ct)) {
12386                 rte_spinlock_unlock(&mng->ct_sl);
12387                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
12388                                    NULL, "failed to create ASO CT pool");
12389                 return 0;
12390         }
12391         rte_spinlock_unlock(&mng->ct_sl);
12392         pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
12393         ct_idx = MLX5_MAKE_CT_IDX(pool->index, ct->offset);
12394         /* 0: inactive, 1: created, 2+: used by flows. */
12395         __atomic_store_n(&ct->refcnt, 1, __ATOMIC_RELAXED);
12396         reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, error);
12397         if (!ct->dr_action_orig) {
12398 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12399                 ct->dr_action_orig = mlx5_glue->dv_create_flow_action_aso
12400                         (priv->sh->rx_domain, pool->devx_obj->obj,
12401                          ct->offset,
12402                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_INITIATOR,
12403                          reg_c - REG_C_0);
12404 #else
12405                 RTE_SET_USED(reg_c);
12406 #endif
12407                 if (!ct->dr_action_orig) {
12408                         flow_dv_aso_ct_dev_release(dev, ct_idx);
12409                         rte_flow_error_set(error, rte_errno,
12410                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12411                                            "failed to create ASO CT action");
12412                         return 0;
12413                 }
12414         }
12415         if (!ct->dr_action_rply) {
12416 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12417                 ct->dr_action_rply = mlx5_glue->dv_create_flow_action_aso
12418                         (priv->sh->rx_domain, pool->devx_obj->obj,
12419                          ct->offset,
12420                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_RESPONDER,
12421                          reg_c - REG_C_0);
12422 #endif
12423                 if (!ct->dr_action_rply) {
12424                         flow_dv_aso_ct_dev_release(dev, ct_idx);
12425                         rte_flow_error_set(error, rte_errno,
12426                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12427                                            "failed to create ASO CT action");
12428                         return 0;
12429                 }
12430         }
12431         return ct_idx;
12432 }
12433
12434 /*
12435  * Create a conntrack object with context and actions by using ASO mechanism.
12436  *
12437  * @param[in] dev
12438  *   Pointer to rte_eth_dev structure.
12439  * @param[in] pro
12440  *   Pointer to conntrack information profile.
12441  * @param[out] error
12442  *   Pointer to the error structure.
12443  *
12444  * @return
12445  *   Index to conntrack object on success, 0 otherwise.
12446  */
12447 static uint32_t
12448 flow_dv_translate_create_conntrack(struct rte_eth_dev *dev,
12449                                    const struct rte_flow_action_conntrack *pro,
12450                                    struct rte_flow_error *error)
12451 {
12452         struct mlx5_priv *priv = dev->data->dev_private;
12453         struct mlx5_dev_ctx_shared *sh = priv->sh;
12454         struct mlx5_aso_ct_action *ct;
12455         uint32_t idx;
12456
12457         if (!sh->ct_aso_en)
12458                 return rte_flow_error_set(error, ENOTSUP,
12459                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12460                                           "Connection is not supported");
12461         idx = flow_dv_aso_ct_alloc(dev, error);
12462         if (!idx)
12463                 return rte_flow_error_set(error, rte_errno,
12464                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12465                                           "Failed to allocate CT object");
12466         ct = flow_aso_ct_get_by_dev_idx(dev, idx);
12467         if (mlx5_aso_ct_update_by_wqe(sh, ct, pro))
12468                 return rte_flow_error_set(error, EBUSY,
12469                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12470                                           "Failed to update CT");
12471         ct->is_original = !!pro->is_original_dir;
12472         ct->peer = pro->peer_port;
12473         return idx;
12474 }
12475
12476 /**
12477  * Fill the flow with DV spec, lock free
12478  * (mutex should be acquired by caller).
12479  *
12480  * @param[in] dev
12481  *   Pointer to rte_eth_dev structure.
12482  * @param[in, out] dev_flow
12483  *   Pointer to the sub flow.
12484  * @param[in] attr
12485  *   Pointer to the flow attributes.
12486  * @param[in] items
12487  *   Pointer to the list of items.
12488  * @param[in] actions
12489  *   Pointer to the list of actions.
12490  * @param[out] error
12491  *   Pointer to the error structure.
12492  *
12493  * @return
12494  *   0 on success, a negative errno value otherwise and rte_errno is set.
12495  */
12496 static int
12497 flow_dv_translate(struct rte_eth_dev *dev,
12498                   struct mlx5_flow *dev_flow,
12499                   const struct rte_flow_attr *attr,
12500                   const struct rte_flow_item items[],
12501                   const struct rte_flow_action actions[],
12502                   struct rte_flow_error *error)
12503 {
12504         struct mlx5_priv *priv = dev->data->dev_private;
12505         struct mlx5_dev_config *dev_conf = &priv->config;
12506         struct rte_flow *flow = dev_flow->flow;
12507         struct mlx5_flow_handle *handle = dev_flow->handle;
12508         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
12509         struct mlx5_flow_rss_desc *rss_desc;
12510         uint64_t item_flags = 0;
12511         uint64_t last_item = 0;
12512         uint64_t action_flags = 0;
12513         struct mlx5_flow_dv_matcher matcher = {
12514                 .mask = {
12515                         .size = sizeof(matcher.mask.buf),
12516                 },
12517         };
12518         int actions_n = 0;
12519         bool actions_end = false;
12520         union {
12521                 struct mlx5_flow_dv_modify_hdr_resource res;
12522                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
12523                             sizeof(struct mlx5_modification_cmd) *
12524                             (MLX5_MAX_MODIFY_NUM + 1)];
12525         } mhdr_dummy;
12526         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
12527         const struct rte_flow_action_count *count = NULL;
12528         const struct rte_flow_action_age *non_shared_age = NULL;
12529         union flow_dv_attr flow_attr = { .attr = 0 };
12530         uint32_t tag_be;
12531         union mlx5_flow_tbl_key tbl_key;
12532         uint32_t modify_action_position = UINT32_MAX;
12533         void *match_mask = matcher.mask.buf;
12534         void *match_value = dev_flow->dv.value.buf;
12535         uint8_t next_protocol = 0xff;
12536         struct rte_vlan_hdr vlan = { 0 };
12537         struct mlx5_flow_dv_dest_array_resource mdest_res;
12538         struct mlx5_flow_dv_sample_resource sample_res;
12539         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
12540         const struct rte_flow_action_sample *sample = NULL;
12541         struct mlx5_flow_sub_actions_list *sample_act;
12542         uint32_t sample_act_pos = UINT32_MAX;
12543         uint32_t age_act_pos = UINT32_MAX;
12544         uint32_t num_of_dest = 0;
12545         int tmp_actions_n = 0;
12546         uint32_t table;
12547         int ret = 0;
12548         const struct mlx5_flow_tunnel *tunnel = NULL;
12549         struct flow_grp_info grp_info = {
12550                 .external = !!dev_flow->external,
12551                 .transfer = !!attr->transfer,
12552                 .fdb_def_rule = !!priv->fdb_def_rule,
12553                 .skip_scale = dev_flow->skip_scale &
12554                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
12555                 .std_tbl_fix = true,
12556         };
12557         const struct rte_flow_item *head_item = items;
12558
12559         if (!wks)
12560                 return rte_flow_error_set(error, ENOMEM,
12561                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12562                                           NULL,
12563                                           "failed to push flow workspace");
12564         rss_desc = &wks->rss_desc;
12565         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
12566         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
12567         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12568                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12569         /* update normal path action resource into last index of array */
12570         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
12571         if (is_tunnel_offload_active(dev)) {
12572                 if (dev_flow->tunnel) {
12573                         RTE_VERIFY(dev_flow->tof_type ==
12574                                    MLX5_TUNNEL_OFFLOAD_MISS_RULE);
12575                         tunnel = dev_flow->tunnel;
12576                 } else {
12577                         tunnel = mlx5_get_tof(items, actions,
12578                                               &dev_flow->tof_type);
12579                         dev_flow->tunnel = tunnel;
12580                 }
12581                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
12582                                         (dev, attr, tunnel, dev_flow->tof_type);
12583         }
12584         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12585                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12586         ret = mlx5_flow_group_to_table(dev, tunnel, attr->group, &table,
12587                                        &grp_info, error);
12588         if (ret)
12589                 return ret;
12590         dev_flow->dv.group = table;
12591         if (attr->transfer)
12592                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
12593         /* number of actions must be set to 0 in case of dirty stack. */
12594         mhdr_res->actions_num = 0;
12595         if (is_flow_tunnel_match_rule(dev_flow->tof_type)) {
12596                 /*
12597                  * do not add decap action if match rule drops packet
12598                  * HW rejects rules with decap & drop
12599                  *
12600                  * if tunnel match rule was inserted before matching tunnel set
12601                  * rule flow table used in the match rule must be registered.
12602                  * current implementation handles that in the
12603                  * flow_dv_match_register() at the function end.
12604                  */
12605                 bool add_decap = true;
12606                 const struct rte_flow_action *ptr = actions;
12607
12608                 for (; ptr->type != RTE_FLOW_ACTION_TYPE_END; ptr++) {
12609                         if (ptr->type == RTE_FLOW_ACTION_TYPE_DROP) {
12610                                 add_decap = false;
12611                                 break;
12612                         }
12613                 }
12614                 if (add_decap) {
12615                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
12616                                                            attr->transfer,
12617                                                            error))
12618                                 return -rte_errno;
12619                         dev_flow->dv.actions[actions_n++] =
12620                                         dev_flow->dv.encap_decap->action;
12621                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12622                 }
12623         }
12624         for (; !actions_end ; actions++) {
12625                 const struct rte_flow_action_queue *queue;
12626                 const struct rte_flow_action_rss *rss;
12627                 const struct rte_flow_action *action = actions;
12628                 const uint8_t *rss_key;
12629                 struct mlx5_flow_tbl_resource *tbl;
12630                 struct mlx5_aso_age_action *age_act;
12631                 struct mlx5_flow_counter *cnt_act;
12632                 uint32_t port_id = 0;
12633                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
12634                 int action_type = actions->type;
12635                 const struct rte_flow_action *found_action = NULL;
12636                 uint32_t jump_group = 0;
12637                 uint32_t owner_idx;
12638                 struct mlx5_aso_ct_action *ct;
12639
12640                 if (!mlx5_flow_os_action_supported(action_type))
12641                         return rte_flow_error_set(error, ENOTSUP,
12642                                                   RTE_FLOW_ERROR_TYPE_ACTION,
12643                                                   actions,
12644                                                   "action not supported");
12645                 switch (action_type) {
12646                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
12647                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
12648                         break;
12649                 case RTE_FLOW_ACTION_TYPE_VOID:
12650                         break;
12651                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
12652                 case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
12653                         if (flow_dv_translate_action_port_id(dev, action,
12654                                                              &port_id, error))
12655                                 return -rte_errno;
12656                         port_id_resource.port_id = port_id;
12657                         MLX5_ASSERT(!handle->rix_port_id_action);
12658                         if (flow_dv_port_id_action_resource_register
12659                             (dev, &port_id_resource, dev_flow, error))
12660                                 return -rte_errno;
12661                         dev_flow->dv.actions[actions_n++] =
12662                                         dev_flow->dv.port_id_action->action;
12663                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
12664                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
12665                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
12666                         num_of_dest++;
12667                         break;
12668                 case RTE_FLOW_ACTION_TYPE_FLAG:
12669                         action_flags |= MLX5_FLOW_ACTION_FLAG;
12670                         dev_flow->handle->mark = 1;
12671                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
12672                                 struct rte_flow_action_mark mark = {
12673                                         .id = MLX5_FLOW_MARK_DEFAULT,
12674                                 };
12675
12676                                 if (flow_dv_convert_action_mark(dev, &mark,
12677                                                                 mhdr_res,
12678                                                                 error))
12679                                         return -rte_errno;
12680                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
12681                                 break;
12682                         }
12683                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
12684                         /*
12685                          * Only one FLAG or MARK is supported per device flow
12686                          * right now. So the pointer to the tag resource must be
12687                          * zero before the register process.
12688                          */
12689                         MLX5_ASSERT(!handle->dvh.rix_tag);
12690                         if (flow_dv_tag_resource_register(dev, tag_be,
12691                                                           dev_flow, error))
12692                                 return -rte_errno;
12693                         MLX5_ASSERT(dev_flow->dv.tag_resource);
12694                         dev_flow->dv.actions[actions_n++] =
12695                                         dev_flow->dv.tag_resource->action;
12696                         break;
12697                 case RTE_FLOW_ACTION_TYPE_MARK:
12698                         action_flags |= MLX5_FLOW_ACTION_MARK;
12699                         dev_flow->handle->mark = 1;
12700                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
12701                                 const struct rte_flow_action_mark *mark =
12702                                         (const struct rte_flow_action_mark *)
12703                                                 actions->conf;
12704
12705                                 if (flow_dv_convert_action_mark(dev, mark,
12706                                                                 mhdr_res,
12707                                                                 error))
12708                                         return -rte_errno;
12709                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
12710                                 break;
12711                         }
12712                         /* Fall-through */
12713                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
12714                         /* Legacy (non-extensive) MARK action. */
12715                         tag_be = mlx5_flow_mark_set
12716                               (((const struct rte_flow_action_mark *)
12717                                (actions->conf))->id);
12718                         MLX5_ASSERT(!handle->dvh.rix_tag);
12719                         if (flow_dv_tag_resource_register(dev, tag_be,
12720                                                           dev_flow, error))
12721                                 return -rte_errno;
12722                         MLX5_ASSERT(dev_flow->dv.tag_resource);
12723                         dev_flow->dv.actions[actions_n++] =
12724                                         dev_flow->dv.tag_resource->action;
12725                         break;
12726                 case RTE_FLOW_ACTION_TYPE_SET_META:
12727                         if (flow_dv_convert_action_set_meta
12728                                 (dev, mhdr_res, attr,
12729                                  (const struct rte_flow_action_set_meta *)
12730                                   actions->conf, error))
12731                                 return -rte_errno;
12732                         action_flags |= MLX5_FLOW_ACTION_SET_META;
12733                         break;
12734                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
12735                         if (flow_dv_convert_action_set_tag
12736                                 (dev, mhdr_res,
12737                                  (const struct rte_flow_action_set_tag *)
12738                                   actions->conf, error))
12739                                 return -rte_errno;
12740                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
12741                         break;
12742                 case RTE_FLOW_ACTION_TYPE_DROP:
12743                         action_flags |= MLX5_FLOW_ACTION_DROP;
12744                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
12745                         break;
12746                 case RTE_FLOW_ACTION_TYPE_QUEUE:
12747                         queue = actions->conf;
12748                         rss_desc->queue_num = 1;
12749                         rss_desc->queue[0] = queue->index;
12750                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
12751                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
12752                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
12753                         num_of_dest++;
12754                         break;
12755                 case RTE_FLOW_ACTION_TYPE_RSS:
12756                         rss = actions->conf;
12757                         memcpy(rss_desc->queue, rss->queue,
12758                                rss->queue_num * sizeof(uint16_t));
12759                         rss_desc->queue_num = rss->queue_num;
12760                         /* NULL RSS key indicates default RSS key. */
12761                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
12762                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
12763                         /*
12764                          * rss->level and rss.types should be set in advance
12765                          * when expanding items for RSS.
12766                          */
12767                         action_flags |= MLX5_FLOW_ACTION_RSS;
12768                         dev_flow->handle->fate_action = rss_desc->shared_rss ?
12769                                 MLX5_FLOW_FATE_SHARED_RSS :
12770                                 MLX5_FLOW_FATE_QUEUE;
12771                         break;
12772                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
12773                         owner_idx = (uint32_t)(uintptr_t)action->conf;
12774                         age_act = flow_aso_age_get_by_idx(dev, owner_idx);
12775                         if (flow->age == 0) {
12776                                 flow->age = owner_idx;
12777                                 __atomic_fetch_add(&age_act->refcnt, 1,
12778                                                    __ATOMIC_RELAXED);
12779                         }
12780                         age_act_pos = actions_n++;
12781                         action_flags |= MLX5_FLOW_ACTION_AGE;
12782                         break;
12783                 case RTE_FLOW_ACTION_TYPE_AGE:
12784                         non_shared_age = action->conf;
12785                         age_act_pos = actions_n++;
12786                         action_flags |= MLX5_FLOW_ACTION_AGE;
12787                         break;
12788                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
12789                         owner_idx = (uint32_t)(uintptr_t)action->conf;
12790                         cnt_act = flow_dv_counter_get_by_idx(dev, owner_idx,
12791                                                              NULL);
12792                         MLX5_ASSERT(cnt_act != NULL);
12793                         /**
12794                          * When creating meter drop flow in drop table, the
12795                          * counter should not overwrite the rte flow counter.
12796                          */
12797                         if (attr->group == MLX5_FLOW_TABLE_LEVEL_METER &&
12798                             dev_flow->dv.table_id == MLX5_MTR_TABLE_ID_DROP) {
12799                                 dev_flow->dv.actions[actions_n++] =
12800                                                         cnt_act->action;
12801                         } else {
12802                                 if (flow->counter == 0) {
12803                                         flow->counter = owner_idx;
12804                                         __atomic_fetch_add
12805                                                 (&cnt_act->shared_info.refcnt,
12806                                                  1, __ATOMIC_RELAXED);
12807                                 }
12808                                 /* Save information first, will apply later. */
12809                                 action_flags |= MLX5_FLOW_ACTION_COUNT;
12810                         }
12811                         break;
12812                 case RTE_FLOW_ACTION_TYPE_COUNT:
12813                         if (!priv->sh->devx) {
12814                                 return rte_flow_error_set
12815                                               (error, ENOTSUP,
12816                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12817                                                NULL,
12818                                                "count action not supported");
12819                         }
12820                         /* Save information first, will apply later. */
12821                         count = action->conf;
12822                         action_flags |= MLX5_FLOW_ACTION_COUNT;
12823                         break;
12824                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
12825                         dev_flow->dv.actions[actions_n++] =
12826                                                 priv->sh->pop_vlan_action;
12827                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
12828                         break;
12829                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
12830                         if (!(action_flags &
12831                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
12832                                 flow_dev_get_vlan_info_from_items(items, &vlan);
12833                         vlan.eth_proto = rte_be_to_cpu_16
12834                              ((((const struct rte_flow_action_of_push_vlan *)
12835                                                    actions->conf)->ethertype));
12836                         found_action = mlx5_flow_find_action
12837                                         (actions + 1,
12838                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
12839                         if (found_action)
12840                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
12841                         found_action = mlx5_flow_find_action
12842                                         (actions + 1,
12843                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
12844                         if (found_action)
12845                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
12846                         if (flow_dv_create_action_push_vlan
12847                                             (dev, attr, &vlan, dev_flow, error))
12848                                 return -rte_errno;
12849                         dev_flow->dv.actions[actions_n++] =
12850                                         dev_flow->dv.push_vlan_res->action;
12851                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
12852                         break;
12853                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
12854                         /* of_vlan_push action handled this action */
12855                         MLX5_ASSERT(action_flags &
12856                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
12857                         break;
12858                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
12859                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
12860                                 break;
12861                         flow_dev_get_vlan_info_from_items(items, &vlan);
12862                         mlx5_update_vlan_vid_pcp(actions, &vlan);
12863                         /* If no VLAN push - this is a modify header action */
12864                         if (flow_dv_convert_action_modify_vlan_vid
12865                                                 (mhdr_res, actions, error))
12866                                 return -rte_errno;
12867                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
12868                         break;
12869                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
12870                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
12871                         if (flow_dv_create_action_l2_encap(dev, actions,
12872                                                            dev_flow,
12873                                                            attr->transfer,
12874                                                            error))
12875                                 return -rte_errno;
12876                         dev_flow->dv.actions[actions_n++] =
12877                                         dev_flow->dv.encap_decap->action;
12878                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
12879                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
12880                                 sample_act->action_flags |=
12881                                                         MLX5_FLOW_ACTION_ENCAP;
12882                         break;
12883                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
12884                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
12885                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
12886                                                            attr->transfer,
12887                                                            error))
12888                                 return -rte_errno;
12889                         dev_flow->dv.actions[actions_n++] =
12890                                         dev_flow->dv.encap_decap->action;
12891                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12892                         break;
12893                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
12894                         /* Handle encap with preceding decap. */
12895                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
12896                                 if (flow_dv_create_action_raw_encap
12897                                         (dev, actions, dev_flow, attr, error))
12898                                         return -rte_errno;
12899                                 dev_flow->dv.actions[actions_n++] =
12900                                         dev_flow->dv.encap_decap->action;
12901                         } else {
12902                                 /* Handle encap without preceding decap. */
12903                                 if (flow_dv_create_action_l2_encap
12904                                     (dev, actions, dev_flow, attr->transfer,
12905                                      error))
12906                                         return -rte_errno;
12907                                 dev_flow->dv.actions[actions_n++] =
12908                                         dev_flow->dv.encap_decap->action;
12909                         }
12910                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
12911                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
12912                                 sample_act->action_flags |=
12913                                                         MLX5_FLOW_ACTION_ENCAP;
12914                         break;
12915                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
12916                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
12917                                 ;
12918                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
12919                                 if (flow_dv_create_action_l2_decap
12920                                     (dev, dev_flow, attr->transfer, error))
12921                                         return -rte_errno;
12922                                 dev_flow->dv.actions[actions_n++] =
12923                                         dev_flow->dv.encap_decap->action;
12924                         }
12925                         /* If decap is followed by encap, handle it at encap. */
12926                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12927                         break;
12928                 case MLX5_RTE_FLOW_ACTION_TYPE_JUMP:
12929                         dev_flow->dv.actions[actions_n++] =
12930                                 (void *)(uintptr_t)action->conf;
12931                         action_flags |= MLX5_FLOW_ACTION_JUMP;
12932                         break;
12933                 case RTE_FLOW_ACTION_TYPE_JUMP:
12934                         jump_group = ((const struct rte_flow_action_jump *)
12935                                                         action->conf)->group;
12936                         grp_info.std_tbl_fix = 0;
12937                         if (dev_flow->skip_scale &
12938                                 (1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT))
12939                                 grp_info.skip_scale = 1;
12940                         else
12941                                 grp_info.skip_scale = 0;
12942                         ret = mlx5_flow_group_to_table(dev, tunnel,
12943                                                        jump_group,
12944                                                        &table,
12945                                                        &grp_info, error);
12946                         if (ret)
12947                                 return ret;
12948                         tbl = flow_dv_tbl_resource_get(dev, table, attr->egress,
12949                                                        attr->transfer,
12950                                                        !!dev_flow->external,
12951                                                        tunnel, jump_group, 0,
12952                                                        0, error);
12953                         if (!tbl)
12954                                 return rte_flow_error_set
12955                                                 (error, errno,
12956                                                  RTE_FLOW_ERROR_TYPE_ACTION,
12957                                                  NULL,
12958                                                  "cannot create jump action.");
12959                         if (flow_dv_jump_tbl_resource_register
12960                             (dev, tbl, dev_flow, error)) {
12961                                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
12962                                 return rte_flow_error_set
12963                                                 (error, errno,
12964                                                  RTE_FLOW_ERROR_TYPE_ACTION,
12965                                                  NULL,
12966                                                  "cannot create jump action.");
12967                         }
12968                         dev_flow->dv.actions[actions_n++] =
12969                                         dev_flow->dv.jump->action;
12970                         action_flags |= MLX5_FLOW_ACTION_JUMP;
12971                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
12972                         sample_act->action_flags |= MLX5_FLOW_ACTION_JUMP;
12973                         num_of_dest++;
12974                         break;
12975                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
12976                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
12977                         if (flow_dv_convert_action_modify_mac
12978                                         (mhdr_res, actions, error))
12979                                 return -rte_errno;
12980                         action_flags |= actions->type ==
12981                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
12982                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
12983                                         MLX5_FLOW_ACTION_SET_MAC_DST;
12984                         break;
12985                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
12986                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
12987                         if (flow_dv_convert_action_modify_ipv4
12988                                         (mhdr_res, actions, error))
12989                                 return -rte_errno;
12990                         action_flags |= actions->type ==
12991                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
12992                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
12993                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
12994                         break;
12995                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
12996                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
12997                         if (flow_dv_convert_action_modify_ipv6
12998                                         (mhdr_res, actions, error))
12999                                 return -rte_errno;
13000                         action_flags |= actions->type ==
13001                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
13002                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
13003                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
13004                         break;
13005                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
13006                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
13007                         if (flow_dv_convert_action_modify_tp
13008                                         (mhdr_res, actions, items,
13009                                          &flow_attr, dev_flow, !!(action_flags &
13010                                          MLX5_FLOW_ACTION_DECAP), error))
13011                                 return -rte_errno;
13012                         action_flags |= actions->type ==
13013                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
13014                                         MLX5_FLOW_ACTION_SET_TP_SRC :
13015                                         MLX5_FLOW_ACTION_SET_TP_DST;
13016                         break;
13017                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
13018                         if (flow_dv_convert_action_modify_dec_ttl
13019                                         (mhdr_res, items, &flow_attr, dev_flow,
13020                                          !!(action_flags &
13021                                          MLX5_FLOW_ACTION_DECAP), error))
13022                                 return -rte_errno;
13023                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
13024                         break;
13025                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
13026                         if (flow_dv_convert_action_modify_ttl
13027                                         (mhdr_res, actions, items, &flow_attr,
13028                                          dev_flow, !!(action_flags &
13029                                          MLX5_FLOW_ACTION_DECAP), error))
13030                                 return -rte_errno;
13031                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
13032                         break;
13033                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
13034                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
13035                         if (flow_dv_convert_action_modify_tcp_seq
13036                                         (mhdr_res, actions, error))
13037                                 return -rte_errno;
13038                         action_flags |= actions->type ==
13039                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
13040                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
13041                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
13042                         break;
13043
13044                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
13045                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
13046                         if (flow_dv_convert_action_modify_tcp_ack
13047                                         (mhdr_res, actions, error))
13048                                 return -rte_errno;
13049                         action_flags |= actions->type ==
13050                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
13051                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
13052                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
13053                         break;
13054                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
13055                         if (flow_dv_convert_action_set_reg
13056                                         (mhdr_res, actions, error))
13057                                 return -rte_errno;
13058                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
13059                         break;
13060                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
13061                         if (flow_dv_convert_action_copy_mreg
13062                                         (dev, mhdr_res, actions, error))
13063                                 return -rte_errno;
13064                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
13065                         break;
13066                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
13067                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
13068                         dev_flow->handle->fate_action =
13069                                         MLX5_FLOW_FATE_DEFAULT_MISS;
13070                         break;
13071                 case RTE_FLOW_ACTION_TYPE_METER:
13072                         if (!wks->fm)
13073                                 return rte_flow_error_set(error, rte_errno,
13074                                         RTE_FLOW_ERROR_TYPE_ACTION,
13075                                         NULL, "Failed to get meter in flow.");
13076                         /* Set the meter action. */
13077                         dev_flow->dv.actions[actions_n++] =
13078                                 wks->fm->meter_action;
13079                         action_flags |= MLX5_FLOW_ACTION_METER;
13080                         break;
13081                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
13082                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
13083                                                               actions, error))
13084                                 return -rte_errno;
13085                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
13086                         break;
13087                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
13088                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
13089                                                               actions, error))
13090                                 return -rte_errno;
13091                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
13092                         break;
13093                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
13094                         sample_act_pos = actions_n;
13095                         sample = (const struct rte_flow_action_sample *)
13096                                  action->conf;
13097                         actions_n++;
13098                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
13099                         /* put encap action into group if work with port id */
13100                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
13101                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
13102                                 sample_act->action_flags |=
13103                                                         MLX5_FLOW_ACTION_ENCAP;
13104                         break;
13105                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
13106                         if (flow_dv_convert_action_modify_field
13107                                         (dev, mhdr_res, actions, attr, error))
13108                                 return -rte_errno;
13109                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
13110                         break;
13111                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
13112                         owner_idx = (uint32_t)(uintptr_t)action->conf;
13113                         ct = flow_aso_ct_get_by_idx(dev, owner_idx);
13114                         if (!ct)
13115                                 return rte_flow_error_set(error, EINVAL,
13116                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13117                                                 NULL,
13118                                                 "Failed to get CT object.");
13119                         if (mlx5_aso_ct_available(priv->sh, ct))
13120                                 return rte_flow_error_set(error, rte_errno,
13121                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13122                                                 NULL,
13123                                                 "CT is unavailable.");
13124                         if (ct->is_original)
13125                                 dev_flow->dv.actions[actions_n] =
13126                                                         ct->dr_action_orig;
13127                         else
13128                                 dev_flow->dv.actions[actions_n] =
13129                                                         ct->dr_action_rply;
13130                         if (flow->ct == 0) {
13131                                 flow->indirect_type =
13132                                                 MLX5_INDIRECT_ACTION_TYPE_CT;
13133                                 flow->ct = owner_idx;
13134                                 __atomic_fetch_add(&ct->refcnt, 1,
13135                                                    __ATOMIC_RELAXED);
13136                         }
13137                         actions_n++;
13138                         action_flags |= MLX5_FLOW_ACTION_CT;
13139                         break;
13140                 case RTE_FLOW_ACTION_TYPE_END:
13141                         actions_end = true;
13142                         if (mhdr_res->actions_num) {
13143                                 /* create modify action if needed. */
13144                                 if (flow_dv_modify_hdr_resource_register
13145                                         (dev, mhdr_res, dev_flow, error))
13146                                         return -rte_errno;
13147                                 dev_flow->dv.actions[modify_action_position] =
13148                                         handle->dvh.modify_hdr->action;
13149                         }
13150                         /*
13151                          * Handle AGE and COUNT action by single HW counter
13152                          * when they are not shared.
13153                          */
13154                         if (action_flags & MLX5_FLOW_ACTION_AGE) {
13155                                 if ((non_shared_age && count) ||
13156                                     !(priv->sh->flow_hit_aso_en &&
13157                                       (attr->group || attr->transfer))) {
13158                                         /* Creates age by counters. */
13159                                         cnt_act = flow_dv_prepare_counter
13160                                                                 (dev, dev_flow,
13161                                                                  flow, count,
13162                                                                  non_shared_age,
13163                                                                  error);
13164                                         if (!cnt_act)
13165                                                 return -rte_errno;
13166                                         dev_flow->dv.actions[age_act_pos] =
13167                                                                 cnt_act->action;
13168                                         break;
13169                                 }
13170                                 if (!flow->age && non_shared_age) {
13171                                         flow->age = flow_dv_aso_age_alloc
13172                                                                 (dev, error);
13173                                         if (!flow->age)
13174                                                 return -rte_errno;
13175                                         flow_dv_aso_age_params_init
13176                                                     (dev, flow->age,
13177                                                      non_shared_age->context ?
13178                                                      non_shared_age->context :
13179                                                      (void *)(uintptr_t)
13180                                                      (dev_flow->flow_idx),
13181                                                      non_shared_age->timeout);
13182                                 }
13183                                 age_act = flow_aso_age_get_by_idx(dev,
13184                                                                   flow->age);
13185                                 dev_flow->dv.actions[age_act_pos] =
13186                                                              age_act->dr_action;
13187                         }
13188                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
13189                                 /*
13190                                  * Create one count action, to be used
13191                                  * by all sub-flows.
13192                                  */
13193                                 cnt_act = flow_dv_prepare_counter(dev, dev_flow,
13194                                                                   flow, count,
13195                                                                   NULL, error);
13196                                 if (!cnt_act)
13197                                         return -rte_errno;
13198                                 dev_flow->dv.actions[actions_n++] =
13199                                                                 cnt_act->action;
13200                         }
13201                 default:
13202                         break;
13203                 }
13204                 if (mhdr_res->actions_num &&
13205                     modify_action_position == UINT32_MAX)
13206                         modify_action_position = actions_n++;
13207         }
13208         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
13209                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
13210                 int item_type = items->type;
13211
13212                 if (!mlx5_flow_os_item_supported(item_type))
13213                         return rte_flow_error_set(error, ENOTSUP,
13214                                                   RTE_FLOW_ERROR_TYPE_ITEM,
13215                                                   NULL, "item not supported");
13216                 switch (item_type) {
13217                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
13218                         flow_dv_translate_item_port_id
13219                                 (dev, match_mask, match_value, items, attr);
13220                         last_item = MLX5_FLOW_ITEM_PORT_ID;
13221                         break;
13222                 case RTE_FLOW_ITEM_TYPE_ETH:
13223                         flow_dv_translate_item_eth(match_mask, match_value,
13224                                                    items, tunnel,
13225                                                    dev_flow->dv.group);
13226                         matcher.priority = action_flags &
13227                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
13228                                         !dev_flow->external ?
13229                                         MLX5_PRIORITY_MAP_L3 :
13230                                         MLX5_PRIORITY_MAP_L2;
13231                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
13232                                              MLX5_FLOW_LAYER_OUTER_L2;
13233                         break;
13234                 case RTE_FLOW_ITEM_TYPE_VLAN:
13235                         flow_dv_translate_item_vlan(dev_flow,
13236                                                     match_mask, match_value,
13237                                                     items, tunnel,
13238                                                     dev_flow->dv.group);
13239                         matcher.priority = MLX5_PRIORITY_MAP_L2;
13240                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
13241                                               MLX5_FLOW_LAYER_INNER_VLAN) :
13242                                              (MLX5_FLOW_LAYER_OUTER_L2 |
13243                                               MLX5_FLOW_LAYER_OUTER_VLAN);
13244                         break;
13245                 case RTE_FLOW_ITEM_TYPE_IPV4:
13246                         mlx5_flow_tunnel_ip_check(items, next_protocol,
13247                                                   &item_flags, &tunnel);
13248                         flow_dv_translate_item_ipv4(match_mask, match_value,
13249                                                     items, tunnel,
13250                                                     dev_flow->dv.group);
13251                         matcher.priority = MLX5_PRIORITY_MAP_L3;
13252                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
13253                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
13254                         if (items->mask != NULL &&
13255                             ((const struct rte_flow_item_ipv4 *)
13256                              items->mask)->hdr.next_proto_id) {
13257                                 next_protocol =
13258                                         ((const struct rte_flow_item_ipv4 *)
13259                                          (items->spec))->hdr.next_proto_id;
13260                                 next_protocol &=
13261                                         ((const struct rte_flow_item_ipv4 *)
13262                                          (items->mask))->hdr.next_proto_id;
13263                         } else {
13264                                 /* Reset for inner layer. */
13265                                 next_protocol = 0xff;
13266                         }
13267                         break;
13268                 case RTE_FLOW_ITEM_TYPE_IPV6:
13269                         mlx5_flow_tunnel_ip_check(items, next_protocol,
13270                                                   &item_flags, &tunnel);
13271                         flow_dv_translate_item_ipv6(match_mask, match_value,
13272                                                     items, tunnel,
13273                                                     dev_flow->dv.group);
13274                         matcher.priority = MLX5_PRIORITY_MAP_L3;
13275                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
13276                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
13277                         if (items->mask != NULL &&
13278                             ((const struct rte_flow_item_ipv6 *)
13279                              items->mask)->hdr.proto) {
13280                                 next_protocol =
13281                                         ((const struct rte_flow_item_ipv6 *)
13282                                          items->spec)->hdr.proto;
13283                                 next_protocol &=
13284                                         ((const struct rte_flow_item_ipv6 *)
13285                                          items->mask)->hdr.proto;
13286                         } else {
13287                                 /* Reset for inner layer. */
13288                                 next_protocol = 0xff;
13289                         }
13290                         break;
13291                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
13292                         flow_dv_translate_item_ipv6_frag_ext(match_mask,
13293                                                              match_value,
13294                                                              items, tunnel);
13295                         last_item = tunnel ?
13296                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
13297                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
13298                         if (items->mask != NULL &&
13299                             ((const struct rte_flow_item_ipv6_frag_ext *)
13300                              items->mask)->hdr.next_header) {
13301                                 next_protocol =
13302                                 ((const struct rte_flow_item_ipv6_frag_ext *)
13303                                  items->spec)->hdr.next_header;
13304                                 next_protocol &=
13305                                 ((const struct rte_flow_item_ipv6_frag_ext *)
13306                                  items->mask)->hdr.next_header;
13307                         } else {
13308                                 /* Reset for inner layer. */
13309                                 next_protocol = 0xff;
13310                         }
13311                         break;
13312                 case RTE_FLOW_ITEM_TYPE_TCP:
13313                         flow_dv_translate_item_tcp(match_mask, match_value,
13314                                                    items, tunnel);
13315                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13316                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
13317                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
13318                         break;
13319                 case RTE_FLOW_ITEM_TYPE_UDP:
13320                         flow_dv_translate_item_udp(match_mask, match_value,
13321                                                    items, tunnel);
13322                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13323                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
13324                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
13325                         break;
13326                 case RTE_FLOW_ITEM_TYPE_GRE:
13327                         flow_dv_translate_item_gre(match_mask, match_value,
13328                                                    items, tunnel);
13329                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13330                         last_item = MLX5_FLOW_LAYER_GRE;
13331                         break;
13332                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
13333                         flow_dv_translate_item_gre_key(match_mask,
13334                                                        match_value, items);
13335                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
13336                         break;
13337                 case RTE_FLOW_ITEM_TYPE_NVGRE:
13338                         flow_dv_translate_item_nvgre(match_mask, match_value,
13339                                                      items, tunnel);
13340                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13341                         last_item = MLX5_FLOW_LAYER_GRE;
13342                         break;
13343                 case RTE_FLOW_ITEM_TYPE_VXLAN:
13344                         flow_dv_translate_item_vxlan(dev, attr,
13345                                                      match_mask, match_value,
13346                                                      items, tunnel);
13347                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13348                         last_item = MLX5_FLOW_LAYER_VXLAN;
13349                         break;
13350                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
13351                         flow_dv_translate_item_vxlan_gpe(match_mask,
13352                                                          match_value, items,
13353                                                          tunnel);
13354                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13355                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
13356                         break;
13357                 case RTE_FLOW_ITEM_TYPE_GENEVE:
13358                         flow_dv_translate_item_geneve(match_mask, match_value,
13359                                                       items, tunnel);
13360                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13361                         last_item = MLX5_FLOW_LAYER_GENEVE;
13362                         break;
13363                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
13364                         ret = flow_dv_translate_item_geneve_opt(dev, match_mask,
13365                                                           match_value,
13366                                                           items, error);
13367                         if (ret)
13368                                 return rte_flow_error_set(error, -ret,
13369                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
13370                                         "cannot create GENEVE TLV option");
13371                         flow->geneve_tlv_option = 1;
13372                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
13373                         break;
13374                 case RTE_FLOW_ITEM_TYPE_MPLS:
13375                         flow_dv_translate_item_mpls(match_mask, match_value,
13376                                                     items, last_item, tunnel);
13377                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13378                         last_item = MLX5_FLOW_LAYER_MPLS;
13379                         break;
13380                 case RTE_FLOW_ITEM_TYPE_MARK:
13381                         flow_dv_translate_item_mark(dev, match_mask,
13382                                                     match_value, items);
13383                         last_item = MLX5_FLOW_ITEM_MARK;
13384                         break;
13385                 case RTE_FLOW_ITEM_TYPE_META:
13386                         flow_dv_translate_item_meta(dev, match_mask,
13387                                                     match_value, attr, items);
13388                         last_item = MLX5_FLOW_ITEM_METADATA;
13389                         break;
13390                 case RTE_FLOW_ITEM_TYPE_ICMP:
13391                         flow_dv_translate_item_icmp(match_mask, match_value,
13392                                                     items, tunnel);
13393                         last_item = MLX5_FLOW_LAYER_ICMP;
13394                         break;
13395                 case RTE_FLOW_ITEM_TYPE_ICMP6:
13396                         flow_dv_translate_item_icmp6(match_mask, match_value,
13397                                                       items, tunnel);
13398                         last_item = MLX5_FLOW_LAYER_ICMP6;
13399                         break;
13400                 case RTE_FLOW_ITEM_TYPE_TAG:
13401                         flow_dv_translate_item_tag(dev, match_mask,
13402                                                    match_value, items);
13403                         last_item = MLX5_FLOW_ITEM_TAG;
13404                         break;
13405                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
13406                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
13407                                                         match_value, items);
13408                         last_item = MLX5_FLOW_ITEM_TAG;
13409                         break;
13410                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
13411                         flow_dv_translate_item_tx_queue(dev, match_mask,
13412                                                         match_value,
13413                                                         items);
13414                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
13415                         break;
13416                 case RTE_FLOW_ITEM_TYPE_GTP:
13417                         flow_dv_translate_item_gtp(match_mask, match_value,
13418                                                    items, tunnel);
13419                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13420                         last_item = MLX5_FLOW_LAYER_GTP;
13421                         break;
13422                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
13423                         ret = flow_dv_translate_item_gtp_psc(match_mask,
13424                                                           match_value,
13425                                                           items);
13426                         if (ret)
13427                                 return rte_flow_error_set(error, -ret,
13428                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
13429                                         "cannot create GTP PSC item");
13430                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
13431                         break;
13432                 case RTE_FLOW_ITEM_TYPE_ECPRI:
13433                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
13434                                 /* Create it only the first time to be used. */
13435                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
13436                                 if (ret)
13437                                         return rte_flow_error_set
13438                                                 (error, -ret,
13439                                                 RTE_FLOW_ERROR_TYPE_ITEM,
13440                                                 NULL,
13441                                                 "cannot create eCPRI parser");
13442                         }
13443                         flow_dv_translate_item_ecpri(dev, match_mask,
13444                                                      match_value, items,
13445                                                      last_item);
13446                         /* No other protocol should follow eCPRI layer. */
13447                         last_item = MLX5_FLOW_LAYER_ECPRI;
13448                         break;
13449                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
13450                         flow_dv_translate_item_integrity(match_mask,
13451                                                          match_value,
13452                                                          head_item, items);
13453                         break;
13454                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
13455                         flow_dv_translate_item_aso_ct(dev, match_mask,
13456                                                       match_value, items);
13457                         break;
13458                 default:
13459                         break;
13460                 }
13461                 item_flags |= last_item;
13462         }
13463         /*
13464          * When E-Switch mode is enabled, we have two cases where we need to
13465          * set the source port manually.
13466          * The first one, is in case of Nic steering rule, and the second is
13467          * E-Switch rule where no port_id item was found. In both cases
13468          * the source port is set according the current port in use.
13469          */
13470         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
13471             (priv->representor || priv->master)) {
13472                 if (flow_dv_translate_item_port_id(dev, match_mask,
13473                                                    match_value, NULL, attr))
13474                         return -rte_errno;
13475         }
13476 #ifdef RTE_LIBRTE_MLX5_DEBUG
13477         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
13478                                               dev_flow->dv.value.buf));
13479 #endif
13480         /*
13481          * Layers may be already initialized from prefix flow if this dev_flow
13482          * is the suffix flow.
13483          */
13484         handle->layers |= item_flags;
13485         if (action_flags & MLX5_FLOW_ACTION_RSS)
13486                 flow_dv_hashfields_set(dev_flow, rss_desc);
13487         /* If has RSS action in the sample action, the Sample/Mirror resource
13488          * should be registered after the hash filed be update.
13489          */
13490         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
13491                 ret = flow_dv_translate_action_sample(dev,
13492                                                       sample,
13493                                                       dev_flow, attr,
13494                                                       &num_of_dest,
13495                                                       sample_actions,
13496                                                       &sample_res,
13497                                                       error);
13498                 if (ret < 0)
13499                         return ret;
13500                 ret = flow_dv_create_action_sample(dev,
13501                                                    dev_flow,
13502                                                    num_of_dest,
13503                                                    &sample_res,
13504                                                    &mdest_res,
13505                                                    sample_actions,
13506                                                    action_flags,
13507                                                    error);
13508                 if (ret < 0)
13509                         return rte_flow_error_set
13510                                                 (error, rte_errno,
13511                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13512                                                 NULL,
13513                                                 "cannot create sample action");
13514                 if (num_of_dest > 1) {
13515                         dev_flow->dv.actions[sample_act_pos] =
13516                         dev_flow->dv.dest_array_res->action;
13517                 } else {
13518                         dev_flow->dv.actions[sample_act_pos] =
13519                         dev_flow->dv.sample_res->verbs_action;
13520                 }
13521         }
13522         /*
13523          * For multiple destination (sample action with ratio=1), the encap
13524          * action and port id action will be combined into group action.
13525          * So need remove the original these actions in the flow and only
13526          * use the sample action instead of.
13527          */
13528         if (num_of_dest > 1 &&
13529             (sample_act->dr_port_id_action || sample_act->dr_jump_action)) {
13530                 int i;
13531                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
13532
13533                 for (i = 0; i < actions_n; i++) {
13534                         if ((sample_act->dr_encap_action &&
13535                                 sample_act->dr_encap_action ==
13536                                 dev_flow->dv.actions[i]) ||
13537                                 (sample_act->dr_port_id_action &&
13538                                 sample_act->dr_port_id_action ==
13539                                 dev_flow->dv.actions[i]) ||
13540                                 (sample_act->dr_jump_action &&
13541                                 sample_act->dr_jump_action ==
13542                                 dev_flow->dv.actions[i]))
13543                                 continue;
13544                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
13545                 }
13546                 memcpy((void *)dev_flow->dv.actions,
13547                                 (void *)temp_actions,
13548                                 tmp_actions_n * sizeof(void *));
13549                 actions_n = tmp_actions_n;
13550         }
13551         dev_flow->dv.actions_n = actions_n;
13552         dev_flow->act_flags = action_flags;
13553         if (wks->skip_matcher_reg)
13554                 return 0;
13555         /* Register matcher. */
13556         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
13557                                     matcher.mask.size);
13558         matcher.priority = mlx5_get_matcher_priority(dev, attr,
13559                                                      matcher.priority,
13560                                                      dev_flow->external);
13561         /**
13562          * When creating meter drop flow in drop table, using original
13563          * 5-tuple match, the matcher priority should be lower than
13564          * mtr_id matcher.
13565          */
13566         if (attr->group == MLX5_FLOW_TABLE_LEVEL_METER &&
13567             dev_flow->dv.table_id == MLX5_MTR_TABLE_ID_DROP &&
13568             matcher.priority <= MLX5_REG_BITS)
13569                 matcher.priority += MLX5_REG_BITS;
13570         /* reserved field no needs to be set to 0 here. */
13571         tbl_key.is_fdb = attr->transfer;
13572         tbl_key.is_egress = attr->egress;
13573         tbl_key.level = dev_flow->dv.group;
13574         tbl_key.id = dev_flow->dv.table_id;
13575         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow,
13576                                      tunnel, attr->group, error))
13577                 return -rte_errno;
13578         return 0;
13579 }
13580
13581 /**
13582  * Set hash RX queue by hash fields (see enum ibv_rx_hash_fields)
13583  * and tunnel.
13584  *
13585  * @param[in, out] action
13586  *   Shred RSS action holding hash RX queue objects.
13587  * @param[in] hash_fields
13588  *   Defines combination of packet fields to participate in RX hash.
13589  * @param[in] tunnel
13590  *   Tunnel type
13591  * @param[in] hrxq_idx
13592  *   Hash RX queue index to set.
13593  *
13594  * @return
13595  *   0 on success, otherwise negative errno value.
13596  */
13597 static int
13598 __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action,
13599                               const uint64_t hash_fields,
13600                               uint32_t hrxq_idx)
13601 {
13602         uint32_t *hrxqs = action->hrxq;
13603
13604         switch (hash_fields & ~IBV_RX_HASH_INNER) {
13605         case MLX5_RSS_HASH_IPV4:
13606                 /* fall-through. */
13607         case MLX5_RSS_HASH_IPV4_DST_ONLY:
13608                 /* fall-through. */
13609         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
13610                 hrxqs[0] = hrxq_idx;
13611                 return 0;
13612         case MLX5_RSS_HASH_IPV4_TCP:
13613                 /* fall-through. */
13614         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
13615                 /* fall-through. */
13616         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
13617                 hrxqs[1] = hrxq_idx;
13618                 return 0;
13619         case MLX5_RSS_HASH_IPV4_UDP:
13620                 /* fall-through. */
13621         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
13622                 /* fall-through. */
13623         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
13624                 hrxqs[2] = hrxq_idx;
13625                 return 0;
13626         case MLX5_RSS_HASH_IPV6:
13627                 /* fall-through. */
13628         case MLX5_RSS_HASH_IPV6_DST_ONLY:
13629                 /* fall-through. */
13630         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
13631                 hrxqs[3] = hrxq_idx;
13632                 return 0;
13633         case MLX5_RSS_HASH_IPV6_TCP:
13634                 /* fall-through. */
13635         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
13636                 /* fall-through. */
13637         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
13638                 hrxqs[4] = hrxq_idx;
13639                 return 0;
13640         case MLX5_RSS_HASH_IPV6_UDP:
13641                 /* fall-through. */
13642         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
13643                 /* fall-through. */
13644         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
13645                 hrxqs[5] = hrxq_idx;
13646                 return 0;
13647         case MLX5_RSS_HASH_NONE:
13648                 hrxqs[6] = hrxq_idx;
13649                 return 0;
13650         default:
13651                 return -1;
13652         }
13653 }
13654
13655 /**
13656  * Look up for hash RX queue by hash fields (see enum ibv_rx_hash_fields)
13657  * and tunnel.
13658  *
13659  * @param[in] dev
13660  *   Pointer to the Ethernet device structure.
13661  * @param[in] idx
13662  *   Shared RSS action ID holding hash RX queue objects.
13663  * @param[in] hash_fields
13664  *   Defines combination of packet fields to participate in RX hash.
13665  * @param[in] tunnel
13666  *   Tunnel type
13667  *
13668  * @return
13669  *   Valid hash RX queue index, otherwise 0.
13670  */
13671 static uint32_t
13672 __flow_dv_action_rss_hrxq_lookup(struct rte_eth_dev *dev, uint32_t idx,
13673                                  const uint64_t hash_fields)
13674 {
13675         struct mlx5_priv *priv = dev->data->dev_private;
13676         struct mlx5_shared_action_rss *shared_rss =
13677             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
13678         const uint32_t *hrxqs = shared_rss->hrxq;
13679
13680         switch (hash_fields & ~IBV_RX_HASH_INNER) {
13681         case MLX5_RSS_HASH_IPV4:
13682                 /* fall-through. */
13683         case MLX5_RSS_HASH_IPV4_DST_ONLY:
13684                 /* fall-through. */
13685         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
13686                 return hrxqs[0];
13687         case MLX5_RSS_HASH_IPV4_TCP:
13688                 /* fall-through. */
13689         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
13690                 /* fall-through. */
13691         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
13692                 return hrxqs[1];
13693         case MLX5_RSS_HASH_IPV4_UDP:
13694                 /* fall-through. */
13695         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
13696                 /* fall-through. */
13697         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
13698                 return hrxqs[2];
13699         case MLX5_RSS_HASH_IPV6:
13700                 /* fall-through. */
13701         case MLX5_RSS_HASH_IPV6_DST_ONLY:
13702                 /* fall-through. */
13703         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
13704                 return hrxqs[3];
13705         case MLX5_RSS_HASH_IPV6_TCP:
13706                 /* fall-through. */
13707         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
13708                 /* fall-through. */
13709         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
13710                 return hrxqs[4];
13711         case MLX5_RSS_HASH_IPV6_UDP:
13712                 /* fall-through. */
13713         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
13714                 /* fall-through. */
13715         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
13716                 return hrxqs[5];
13717         case MLX5_RSS_HASH_NONE:
13718                 return hrxqs[6];
13719         default:
13720                 return 0;
13721         }
13722
13723 }
13724
13725 /**
13726  * Apply the flow to the NIC, lock free,
13727  * (mutex should be acquired by caller).
13728  *
13729  * @param[in] dev
13730  *   Pointer to the Ethernet device structure.
13731  * @param[in, out] flow
13732  *   Pointer to flow structure.
13733  * @param[out] error
13734  *   Pointer to error structure.
13735  *
13736  * @return
13737  *   0 on success, a negative errno value otherwise and rte_errno is set.
13738  */
13739 static int
13740 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
13741               struct rte_flow_error *error)
13742 {
13743         struct mlx5_flow_dv_workspace *dv;
13744         struct mlx5_flow_handle *dh;
13745         struct mlx5_flow_handle_dv *dv_h;
13746         struct mlx5_flow *dev_flow;
13747         struct mlx5_priv *priv = dev->data->dev_private;
13748         uint32_t handle_idx;
13749         int n;
13750         int err;
13751         int idx;
13752         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
13753         struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc;
13754         uint8_t misc_mask;
13755
13756         MLX5_ASSERT(wks);
13757         for (idx = wks->flow_idx - 1; idx >= 0; idx--) {
13758                 dev_flow = &wks->flows[idx];
13759                 dv = &dev_flow->dv;
13760                 dh = dev_flow->handle;
13761                 dv_h = &dh->dvh;
13762                 n = dv->actions_n;
13763                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
13764                         if (dv->transfer) {
13765                                 MLX5_ASSERT(priv->sh->dr_drop_action);
13766                                 dv->actions[n++] = priv->sh->dr_drop_action;
13767                         } else {
13768 #ifdef HAVE_MLX5DV_DR
13769                                 /* DR supports drop action placeholder. */
13770                                 MLX5_ASSERT(priv->sh->dr_drop_action);
13771                                 dv->actions[n++] = dv->group ?
13772                                         priv->sh->dr_drop_action :
13773                                         priv->root_drop_action;
13774 #else
13775                                 /* For DV we use the explicit drop queue. */
13776                                 MLX5_ASSERT(priv->drop_queue.hrxq);
13777                                 dv->actions[n++] =
13778                                                 priv->drop_queue.hrxq->action;
13779 #endif
13780                         }
13781                 } else if ((dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
13782                            !dv_h->rix_sample && !dv_h->rix_dest_array)) {
13783                         struct mlx5_hrxq *hrxq;
13784                         uint32_t hrxq_idx;
13785
13786                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow, rss_desc,
13787                                                     &hrxq_idx);
13788                         if (!hrxq) {
13789                                 rte_flow_error_set
13790                                         (error, rte_errno,
13791                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13792                                          "cannot get hash queue");
13793                                 goto error;
13794                         }
13795                         dh->rix_hrxq = hrxq_idx;
13796                         dv->actions[n++] = hrxq->action;
13797                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
13798                         struct mlx5_hrxq *hrxq = NULL;
13799                         uint32_t hrxq_idx;
13800
13801                         hrxq_idx = __flow_dv_action_rss_hrxq_lookup(dev,
13802                                                 rss_desc->shared_rss,
13803                                                 dev_flow->hash_fields);
13804                         if (hrxq_idx)
13805                                 hrxq = mlx5_ipool_get
13806                                         (priv->sh->ipool[MLX5_IPOOL_HRXQ],
13807                                          hrxq_idx);
13808                         if (!hrxq) {
13809                                 rte_flow_error_set
13810                                         (error, rte_errno,
13811                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13812                                          "cannot get hash queue");
13813                                 goto error;
13814                         }
13815                         dh->rix_srss = rss_desc->shared_rss;
13816                         dv->actions[n++] = hrxq->action;
13817                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
13818                         if (!priv->sh->default_miss_action) {
13819                                 rte_flow_error_set
13820                                         (error, rte_errno,
13821                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13822                                          "default miss action not be created.");
13823                                 goto error;
13824                         }
13825                         dv->actions[n++] = priv->sh->default_miss_action;
13826                 }
13827                 misc_mask = flow_dv_matcher_enable(dv->value.buf);
13828                 __flow_dv_adjust_buf_size(&dv->value.size, misc_mask);
13829                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
13830                                                (void *)&dv->value, n,
13831                                                dv->actions, &dh->drv_flow);
13832                 if (err) {
13833                         rte_flow_error_set
13834                                 (error, errno,
13835                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
13836                                 NULL,
13837                                 (!priv->config.allow_duplicate_pattern &&
13838                                 errno == EEXIST) ?
13839                                 "duplicating pattern is not allowed" :
13840                                 "hardware refuses to create flow");
13841                         goto error;
13842                 }
13843                 if (priv->vmwa_context &&
13844                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
13845                         /*
13846                          * The rule contains the VLAN pattern.
13847                          * For VF we are going to create VLAN
13848                          * interface to make hypervisor set correct
13849                          * e-Switch vport context.
13850                          */
13851                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
13852                 }
13853         }
13854         return 0;
13855 error:
13856         err = rte_errno; /* Save rte_errno before cleanup. */
13857         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
13858                        handle_idx, dh, next) {
13859                 /* hrxq is union, don't clear it if the flag is not set. */
13860                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && dh->rix_hrxq) {
13861                         mlx5_hrxq_release(dev, dh->rix_hrxq);
13862                         dh->rix_hrxq = 0;
13863                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
13864                         dh->rix_srss = 0;
13865                 }
13866                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
13867                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
13868         }
13869         rte_errno = err; /* Restore rte_errno. */
13870         return -rte_errno;
13871 }
13872
13873 void
13874 flow_dv_matcher_remove_cb(void *tool_ctx __rte_unused,
13875                           struct mlx5_list_entry *entry)
13876 {
13877         struct mlx5_flow_dv_matcher *resource = container_of(entry,
13878                                                              typeof(*resource),
13879                                                              entry);
13880
13881         claim_zero(mlx5_flow_os_destroy_flow_matcher(resource->matcher_object));
13882         mlx5_free(resource);
13883 }
13884
13885 /**
13886  * Release the flow matcher.
13887  *
13888  * @param dev
13889  *   Pointer to Ethernet device.
13890  * @param port_id
13891  *   Index to port ID action resource.
13892  *
13893  * @return
13894  *   1 while a reference on it exists, 0 when freed.
13895  */
13896 static int
13897 flow_dv_matcher_release(struct rte_eth_dev *dev,
13898                         struct mlx5_flow_handle *handle)
13899 {
13900         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
13901         struct mlx5_flow_tbl_data_entry *tbl = container_of(matcher->tbl,
13902                                                             typeof(*tbl), tbl);
13903         int ret;
13904
13905         MLX5_ASSERT(matcher->matcher_object);
13906         ret = mlx5_list_unregister(tbl->matchers, &matcher->entry);
13907         flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
13908         return ret;
13909 }
13910
13911 void
13912 flow_dv_encap_decap_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
13913 {
13914         struct mlx5_dev_ctx_shared *sh = tool_ctx;
13915         struct mlx5_flow_dv_encap_decap_resource *res =
13916                                        container_of(entry, typeof(*res), entry);
13917
13918         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
13919         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
13920 }
13921
13922 /**
13923  * Release an encap/decap resource.
13924  *
13925  * @param dev
13926  *   Pointer to Ethernet device.
13927  * @param encap_decap_idx
13928  *   Index of encap decap resource.
13929  *
13930  * @return
13931  *   1 while a reference on it exists, 0 when freed.
13932  */
13933 static int
13934 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
13935                                      uint32_t encap_decap_idx)
13936 {
13937         struct mlx5_priv *priv = dev->data->dev_private;
13938         struct mlx5_flow_dv_encap_decap_resource *resource;
13939
13940         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
13941                                   encap_decap_idx);
13942         if (!resource)
13943                 return 0;
13944         MLX5_ASSERT(resource->action);
13945         return mlx5_hlist_unregister(priv->sh->encaps_decaps, &resource->entry);
13946 }
13947
13948 /**
13949  * Release an jump to table action resource.
13950  *
13951  * @param dev
13952  *   Pointer to Ethernet device.
13953  * @param rix_jump
13954  *   Index to the jump action resource.
13955  *
13956  * @return
13957  *   1 while a reference on it exists, 0 when freed.
13958  */
13959 static int
13960 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
13961                                   uint32_t rix_jump)
13962 {
13963         struct mlx5_priv *priv = dev->data->dev_private;
13964         struct mlx5_flow_tbl_data_entry *tbl_data;
13965
13966         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
13967                                   rix_jump);
13968         if (!tbl_data)
13969                 return 0;
13970         return flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl_data->tbl);
13971 }
13972
13973 void
13974 flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
13975 {
13976         struct mlx5_flow_dv_modify_hdr_resource *res =
13977                 container_of(entry, typeof(*res), entry);
13978         struct mlx5_dev_ctx_shared *sh = tool_ctx;
13979
13980         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
13981         mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
13982 }
13983
13984 /**
13985  * Release a modify-header resource.
13986  *
13987  * @param dev
13988  *   Pointer to Ethernet device.
13989  * @param handle
13990  *   Pointer to mlx5_flow_handle.
13991  *
13992  * @return
13993  *   1 while a reference on it exists, 0 when freed.
13994  */
13995 static int
13996 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
13997                                     struct mlx5_flow_handle *handle)
13998 {
13999         struct mlx5_priv *priv = dev->data->dev_private;
14000         struct mlx5_flow_dv_modify_hdr_resource *entry = handle->dvh.modify_hdr;
14001
14002         MLX5_ASSERT(entry->action);
14003         return mlx5_hlist_unregister(priv->sh->modify_cmds, &entry->entry);
14004 }
14005
14006 void
14007 flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
14008 {
14009         struct mlx5_dev_ctx_shared *sh = tool_ctx;
14010         struct mlx5_flow_dv_port_id_action_resource *resource =
14011                                   container_of(entry, typeof(*resource), entry);
14012
14013         claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
14014         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
14015 }
14016
14017 /**
14018  * Release port ID action resource.
14019  *
14020  * @param dev
14021  *   Pointer to Ethernet device.
14022  * @param handle
14023  *   Pointer to mlx5_flow_handle.
14024  *
14025  * @return
14026  *   1 while a reference on it exists, 0 when freed.
14027  */
14028 static int
14029 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
14030                                         uint32_t port_id)
14031 {
14032         struct mlx5_priv *priv = dev->data->dev_private;
14033         struct mlx5_flow_dv_port_id_action_resource *resource;
14034
14035         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
14036         if (!resource)
14037                 return 0;
14038         MLX5_ASSERT(resource->action);
14039         return mlx5_list_unregister(priv->sh->port_id_action_list,
14040                                     &resource->entry);
14041 }
14042
14043 /**
14044  * Release shared RSS action resource.
14045  *
14046  * @param dev
14047  *   Pointer to Ethernet device.
14048  * @param srss
14049  *   Shared RSS action index.
14050  */
14051 static void
14052 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
14053 {
14054         struct mlx5_priv *priv = dev->data->dev_private;
14055         struct mlx5_shared_action_rss *shared_rss;
14056
14057         shared_rss = mlx5_ipool_get
14058                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], srss);
14059         __atomic_sub_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
14060 }
14061
14062 void
14063 flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
14064 {
14065         struct mlx5_dev_ctx_shared *sh = tool_ctx;
14066         struct mlx5_flow_dv_push_vlan_action_resource *resource =
14067                         container_of(entry, typeof(*resource), entry);
14068
14069         claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
14070         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
14071 }
14072
14073 /**
14074  * Release push vlan action resource.
14075  *
14076  * @param dev
14077  *   Pointer to Ethernet device.
14078  * @param handle
14079  *   Pointer to mlx5_flow_handle.
14080  *
14081  * @return
14082  *   1 while a reference on it exists, 0 when freed.
14083  */
14084 static int
14085 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
14086                                           struct mlx5_flow_handle *handle)
14087 {
14088         struct mlx5_priv *priv = dev->data->dev_private;
14089         struct mlx5_flow_dv_push_vlan_action_resource *resource;
14090         uint32_t idx = handle->dvh.rix_push_vlan;
14091
14092         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
14093         if (!resource)
14094                 return 0;
14095         MLX5_ASSERT(resource->action);
14096         return mlx5_list_unregister(priv->sh->push_vlan_action_list,
14097                                     &resource->entry);
14098 }
14099
14100 /**
14101  * Release the fate resource.
14102  *
14103  * @param dev
14104  *   Pointer to Ethernet device.
14105  * @param handle
14106  *   Pointer to mlx5_flow_handle.
14107  */
14108 static void
14109 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
14110                                struct mlx5_flow_handle *handle)
14111 {
14112         if (!handle->rix_fate)
14113                 return;
14114         switch (handle->fate_action) {
14115         case MLX5_FLOW_FATE_QUEUE:
14116                 if (!handle->dvh.rix_sample && !handle->dvh.rix_dest_array)
14117                         mlx5_hrxq_release(dev, handle->rix_hrxq);
14118                 break;
14119         case MLX5_FLOW_FATE_JUMP:
14120                 flow_dv_jump_tbl_resource_release(dev, handle->rix_jump);
14121                 break;
14122         case MLX5_FLOW_FATE_PORT_ID:
14123                 flow_dv_port_id_action_resource_release(dev,
14124                                 handle->rix_port_id_action);
14125                 break;
14126         default:
14127                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
14128                 break;
14129         }
14130         handle->rix_fate = 0;
14131 }
14132
14133 void
14134 flow_dv_sample_remove_cb(void *tool_ctx __rte_unused,
14135                          struct mlx5_list_entry *entry)
14136 {
14137         struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
14138                                                               typeof(*resource),
14139                                                               entry);
14140         struct rte_eth_dev *dev = resource->dev;
14141         struct mlx5_priv *priv = dev->data->dev_private;
14142
14143         if (resource->verbs_action)
14144                 claim_zero(mlx5_flow_os_destroy_flow_action
14145                                                       (resource->verbs_action));
14146         if (resource->normal_path_tbl)
14147                 flow_dv_tbl_resource_release(MLX5_SH(dev),
14148                                              resource->normal_path_tbl);
14149         flow_dv_sample_sub_actions_release(dev, &resource->sample_idx);
14150         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
14151         DRV_LOG(DEBUG, "sample resource %p: removed", (void *)resource);
14152 }
14153
14154 /**
14155  * Release an sample resource.
14156  *
14157  * @param dev
14158  *   Pointer to Ethernet device.
14159  * @param handle
14160  *   Pointer to mlx5_flow_handle.
14161  *
14162  * @return
14163  *   1 while a reference on it exists, 0 when freed.
14164  */
14165 static int
14166 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
14167                                      struct mlx5_flow_handle *handle)
14168 {
14169         struct mlx5_priv *priv = dev->data->dev_private;
14170         struct mlx5_flow_dv_sample_resource *resource;
14171
14172         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
14173                                   handle->dvh.rix_sample);
14174         if (!resource)
14175                 return 0;
14176         MLX5_ASSERT(resource->verbs_action);
14177         return mlx5_list_unregister(priv->sh->sample_action_list,
14178                                     &resource->entry);
14179 }
14180
14181 void
14182 flow_dv_dest_array_remove_cb(void *tool_ctx __rte_unused,
14183                              struct mlx5_list_entry *entry)
14184 {
14185         struct mlx5_flow_dv_dest_array_resource *resource =
14186                         container_of(entry, typeof(*resource), entry);
14187         struct rte_eth_dev *dev = resource->dev;
14188         struct mlx5_priv *priv = dev->data->dev_private;
14189         uint32_t i = 0;
14190
14191         MLX5_ASSERT(resource->action);
14192         if (resource->action)
14193                 claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
14194         for (; i < resource->num_of_dest; i++)
14195                 flow_dv_sample_sub_actions_release(dev,
14196                                                    &resource->sample_idx[i]);
14197         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
14198         DRV_LOG(DEBUG, "destination array resource %p: removed",
14199                 (void *)resource);
14200 }
14201
14202 /**
14203  * Release an destination array resource.
14204  *
14205  * @param dev
14206  *   Pointer to Ethernet device.
14207  * @param handle
14208  *   Pointer to mlx5_flow_handle.
14209  *
14210  * @return
14211  *   1 while a reference on it exists, 0 when freed.
14212  */
14213 static int
14214 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
14215                                     struct mlx5_flow_handle *handle)
14216 {
14217         struct mlx5_priv *priv = dev->data->dev_private;
14218         struct mlx5_flow_dv_dest_array_resource *resource;
14219
14220         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
14221                                   handle->dvh.rix_dest_array);
14222         if (!resource)
14223                 return 0;
14224         MLX5_ASSERT(resource->action);
14225         return mlx5_list_unregister(priv->sh->dest_array_list,
14226                                     &resource->entry);
14227 }
14228
14229 static void
14230 flow_dv_geneve_tlv_option_resource_release(struct rte_eth_dev *dev)
14231 {
14232         struct mlx5_priv *priv = dev->data->dev_private;
14233         struct mlx5_dev_ctx_shared *sh = priv->sh;
14234         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
14235                                 sh->geneve_tlv_option_resource;
14236         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
14237         if (geneve_opt_resource) {
14238                 if (!(__atomic_sub_fetch(&geneve_opt_resource->refcnt, 1,
14239                                          __ATOMIC_RELAXED))) {
14240                         claim_zero(mlx5_devx_cmd_destroy
14241                                         (geneve_opt_resource->obj));
14242                         mlx5_free(sh->geneve_tlv_option_resource);
14243                         sh->geneve_tlv_option_resource = NULL;
14244                 }
14245         }
14246         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
14247 }
14248
14249 /**
14250  * Remove the flow from the NIC but keeps it in memory.
14251  * Lock free, (mutex should be acquired by caller).
14252  *
14253  * @param[in] dev
14254  *   Pointer to Ethernet device.
14255  * @param[in, out] flow
14256  *   Pointer to flow structure.
14257  */
14258 static void
14259 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
14260 {
14261         struct mlx5_flow_handle *dh;
14262         uint32_t handle_idx;
14263         struct mlx5_priv *priv = dev->data->dev_private;
14264
14265         if (!flow)
14266                 return;
14267         handle_idx = flow->dev_handles;
14268         while (handle_idx) {
14269                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
14270                                     handle_idx);
14271                 if (!dh)
14272                         return;
14273                 if (dh->drv_flow) {
14274                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
14275                         dh->drv_flow = NULL;
14276                 }
14277                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE)
14278                         flow_dv_fate_resource_release(dev, dh);
14279                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
14280                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
14281                 handle_idx = dh->next.next;
14282         }
14283 }
14284
14285 /**
14286  * Remove the flow from the NIC and the memory.
14287  * Lock free, (mutex should be acquired by caller).
14288  *
14289  * @param[in] dev
14290  *   Pointer to the Ethernet device structure.
14291  * @param[in, out] flow
14292  *   Pointer to flow structure.
14293  */
14294 static void
14295 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
14296 {
14297         struct mlx5_flow_handle *dev_handle;
14298         struct mlx5_priv *priv = dev->data->dev_private;
14299         struct mlx5_flow_meter_info *fm = NULL;
14300         uint32_t srss = 0;
14301
14302         if (!flow)
14303                 return;
14304         flow_dv_remove(dev, flow);
14305         if (flow->counter) {
14306                 flow_dv_counter_free(dev, flow->counter);
14307                 flow->counter = 0;
14308         }
14309         if (flow->meter) {
14310                 fm = flow_dv_meter_find_by_idx(priv, flow->meter);
14311                 if (fm)
14312                         mlx5_flow_meter_detach(priv, fm);
14313                 flow->meter = 0;
14314         }
14315         /* Keep the current age handling by default. */
14316         if (flow->indirect_type == MLX5_INDIRECT_ACTION_TYPE_CT && flow->ct)
14317                 flow_dv_aso_ct_release(dev, flow->ct, NULL);
14318         else if (flow->age)
14319                 flow_dv_aso_age_release(dev, flow->age);
14320         if (flow->geneve_tlv_option) {
14321                 flow_dv_geneve_tlv_option_resource_release(dev);
14322                 flow->geneve_tlv_option = 0;
14323         }
14324         while (flow->dev_handles) {
14325                 uint32_t tmp_idx = flow->dev_handles;
14326
14327                 dev_handle = mlx5_ipool_get(priv->sh->ipool
14328                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
14329                 if (!dev_handle)
14330                         return;
14331                 flow->dev_handles = dev_handle->next.next;
14332                 if (dev_handle->dvh.matcher)
14333                         flow_dv_matcher_release(dev, dev_handle);
14334                 if (dev_handle->dvh.rix_sample)
14335                         flow_dv_sample_resource_release(dev, dev_handle);
14336                 if (dev_handle->dvh.rix_dest_array)
14337                         flow_dv_dest_array_resource_release(dev, dev_handle);
14338                 if (dev_handle->dvh.rix_encap_decap)
14339                         flow_dv_encap_decap_resource_release(dev,
14340                                 dev_handle->dvh.rix_encap_decap);
14341                 if (dev_handle->dvh.modify_hdr)
14342                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
14343                 if (dev_handle->dvh.rix_push_vlan)
14344                         flow_dv_push_vlan_action_resource_release(dev,
14345                                                                   dev_handle);
14346                 if (dev_handle->dvh.rix_tag)
14347                         flow_dv_tag_release(dev,
14348                                             dev_handle->dvh.rix_tag);
14349                 if (dev_handle->fate_action != MLX5_FLOW_FATE_SHARED_RSS)
14350                         flow_dv_fate_resource_release(dev, dev_handle);
14351                 else if (!srss)
14352                         srss = dev_handle->rix_srss;
14353                 if (fm && dev_handle->is_meter_flow_id &&
14354                     dev_handle->split_flow_id)
14355                         mlx5_ipool_free(fm->flow_ipool,
14356                                         dev_handle->split_flow_id);
14357                 else if (dev_handle->split_flow_id &&
14358                     !dev_handle->is_meter_flow_id)
14359                         mlx5_ipool_free(priv->sh->ipool
14360                                         [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
14361                                         dev_handle->split_flow_id);
14362                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
14363                            tmp_idx);
14364         }
14365         if (srss)
14366                 flow_dv_shared_rss_action_release(dev, srss);
14367 }
14368
14369 /**
14370  * Release array of hash RX queue objects.
14371  * Helper function.
14372  *
14373  * @param[in] dev
14374  *   Pointer to the Ethernet device structure.
14375  * @param[in, out] hrxqs
14376  *   Array of hash RX queue objects.
14377  *
14378  * @return
14379  *   Total number of references to hash RX queue objects in *hrxqs* array
14380  *   after this operation.
14381  */
14382 static int
14383 __flow_dv_hrxqs_release(struct rte_eth_dev *dev,
14384                         uint32_t (*hrxqs)[MLX5_RSS_HASH_FIELDS_LEN])
14385 {
14386         size_t i;
14387         int remaining = 0;
14388
14389         for (i = 0; i < RTE_DIM(*hrxqs); i++) {
14390                 int ret = mlx5_hrxq_release(dev, (*hrxqs)[i]);
14391
14392                 if (!ret)
14393                         (*hrxqs)[i] = 0;
14394                 remaining += ret;
14395         }
14396         return remaining;
14397 }
14398
14399 /**
14400  * Release all hash RX queue objects representing shared RSS action.
14401  *
14402  * @param[in] dev
14403  *   Pointer to the Ethernet device structure.
14404  * @param[in, out] action
14405  *   Shared RSS action to remove hash RX queue objects from.
14406  *
14407  * @return
14408  *   Total number of references to hash RX queue objects stored in *action*
14409  *   after this operation.
14410  *   Expected to be 0 if no external references held.
14411  */
14412 static int
14413 __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev,
14414                                  struct mlx5_shared_action_rss *shared_rss)
14415 {
14416         return __flow_dv_hrxqs_release(dev, &shared_rss->hrxq);
14417 }
14418
14419 /**
14420  * Adjust L3/L4 hash value of pre-created shared RSS hrxq according to
14421  * user input.
14422  *
14423  * Only one hash value is available for one L3+L4 combination:
14424  * for example:
14425  * MLX5_RSS_HASH_IPV4, MLX5_RSS_HASH_IPV4_SRC_ONLY, and
14426  * MLX5_RSS_HASH_IPV4_DST_ONLY are mutually exclusive so they can share
14427  * same slot in mlx5_rss_hash_fields.
14428  *
14429  * @param[in] rss
14430  *   Pointer to the shared action RSS conf.
14431  * @param[in, out] hash_field
14432  *   hash_field variable needed to be adjusted.
14433  *
14434  * @return
14435  *   void
14436  */
14437 static void
14438 __flow_dv_action_rss_l34_hash_adjust(struct mlx5_shared_action_rss *rss,
14439                                      uint64_t *hash_field)
14440 {
14441         uint64_t rss_types = rss->origin.types;
14442
14443         switch (*hash_field & ~IBV_RX_HASH_INNER) {
14444         case MLX5_RSS_HASH_IPV4:
14445                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
14446                         *hash_field &= ~MLX5_RSS_HASH_IPV4;
14447                         if (rss_types & ETH_RSS_L3_DST_ONLY)
14448                                 *hash_field |= IBV_RX_HASH_DST_IPV4;
14449                         else if (rss_types & ETH_RSS_L3_SRC_ONLY)
14450                                 *hash_field |= IBV_RX_HASH_SRC_IPV4;
14451                         else
14452                                 *hash_field |= MLX5_RSS_HASH_IPV4;
14453                 }
14454                 return;
14455         case MLX5_RSS_HASH_IPV6:
14456                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
14457                         *hash_field &= ~MLX5_RSS_HASH_IPV6;
14458                         if (rss_types & ETH_RSS_L3_DST_ONLY)
14459                                 *hash_field |= IBV_RX_HASH_DST_IPV6;
14460                         else if (rss_types & ETH_RSS_L3_SRC_ONLY)
14461                                 *hash_field |= IBV_RX_HASH_SRC_IPV6;
14462                         else
14463                                 *hash_field |= MLX5_RSS_HASH_IPV6;
14464                 }
14465                 return;
14466         case MLX5_RSS_HASH_IPV4_UDP:
14467                 /* fall-through. */
14468         case MLX5_RSS_HASH_IPV6_UDP:
14469                 if (rss_types & ETH_RSS_UDP) {
14470                         *hash_field &= ~MLX5_UDP_IBV_RX_HASH;
14471                         if (rss_types & ETH_RSS_L4_DST_ONLY)
14472                                 *hash_field |= IBV_RX_HASH_DST_PORT_UDP;
14473                         else if (rss_types & ETH_RSS_L4_SRC_ONLY)
14474                                 *hash_field |= IBV_RX_HASH_SRC_PORT_UDP;
14475                         else
14476                                 *hash_field |= MLX5_UDP_IBV_RX_HASH;
14477                 }
14478                 return;
14479         case MLX5_RSS_HASH_IPV4_TCP:
14480                 /* fall-through. */
14481         case MLX5_RSS_HASH_IPV6_TCP:
14482                 if (rss_types & ETH_RSS_TCP) {
14483                         *hash_field &= ~MLX5_TCP_IBV_RX_HASH;
14484                         if (rss_types & ETH_RSS_L4_DST_ONLY)
14485                                 *hash_field |= IBV_RX_HASH_DST_PORT_TCP;
14486                         else if (rss_types & ETH_RSS_L4_SRC_ONLY)
14487                                 *hash_field |= IBV_RX_HASH_SRC_PORT_TCP;
14488                         else
14489                                 *hash_field |= MLX5_TCP_IBV_RX_HASH;
14490                 }
14491                 return;
14492         default:
14493                 return;
14494         }
14495 }
14496
14497 /**
14498  * Setup shared RSS action.
14499  * Prepare set of hash RX queue objects sufficient to handle all valid
14500  * hash_fields combinations (see enum ibv_rx_hash_fields).
14501  *
14502  * @param[in] dev
14503  *   Pointer to the Ethernet device structure.
14504  * @param[in] action_idx
14505  *   Shared RSS action ipool index.
14506  * @param[in, out] action
14507  *   Partially initialized shared RSS action.
14508  * @param[out] error
14509  *   Perform verbose error reporting if not NULL. Initialized in case of
14510  *   error only.
14511  *
14512  * @return
14513  *   0 on success, otherwise negative errno value.
14514  */
14515 static int
14516 __flow_dv_action_rss_setup(struct rte_eth_dev *dev,
14517                            uint32_t action_idx,
14518                            struct mlx5_shared_action_rss *shared_rss,
14519                            struct rte_flow_error *error)
14520 {
14521         struct mlx5_flow_rss_desc rss_desc = { 0 };
14522         size_t i;
14523         int err;
14524
14525         if (mlx5_ind_table_obj_setup(dev, shared_rss->ind_tbl)) {
14526                 return rte_flow_error_set(error, rte_errno,
14527                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14528                                           "cannot setup indirection table");
14529         }
14530         memcpy(rss_desc.key, shared_rss->origin.key, MLX5_RSS_HASH_KEY_LEN);
14531         rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN;
14532         rss_desc.const_q = shared_rss->origin.queue;
14533         rss_desc.queue_num = shared_rss->origin.queue_num;
14534         /* Set non-zero value to indicate a shared RSS. */
14535         rss_desc.shared_rss = action_idx;
14536         rss_desc.ind_tbl = shared_rss->ind_tbl;
14537         for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) {
14538                 uint32_t hrxq_idx;
14539                 uint64_t hash_fields = mlx5_rss_hash_fields[i];
14540                 int tunnel = 0;
14541
14542                 __flow_dv_action_rss_l34_hash_adjust(shared_rss, &hash_fields);
14543                 if (shared_rss->origin.level > 1) {
14544                         hash_fields |= IBV_RX_HASH_INNER;
14545                         tunnel = 1;
14546                 }
14547                 rss_desc.tunnel = tunnel;
14548                 rss_desc.hash_fields = hash_fields;
14549                 hrxq_idx = mlx5_hrxq_get(dev, &rss_desc);
14550                 if (!hrxq_idx) {
14551                         rte_flow_error_set
14552                                 (error, rte_errno,
14553                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14554                                  "cannot get hash queue");
14555                         goto error_hrxq_new;
14556                 }
14557                 err = __flow_dv_action_rss_hrxq_set
14558                         (shared_rss, hash_fields, hrxq_idx);
14559                 MLX5_ASSERT(!err);
14560         }
14561         return 0;
14562 error_hrxq_new:
14563         err = rte_errno;
14564         __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
14565         if (!mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true))
14566                 shared_rss->ind_tbl = NULL;
14567         rte_errno = err;
14568         return -rte_errno;
14569 }
14570
14571 /**
14572  * Create shared RSS action.
14573  *
14574  * @param[in] dev
14575  *   Pointer to the Ethernet device structure.
14576  * @param[in] conf
14577  *   Shared action configuration.
14578  * @param[in] rss
14579  *   RSS action specification used to create shared action.
14580  * @param[out] error
14581  *   Perform verbose error reporting if not NULL. Initialized in case of
14582  *   error only.
14583  *
14584  * @return
14585  *   A valid shared action ID in case of success, 0 otherwise and
14586  *   rte_errno is set.
14587  */
14588 static uint32_t
14589 __flow_dv_action_rss_create(struct rte_eth_dev *dev,
14590                             const struct rte_flow_indir_action_conf *conf,
14591                             const struct rte_flow_action_rss *rss,
14592                             struct rte_flow_error *error)
14593 {
14594         struct mlx5_priv *priv = dev->data->dev_private;
14595         struct mlx5_shared_action_rss *shared_rss = NULL;
14596         void *queue = NULL;
14597         struct rte_flow_action_rss *origin;
14598         const uint8_t *rss_key;
14599         uint32_t queue_size = rss->queue_num * sizeof(uint16_t);
14600         uint32_t idx;
14601
14602         RTE_SET_USED(conf);
14603         queue = mlx5_malloc(0, RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
14604                             0, SOCKET_ID_ANY);
14605         shared_rss = mlx5_ipool_zmalloc
14606                          (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], &idx);
14607         if (!shared_rss || !queue) {
14608                 rte_flow_error_set(error, ENOMEM,
14609                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14610                                    "cannot allocate resource memory");
14611                 goto error_rss_init;
14612         }
14613         if (idx > (1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET)) {
14614                 rte_flow_error_set(error, E2BIG,
14615                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14616                                    "rss action number out of range");
14617                 goto error_rss_init;
14618         }
14619         shared_rss->ind_tbl = mlx5_malloc(MLX5_MEM_ZERO,
14620                                           sizeof(*shared_rss->ind_tbl),
14621                                           0, SOCKET_ID_ANY);
14622         if (!shared_rss->ind_tbl) {
14623                 rte_flow_error_set(error, ENOMEM,
14624                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14625                                    "cannot allocate resource memory");
14626                 goto error_rss_init;
14627         }
14628         memcpy(queue, rss->queue, queue_size);
14629         shared_rss->ind_tbl->queues = queue;
14630         shared_rss->ind_tbl->queues_n = rss->queue_num;
14631         origin = &shared_rss->origin;
14632         origin->func = rss->func;
14633         origin->level = rss->level;
14634         /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
14635         origin->types = !rss->types ? ETH_RSS_IP : rss->types;
14636         /* NULL RSS key indicates default RSS key. */
14637         rss_key = !rss->key ? rss_hash_default_key : rss->key;
14638         memcpy(shared_rss->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
14639         origin->key = &shared_rss->key[0];
14640         origin->key_len = MLX5_RSS_HASH_KEY_LEN;
14641         origin->queue = queue;
14642         origin->queue_num = rss->queue_num;
14643         if (__flow_dv_action_rss_setup(dev, idx, shared_rss, error))
14644                 goto error_rss_init;
14645         rte_spinlock_init(&shared_rss->action_rss_sl);
14646         __atomic_add_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
14647         rte_spinlock_lock(&priv->shared_act_sl);
14648         ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14649                      &priv->rss_shared_actions, idx, shared_rss, next);
14650         rte_spinlock_unlock(&priv->shared_act_sl);
14651         return idx;
14652 error_rss_init:
14653         if (shared_rss) {
14654                 if (shared_rss->ind_tbl)
14655                         mlx5_free(shared_rss->ind_tbl);
14656                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14657                                 idx);
14658         }
14659         if (queue)
14660                 mlx5_free(queue);
14661         return 0;
14662 }
14663
14664 /**
14665  * Destroy the shared RSS action.
14666  * Release related hash RX queue objects.
14667  *
14668  * @param[in] dev
14669  *   Pointer to the Ethernet device structure.
14670  * @param[in] idx
14671  *   The shared RSS action object ID to be removed.
14672  * @param[out] error
14673  *   Perform verbose error reporting if not NULL. Initialized in case of
14674  *   error only.
14675  *
14676  * @return
14677  *   0 on success, otherwise negative errno value.
14678  */
14679 static int
14680 __flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx,
14681                              struct rte_flow_error *error)
14682 {
14683         struct mlx5_priv *priv = dev->data->dev_private;
14684         struct mlx5_shared_action_rss *shared_rss =
14685             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
14686         uint32_t old_refcnt = 1;
14687         int remaining;
14688         uint16_t *queue = NULL;
14689
14690         if (!shared_rss)
14691                 return rte_flow_error_set(error, EINVAL,
14692                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14693                                           "invalid shared action");
14694         if (!__atomic_compare_exchange_n(&shared_rss->refcnt, &old_refcnt,
14695                                          0, 0, __ATOMIC_ACQUIRE,
14696                                          __ATOMIC_RELAXED))
14697                 return rte_flow_error_set(error, EBUSY,
14698                                           RTE_FLOW_ERROR_TYPE_ACTION,
14699                                           NULL,
14700                                           "shared rss has references");
14701         remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
14702         if (remaining)
14703                 return rte_flow_error_set(error, EBUSY,
14704                                           RTE_FLOW_ERROR_TYPE_ACTION,
14705                                           NULL,
14706                                           "shared rss hrxq has references");
14707         queue = shared_rss->ind_tbl->queues;
14708         remaining = mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true);
14709         if (remaining)
14710                 return rte_flow_error_set(error, EBUSY,
14711                                           RTE_FLOW_ERROR_TYPE_ACTION,
14712                                           NULL,
14713                                           "shared rss indirection table has"
14714                                           " references");
14715         mlx5_free(queue);
14716         rte_spinlock_lock(&priv->shared_act_sl);
14717         ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14718                      &priv->rss_shared_actions, idx, shared_rss, next);
14719         rte_spinlock_unlock(&priv->shared_act_sl);
14720         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14721                         idx);
14722         return 0;
14723 }
14724
14725 /**
14726  * Create indirect action, lock free,
14727  * (mutex should be acquired by caller).
14728  * Dispatcher for action type specific call.
14729  *
14730  * @param[in] dev
14731  *   Pointer to the Ethernet device structure.
14732  * @param[in] conf
14733  *   Shared action configuration.
14734  * @param[in] action
14735  *   Action specification used to create indirect action.
14736  * @param[out] error
14737  *   Perform verbose error reporting if not NULL. Initialized in case of
14738  *   error only.
14739  *
14740  * @return
14741  *   A valid shared action handle in case of success, NULL otherwise and
14742  *   rte_errno is set.
14743  */
14744 static struct rte_flow_action_handle *
14745 flow_dv_action_create(struct rte_eth_dev *dev,
14746                       const struct rte_flow_indir_action_conf *conf,
14747                       const struct rte_flow_action *action,
14748                       struct rte_flow_error *err)
14749 {
14750         struct mlx5_priv *priv = dev->data->dev_private;
14751         uint32_t age_idx = 0;
14752         uint32_t idx = 0;
14753         uint32_t ret = 0;
14754
14755         switch (action->type) {
14756         case RTE_FLOW_ACTION_TYPE_RSS:
14757                 ret = __flow_dv_action_rss_create(dev, conf, action->conf, err);
14758                 idx = (MLX5_INDIRECT_ACTION_TYPE_RSS <<
14759                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
14760                 break;
14761         case RTE_FLOW_ACTION_TYPE_AGE:
14762                 age_idx = flow_dv_aso_age_alloc(dev, err);
14763                 if (!age_idx) {
14764                         ret = -rte_errno;
14765                         break;
14766                 }
14767                 idx = (MLX5_INDIRECT_ACTION_TYPE_AGE <<
14768                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | age_idx;
14769                 flow_dv_aso_age_params_init(dev, age_idx,
14770                                         ((const struct rte_flow_action_age *)
14771                                                 action->conf)->context ?
14772                                         ((const struct rte_flow_action_age *)
14773                                                 action->conf)->context :
14774                                         (void *)(uintptr_t)idx,
14775                                         ((const struct rte_flow_action_age *)
14776                                                 action->conf)->timeout);
14777                 ret = age_idx;
14778                 break;
14779         case RTE_FLOW_ACTION_TYPE_COUNT:
14780                 ret = flow_dv_translate_create_counter(dev, NULL, NULL, NULL);
14781                 idx = (MLX5_INDIRECT_ACTION_TYPE_COUNT <<
14782                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
14783                 break;
14784         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
14785                 ret = flow_dv_translate_create_conntrack(dev, action->conf,
14786                                                          err);
14787                 idx = MLX5_INDIRECT_ACT_CT_GEN_IDX(PORT_ID(priv), ret);
14788                 break;
14789         default:
14790                 rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
14791                                    NULL, "action type not supported");
14792                 break;
14793         }
14794         return ret ? (struct rte_flow_action_handle *)(uintptr_t)idx : NULL;
14795 }
14796
14797 /**
14798  * Destroy the indirect action.
14799  * Release action related resources on the NIC and the memory.
14800  * Lock free, (mutex should be acquired by caller).
14801  * Dispatcher for action type specific call.
14802  *
14803  * @param[in] dev
14804  *   Pointer to the Ethernet device structure.
14805  * @param[in] handle
14806  *   The indirect action object handle to be removed.
14807  * @param[out] error
14808  *   Perform verbose error reporting if not NULL. Initialized in case of
14809  *   error only.
14810  *
14811  * @return
14812  *   0 on success, otherwise negative errno value.
14813  */
14814 static int
14815 flow_dv_action_destroy(struct rte_eth_dev *dev,
14816                        struct rte_flow_action_handle *handle,
14817                        struct rte_flow_error *error)
14818 {
14819         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
14820         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
14821         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
14822         struct mlx5_flow_counter *cnt;
14823         uint32_t no_flow_refcnt = 1;
14824         int ret;
14825
14826         switch (type) {
14827         case MLX5_INDIRECT_ACTION_TYPE_RSS:
14828                 return __flow_dv_action_rss_release(dev, idx, error);
14829         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
14830                 cnt = flow_dv_counter_get_by_idx(dev, idx, NULL);
14831                 if (!__atomic_compare_exchange_n(&cnt->shared_info.refcnt,
14832                                                  &no_flow_refcnt, 1, false,
14833                                                  __ATOMIC_ACQUIRE,
14834                                                  __ATOMIC_RELAXED))
14835                         return rte_flow_error_set(error, EBUSY,
14836                                                   RTE_FLOW_ERROR_TYPE_ACTION,
14837                                                   NULL,
14838                                                   "Indirect count action has references");
14839                 flow_dv_counter_free(dev, idx);
14840                 return 0;
14841         case MLX5_INDIRECT_ACTION_TYPE_AGE:
14842                 ret = flow_dv_aso_age_release(dev, idx);
14843                 if (ret)
14844                         /*
14845                          * In this case, the last flow has a reference will
14846                          * actually release the age action.
14847                          */
14848                         DRV_LOG(DEBUG, "Indirect age action %" PRIu32 " was"
14849                                 " released with references %d.", idx, ret);
14850                 return 0;
14851         case MLX5_INDIRECT_ACTION_TYPE_CT:
14852                 ret = flow_dv_aso_ct_release(dev, idx, error);
14853                 if (ret < 0)
14854                         return ret;
14855                 if (ret > 0)
14856                         DRV_LOG(DEBUG, "Connection tracking object %u still "
14857                                 "has references %d.", idx, ret);
14858                 return 0;
14859         default:
14860                 return rte_flow_error_set(error, ENOTSUP,
14861                                           RTE_FLOW_ERROR_TYPE_ACTION,
14862                                           NULL,
14863                                           "action type not supported");
14864         }
14865 }
14866
14867 /**
14868  * Updates in place shared RSS action configuration.
14869  *
14870  * @param[in] dev
14871  *   Pointer to the Ethernet device structure.
14872  * @param[in] idx
14873  *   The shared RSS action object ID to be updated.
14874  * @param[in] action_conf
14875  *   RSS action specification used to modify *shared_rss*.
14876  * @param[out] error
14877  *   Perform verbose error reporting if not NULL. Initialized in case of
14878  *   error only.
14879  *
14880  * @return
14881  *   0 on success, otherwise negative errno value.
14882  * @note: currently only support update of RSS queues.
14883  */
14884 static int
14885 __flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx,
14886                             const struct rte_flow_action_rss *action_conf,
14887                             struct rte_flow_error *error)
14888 {
14889         struct mlx5_priv *priv = dev->data->dev_private;
14890         struct mlx5_shared_action_rss *shared_rss =
14891             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
14892         int ret = 0;
14893         void *queue = NULL;
14894         uint16_t *queue_old = NULL;
14895         uint32_t queue_size = action_conf->queue_num * sizeof(uint16_t);
14896
14897         if (!shared_rss)
14898                 return rte_flow_error_set(error, EINVAL,
14899                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14900                                           "invalid shared action to update");
14901         if (priv->obj_ops.ind_table_modify == NULL)
14902                 return rte_flow_error_set(error, ENOTSUP,
14903                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14904                                           "cannot modify indirection table");
14905         queue = mlx5_malloc(MLX5_MEM_ZERO,
14906                             RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
14907                             0, SOCKET_ID_ANY);
14908         if (!queue)
14909                 return rte_flow_error_set(error, ENOMEM,
14910                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14911                                           NULL,
14912                                           "cannot allocate resource memory");
14913         memcpy(queue, action_conf->queue, queue_size);
14914         MLX5_ASSERT(shared_rss->ind_tbl);
14915         rte_spinlock_lock(&shared_rss->action_rss_sl);
14916         queue_old = shared_rss->ind_tbl->queues;
14917         ret = mlx5_ind_table_obj_modify(dev, shared_rss->ind_tbl,
14918                                         queue, action_conf->queue_num, true);
14919         if (ret) {
14920                 mlx5_free(queue);
14921                 ret = rte_flow_error_set(error, rte_errno,
14922                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14923                                           "cannot update indirection table");
14924         } else {
14925                 mlx5_free(queue_old);
14926                 shared_rss->origin.queue = queue;
14927                 shared_rss->origin.queue_num = action_conf->queue_num;
14928         }
14929         rte_spinlock_unlock(&shared_rss->action_rss_sl);
14930         return ret;
14931 }
14932
14933 /*
14934  * Updates in place conntrack context or direction.
14935  * Context update should be synchronized.
14936  *
14937  * @param[in] dev
14938  *   Pointer to the Ethernet device structure.
14939  * @param[in] idx
14940  *   The conntrack object ID to be updated.
14941  * @param[in] update
14942  *   Pointer to the structure of information to update.
14943  * @param[out] error
14944  *   Perform verbose error reporting if not NULL. Initialized in case of
14945  *   error only.
14946  *
14947  * @return
14948  *   0 on success, otherwise negative errno value.
14949  */
14950 static int
14951 __flow_dv_action_ct_update(struct rte_eth_dev *dev, uint32_t idx,
14952                            const struct rte_flow_modify_conntrack *update,
14953                            struct rte_flow_error *error)
14954 {
14955         struct mlx5_priv *priv = dev->data->dev_private;
14956         struct mlx5_aso_ct_action *ct;
14957         const struct rte_flow_action_conntrack *new_prf;
14958         int ret = 0;
14959         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
14960         uint32_t dev_idx;
14961
14962         if (PORT_ID(priv) != owner)
14963                 return rte_flow_error_set(error, EACCES,
14964                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14965                                           NULL,
14966                                           "CT object owned by another port");
14967         dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
14968         ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
14969         if (!ct->refcnt)
14970                 return rte_flow_error_set(error, ENOMEM,
14971                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14972                                           NULL,
14973                                           "CT object is inactive");
14974         new_prf = &update->new_ct;
14975         if (update->direction)
14976                 ct->is_original = !!new_prf->is_original_dir;
14977         if (update->state) {
14978                 /* Only validate the profile when it needs to be updated. */
14979                 ret = mlx5_validate_action_ct(dev, new_prf, error);
14980                 if (ret)
14981                         return ret;
14982                 ret = mlx5_aso_ct_update_by_wqe(priv->sh, ct, new_prf);
14983                 if (ret)
14984                         return rte_flow_error_set(error, EIO,
14985                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14986                                         NULL,
14987                                         "Failed to send CT context update WQE");
14988                 /* Block until ready or a failure. */
14989                 ret = mlx5_aso_ct_available(priv->sh, ct);
14990                 if (ret)
14991                         rte_flow_error_set(error, rte_errno,
14992                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14993                                            NULL,
14994                                            "Timeout to get the CT update");
14995         }
14996         return ret;
14997 }
14998
14999 /**
15000  * Updates in place shared action configuration, lock free,
15001  * (mutex should be acquired by caller).
15002  *
15003  * @param[in] dev
15004  *   Pointer to the Ethernet device structure.
15005  * @param[in] handle
15006  *   The indirect action object handle to be updated.
15007  * @param[in] update
15008  *   Action specification used to modify the action pointed by *handle*.
15009  *   *update* could be of same type with the action pointed by the *handle*
15010  *   handle argument, or some other structures like a wrapper, depending on
15011  *   the indirect action type.
15012  * @param[out] error
15013  *   Perform verbose error reporting if not NULL. Initialized in case of
15014  *   error only.
15015  *
15016  * @return
15017  *   0 on success, otherwise negative errno value.
15018  */
15019 static int
15020 flow_dv_action_update(struct rte_eth_dev *dev,
15021                         struct rte_flow_action_handle *handle,
15022                         const void *update,
15023                         struct rte_flow_error *err)
15024 {
15025         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
15026         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
15027         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
15028         const void *action_conf;
15029
15030         switch (type) {
15031         case MLX5_INDIRECT_ACTION_TYPE_RSS:
15032                 action_conf = ((const struct rte_flow_action *)update)->conf;
15033                 return __flow_dv_action_rss_update(dev, idx, action_conf, err);
15034         case MLX5_INDIRECT_ACTION_TYPE_CT:
15035                 return __flow_dv_action_ct_update(dev, idx, update, err);
15036         default:
15037                 return rte_flow_error_set(err, ENOTSUP,
15038                                           RTE_FLOW_ERROR_TYPE_ACTION,
15039                                           NULL,
15040                                           "action type update not supported");
15041         }
15042 }
15043
15044 /**
15045  * Destroy the meter sub policy table rules.
15046  * Lock free, (mutex should be acquired by caller).
15047  *
15048  * @param[in] dev
15049  *   Pointer to Ethernet device.
15050  * @param[in] sub_policy
15051  *   Pointer to meter sub policy table.
15052  */
15053 static void
15054 __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
15055                              struct mlx5_flow_meter_sub_policy *sub_policy)
15056 {
15057         struct mlx5_priv *priv = dev->data->dev_private;
15058         struct mlx5_flow_tbl_data_entry *tbl;
15059         struct mlx5_flow_meter_policy *policy = sub_policy->main_policy;
15060         struct mlx5_flow_meter_info *next_fm;
15061         struct mlx5_sub_policy_color_rule *color_rule;
15062         void *tmp;
15063         uint32_t i;
15064
15065         for (i = 0; i < RTE_COLORS; i++) {
15066                 next_fm = NULL;
15067                 if (i == RTE_COLOR_GREEN && policy &&
15068                     policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR)
15069                         next_fm = mlx5_flow_meter_find(priv,
15070                                         policy->act_cnt[i].next_mtr_id, NULL);
15071                 RTE_TAILQ_FOREACH_SAFE(color_rule, &sub_policy->color_rules[i],
15072                                    next_port, tmp) {
15073                         claim_zero(mlx5_flow_os_destroy_flow(color_rule->rule));
15074                         tbl = container_of(color_rule->matcher->tbl,
15075                                            typeof(*tbl), tbl);
15076                         mlx5_list_unregister(tbl->matchers,
15077                                              &color_rule->matcher->entry);
15078                         TAILQ_REMOVE(&sub_policy->color_rules[i],
15079                                      color_rule, next_port);
15080                         mlx5_free(color_rule);
15081                         if (next_fm)
15082                                 mlx5_flow_meter_detach(priv, next_fm);
15083                 }
15084         }
15085         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
15086                 if (sub_policy->rix_hrxq[i]) {
15087                         if (policy && !policy->is_hierarchy)
15088                                 mlx5_hrxq_release(dev, sub_policy->rix_hrxq[i]);
15089                         sub_policy->rix_hrxq[i] = 0;
15090                 }
15091                 if (sub_policy->jump_tbl[i]) {
15092                         flow_dv_tbl_resource_release(MLX5_SH(dev),
15093                                                      sub_policy->jump_tbl[i]);
15094                         sub_policy->jump_tbl[i] = NULL;
15095                 }
15096         }
15097         if (sub_policy->tbl_rsc) {
15098                 flow_dv_tbl_resource_release(MLX5_SH(dev),
15099                                              sub_policy->tbl_rsc);
15100                 sub_policy->tbl_rsc = NULL;
15101         }
15102 }
15103
15104 /**
15105  * Destroy policy rules, lock free,
15106  * (mutex should be acquired by caller).
15107  * Dispatcher for action type specific call.
15108  *
15109  * @param[in] dev
15110  *   Pointer to the Ethernet device structure.
15111  * @param[in] mtr_policy
15112  *   Meter policy struct.
15113  */
15114 static void
15115 flow_dv_destroy_policy_rules(struct rte_eth_dev *dev,
15116                              struct mlx5_flow_meter_policy *mtr_policy)
15117 {
15118         uint32_t i, j;
15119         struct mlx5_flow_meter_sub_policy *sub_policy;
15120         uint16_t sub_policy_num;
15121
15122         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15123                 sub_policy_num = (mtr_policy->sub_policy_num >>
15124                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
15125                         MLX5_MTR_SUB_POLICY_NUM_MASK;
15126                 for (j = 0; j < sub_policy_num; j++) {
15127                         sub_policy = mtr_policy->sub_policys[i][j];
15128                         if (sub_policy)
15129                                 __flow_dv_destroy_sub_policy_rules(dev,
15130                                                                    sub_policy);
15131                 }
15132         }
15133 }
15134
15135 /**
15136  * Destroy policy action, lock free,
15137  * (mutex should be acquired by caller).
15138  * Dispatcher for action type specific call.
15139  *
15140  * @param[in] dev
15141  *   Pointer to the Ethernet device structure.
15142  * @param[in] mtr_policy
15143  *   Meter policy struct.
15144  */
15145 static void
15146 flow_dv_destroy_mtr_policy_acts(struct rte_eth_dev *dev,
15147                       struct mlx5_flow_meter_policy *mtr_policy)
15148 {
15149         struct rte_flow_action *rss_action;
15150         struct mlx5_flow_handle dev_handle;
15151         uint32_t i, j;
15152
15153         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
15154                 if (mtr_policy->act_cnt[i].rix_mark) {
15155                         flow_dv_tag_release(dev,
15156                                 mtr_policy->act_cnt[i].rix_mark);
15157                         mtr_policy->act_cnt[i].rix_mark = 0;
15158                 }
15159                 if (mtr_policy->act_cnt[i].modify_hdr) {
15160                         dev_handle.dvh.modify_hdr =
15161                                 mtr_policy->act_cnt[i].modify_hdr;
15162                         flow_dv_modify_hdr_resource_release(dev, &dev_handle);
15163                 }
15164                 switch (mtr_policy->act_cnt[i].fate_action) {
15165                 case MLX5_FLOW_FATE_SHARED_RSS:
15166                         rss_action = mtr_policy->act_cnt[i].rss;
15167                         mlx5_free(rss_action);
15168                         break;
15169                 case MLX5_FLOW_FATE_PORT_ID:
15170                         if (mtr_policy->act_cnt[i].rix_port_id_action) {
15171                                 flow_dv_port_id_action_resource_release(dev,
15172                                 mtr_policy->act_cnt[i].rix_port_id_action);
15173                                 mtr_policy->act_cnt[i].rix_port_id_action = 0;
15174                         }
15175                         break;
15176                 case MLX5_FLOW_FATE_DROP:
15177                 case MLX5_FLOW_FATE_JUMP:
15178                         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
15179                                 mtr_policy->act_cnt[i].dr_jump_action[j] =
15180                                                 NULL;
15181                         break;
15182                 default:
15183                         /*Queue action do nothing*/
15184                         break;
15185                 }
15186         }
15187         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
15188                 mtr_policy->dr_drop_action[j] = NULL;
15189 }
15190
15191 /**
15192  * Create policy action per domain, lock free,
15193  * (mutex should be acquired by caller).
15194  * Dispatcher for action type specific call.
15195  *
15196  * @param[in] dev
15197  *   Pointer to the Ethernet device structure.
15198  * @param[in] mtr_policy
15199  *   Meter policy struct.
15200  * @param[in] action
15201  *   Action specification used to create meter actions.
15202  * @param[out] error
15203  *   Perform verbose error reporting if not NULL. Initialized in case of
15204  *   error only.
15205  *
15206  * @return
15207  *   0 on success, otherwise negative errno value.
15208  */
15209 static int
15210 __flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev,
15211                         struct mlx5_flow_meter_policy *mtr_policy,
15212                         const struct rte_flow_action *actions[RTE_COLORS],
15213                         enum mlx5_meter_domain domain,
15214                         struct rte_mtr_error *error)
15215 {
15216         struct mlx5_priv *priv = dev->data->dev_private;
15217         struct rte_flow_error flow_err;
15218         const struct rte_flow_action *act;
15219         uint64_t action_flags;
15220         struct mlx5_flow_handle dh;
15221         struct mlx5_flow dev_flow;
15222         struct mlx5_flow_dv_port_id_action_resource port_id_action;
15223         int i, ret;
15224         uint8_t egress, transfer;
15225         struct mlx5_meter_policy_action_container *act_cnt = NULL;
15226         union {
15227                 struct mlx5_flow_dv_modify_hdr_resource res;
15228                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
15229                             sizeof(struct mlx5_modification_cmd) *
15230                             (MLX5_MAX_MODIFY_NUM + 1)];
15231         } mhdr_dummy;
15232         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
15233
15234         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
15235         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
15236         memset(&dh, 0, sizeof(struct mlx5_flow_handle));
15237         memset(&dev_flow, 0, sizeof(struct mlx5_flow));
15238         memset(&port_id_action, 0,
15239                sizeof(struct mlx5_flow_dv_port_id_action_resource));
15240         memset(mhdr_res, 0, sizeof(*mhdr_res));
15241         mhdr_res->ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
15242                                        (egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
15243                                         MLX5DV_FLOW_TABLE_TYPE_NIC_RX);
15244         dev_flow.handle = &dh;
15245         dev_flow.dv.port_id_action = &port_id_action;
15246         dev_flow.external = true;
15247         for (i = 0; i < RTE_COLORS; i++) {
15248                 if (i < MLX5_MTR_RTE_COLORS)
15249                         act_cnt = &mtr_policy->act_cnt[i];
15250                 /* Skip the color policy actions creation. */
15251                 if ((i == RTE_COLOR_YELLOW && mtr_policy->skip_y) ||
15252                     (i == RTE_COLOR_GREEN && mtr_policy->skip_g))
15253                         continue;
15254                 action_flags = 0;
15255                 for (act = actions[i];
15256                      act && act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
15257                         switch (act->type) {
15258                         case RTE_FLOW_ACTION_TYPE_MARK:
15259                         {
15260                                 uint32_t tag_be = mlx5_flow_mark_set
15261                                         (((const struct rte_flow_action_mark *)
15262                                         (act->conf))->id);
15263
15264                                 if (i >= MLX5_MTR_RTE_COLORS)
15265                                         return -rte_mtr_error_set(error,
15266                                           ENOTSUP,
15267                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15268                                           NULL,
15269                                           "cannot create policy "
15270                                           "mark action for this color");
15271                                 dev_flow.handle->mark = 1;
15272                                 if (flow_dv_tag_resource_register(dev, tag_be,
15273                                                   &dev_flow, &flow_err))
15274                                         return -rte_mtr_error_set(error,
15275                                         ENOTSUP,
15276                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15277                                         NULL,
15278                                         "cannot setup policy mark action");
15279                                 MLX5_ASSERT(dev_flow.dv.tag_resource);
15280                                 act_cnt->rix_mark =
15281                                         dev_flow.handle->dvh.rix_tag;
15282                                 action_flags |= MLX5_FLOW_ACTION_MARK;
15283                                 break;
15284                         }
15285                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
15286                                 if (i >= MLX5_MTR_RTE_COLORS)
15287                                         return -rte_mtr_error_set(error,
15288                                           ENOTSUP,
15289                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15290                                           NULL,
15291                                           "cannot create policy "
15292                                           "set tag action for this color");
15293                                 if (flow_dv_convert_action_set_tag
15294                                 (dev, mhdr_res,
15295                                 (const struct rte_flow_action_set_tag *)
15296                                 act->conf,  &flow_err))
15297                                         return -rte_mtr_error_set(error,
15298                                         ENOTSUP,
15299                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15300                                         NULL, "cannot convert policy "
15301                                         "set tag action");
15302                                 if (!mhdr_res->actions_num)
15303                                         return -rte_mtr_error_set(error,
15304                                         ENOTSUP,
15305                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15306                                         NULL, "cannot find policy "
15307                                         "set tag action");
15308                                 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
15309                                 break;
15310                         case RTE_FLOW_ACTION_TYPE_DROP:
15311                         {
15312                                 struct mlx5_flow_mtr_mng *mtrmng =
15313                                                 priv->sh->mtrmng;
15314                                 struct mlx5_flow_tbl_data_entry *tbl_data;
15315
15316                                 /*
15317                                  * Create the drop table with
15318                                  * METER DROP level.
15319                                  */
15320                                 if (!mtrmng->drop_tbl[domain]) {
15321                                         mtrmng->drop_tbl[domain] =
15322                                         flow_dv_tbl_resource_get(dev,
15323                                         MLX5_FLOW_TABLE_LEVEL_METER,
15324                                         egress, transfer, false, NULL, 0,
15325                                         0, MLX5_MTR_TABLE_ID_DROP, &flow_err);
15326                                         if (!mtrmng->drop_tbl[domain])
15327                                                 return -rte_mtr_error_set
15328                                         (error, ENOTSUP,
15329                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15330                                         NULL,
15331                                         "Failed to create meter drop table");
15332                                 }
15333                                 tbl_data = container_of
15334                                 (mtrmng->drop_tbl[domain],
15335                                 struct mlx5_flow_tbl_data_entry, tbl);
15336                                 if (i < MLX5_MTR_RTE_COLORS) {
15337                                         act_cnt->dr_jump_action[domain] =
15338                                                 tbl_data->jump.action;
15339                                         act_cnt->fate_action =
15340                                                 MLX5_FLOW_FATE_DROP;
15341                                 }
15342                                 if (i == RTE_COLOR_RED)
15343                                         mtr_policy->dr_drop_action[domain] =
15344                                                 tbl_data->jump.action;
15345                                 action_flags |= MLX5_FLOW_ACTION_DROP;
15346                                 break;
15347                         }
15348                         case RTE_FLOW_ACTION_TYPE_QUEUE:
15349                         {
15350                                 if (i >= MLX5_MTR_RTE_COLORS)
15351                                         return -rte_mtr_error_set(error,
15352                                         ENOTSUP,
15353                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15354                                         NULL, "cannot create policy "
15355                                         "fate queue for this color");
15356                                 act_cnt->queue =
15357                                 ((const struct rte_flow_action_queue *)
15358                                         (act->conf))->index;
15359                                 act_cnt->fate_action =
15360                                         MLX5_FLOW_FATE_QUEUE;
15361                                 dev_flow.handle->fate_action =
15362                                         MLX5_FLOW_FATE_QUEUE;
15363                                 mtr_policy->is_queue = 1;
15364                                 action_flags |= MLX5_FLOW_ACTION_QUEUE;
15365                                 break;
15366                         }
15367                         case RTE_FLOW_ACTION_TYPE_RSS:
15368                         {
15369                                 int rss_size;
15370
15371                                 if (i >= MLX5_MTR_RTE_COLORS)
15372                                         return -rte_mtr_error_set(error,
15373                                           ENOTSUP,
15374                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15375                                           NULL,
15376                                           "cannot create policy "
15377                                           "rss action for this color");
15378                                 /*
15379                                  * Save RSS conf into policy struct
15380                                  * for translate stage.
15381                                  */
15382                                 rss_size = (int)rte_flow_conv
15383                                         (RTE_FLOW_CONV_OP_ACTION,
15384                                         NULL, 0, act, &flow_err);
15385                                 if (rss_size <= 0)
15386                                         return -rte_mtr_error_set(error,
15387                                           ENOTSUP,
15388                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15389                                           NULL, "Get the wrong "
15390                                           "rss action struct size");
15391                                 act_cnt->rss = mlx5_malloc(MLX5_MEM_ZERO,
15392                                                 rss_size, 0, SOCKET_ID_ANY);
15393                                 if (!act_cnt->rss)
15394                                         return -rte_mtr_error_set(error,
15395                                           ENOTSUP,
15396                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15397                                           NULL,
15398                                           "Fail to malloc rss action memory");
15399                                 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTION,
15400                                         act_cnt->rss, rss_size,
15401                                         act, &flow_err);
15402                                 if (ret < 0)
15403                                         return -rte_mtr_error_set(error,
15404                                           ENOTSUP,
15405                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15406                                           NULL, "Fail to save "
15407                                           "rss action into policy struct");
15408                                 act_cnt->fate_action =
15409                                         MLX5_FLOW_FATE_SHARED_RSS;
15410                                 action_flags |= MLX5_FLOW_ACTION_RSS;
15411                                 break;
15412                         }
15413                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
15414                         case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
15415                         {
15416                                 struct mlx5_flow_dv_port_id_action_resource
15417                                         port_id_resource;
15418                                 uint32_t port_id = 0;
15419
15420                                 if (i >= MLX5_MTR_RTE_COLORS)
15421                                         return -rte_mtr_error_set(error,
15422                                         ENOTSUP,
15423                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15424                                         NULL, "cannot create policy "
15425                                         "port action for this color");
15426                                 memset(&port_id_resource, 0,
15427                                         sizeof(port_id_resource));
15428                                 if (flow_dv_translate_action_port_id(dev, act,
15429                                                 &port_id, &flow_err))
15430                                         return -rte_mtr_error_set(error,
15431                                         ENOTSUP,
15432                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15433                                         NULL, "cannot translate "
15434                                         "policy port action");
15435                                 port_id_resource.port_id = port_id;
15436                                 if (flow_dv_port_id_action_resource_register
15437                                         (dev, &port_id_resource,
15438                                         &dev_flow, &flow_err))
15439                                         return -rte_mtr_error_set(error,
15440                                         ENOTSUP,
15441                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15442                                         NULL, "cannot setup "
15443                                         "policy port action");
15444                                 act_cnt->rix_port_id_action =
15445                                         dev_flow.handle->rix_port_id_action;
15446                                 act_cnt->fate_action =
15447                                         MLX5_FLOW_FATE_PORT_ID;
15448                                 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
15449                                 break;
15450                         }
15451                         case RTE_FLOW_ACTION_TYPE_JUMP:
15452                         {
15453                                 uint32_t jump_group = 0;
15454                                 uint32_t table = 0;
15455                                 struct mlx5_flow_tbl_data_entry *tbl_data;
15456                                 struct flow_grp_info grp_info = {
15457                                         .external = !!dev_flow.external,
15458                                         .transfer = !!transfer,
15459                                         .fdb_def_rule = !!priv->fdb_def_rule,
15460                                         .std_tbl_fix = 0,
15461                                         .skip_scale = dev_flow.skip_scale &
15462                                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
15463                                 };
15464                                 struct mlx5_flow_meter_sub_policy *sub_policy =
15465                                         mtr_policy->sub_policys[domain][0];
15466
15467                                 if (i >= MLX5_MTR_RTE_COLORS)
15468                                         return -rte_mtr_error_set(error,
15469                                           ENOTSUP,
15470                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15471                                           NULL,
15472                                           "cannot create policy "
15473                                           "jump action for this color");
15474                                 jump_group =
15475                                 ((const struct rte_flow_action_jump *)
15476                                                         act->conf)->group;
15477                                 if (mlx5_flow_group_to_table(dev, NULL,
15478                                                        jump_group,
15479                                                        &table,
15480                                                        &grp_info, &flow_err))
15481                                         return -rte_mtr_error_set(error,
15482                                         ENOTSUP,
15483                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15484                                         NULL, "cannot setup "
15485                                         "policy jump action");
15486                                 sub_policy->jump_tbl[i] =
15487                                 flow_dv_tbl_resource_get(dev,
15488                                         table, egress,
15489                                         transfer,
15490                                         !!dev_flow.external,
15491                                         NULL, jump_group, 0,
15492                                         0, &flow_err);
15493                                 if
15494                                 (!sub_policy->jump_tbl[i])
15495                                         return  -rte_mtr_error_set(error,
15496                                         ENOTSUP,
15497                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15498                                         NULL, "cannot create jump action.");
15499                                 tbl_data = container_of
15500                                 (sub_policy->jump_tbl[i],
15501                                 struct mlx5_flow_tbl_data_entry, tbl);
15502                                 act_cnt->dr_jump_action[domain] =
15503                                         tbl_data->jump.action;
15504                                 act_cnt->fate_action =
15505                                         MLX5_FLOW_FATE_JUMP;
15506                                 action_flags |= MLX5_FLOW_ACTION_JUMP;
15507                                 break;
15508                         }
15509                         /*
15510                          * No need to check meter hierarchy for Y or R colors
15511                          * here since it is done in the validation stage.
15512                          */
15513                         case RTE_FLOW_ACTION_TYPE_METER:
15514                         {
15515                                 const struct rte_flow_action_meter *mtr;
15516                                 struct mlx5_flow_meter_info *next_fm;
15517                                 struct mlx5_flow_meter_policy *next_policy;
15518                                 struct rte_flow_action tag_action;
15519                                 struct mlx5_rte_flow_action_set_tag set_tag;
15520                                 uint32_t next_mtr_idx = 0;
15521
15522                                 mtr = act->conf;
15523                                 next_fm = mlx5_flow_meter_find(priv,
15524                                                         mtr->mtr_id,
15525                                                         &next_mtr_idx);
15526                                 if (!next_fm)
15527                                         return -rte_mtr_error_set(error, EINVAL,
15528                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
15529                                                 "Fail to find next meter.");
15530                                 if (next_fm->def_policy)
15531                                         return -rte_mtr_error_set(error, EINVAL,
15532                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
15533                                 "Hierarchy only supports termination meter.");
15534                                 next_policy = mlx5_flow_meter_policy_find(dev,
15535                                                 next_fm->policy_id, NULL);
15536                                 MLX5_ASSERT(next_policy);
15537                                 if (next_fm->drop_cnt) {
15538                                         set_tag.id =
15539                                                 (enum modify_reg)
15540                                                 mlx5_flow_get_reg_id(dev,
15541                                                 MLX5_MTR_ID,
15542                                                 0,
15543                                                 (struct rte_flow_error *)error);
15544                                         set_tag.offset = (priv->mtr_reg_share ?
15545                                                 MLX5_MTR_COLOR_BITS : 0);
15546                                         set_tag.length = (priv->mtr_reg_share ?
15547                                                MLX5_MTR_IDLE_BITS_IN_COLOR_REG :
15548                                                MLX5_REG_BITS);
15549                                         set_tag.data = next_mtr_idx;
15550                                         tag_action.type =
15551                                                 (enum rte_flow_action_type)
15552                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
15553                                         tag_action.conf = &set_tag;
15554                                         if (flow_dv_convert_action_set_reg
15555                                                 (mhdr_res, &tag_action,
15556                                                 (struct rte_flow_error *)error))
15557                                                 return -rte_errno;
15558                                         action_flags |=
15559                                                 MLX5_FLOW_ACTION_SET_TAG;
15560                                 }
15561                                 act_cnt->fate_action = MLX5_FLOW_FATE_MTR;
15562                                 act_cnt->next_mtr_id = next_fm->meter_id;
15563                                 act_cnt->next_sub_policy = NULL;
15564                                 mtr_policy->is_hierarchy = 1;
15565                                 mtr_policy->dev = next_policy->dev;
15566                                 action_flags |=
15567                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
15568                                 break;
15569                         }
15570                         default:
15571                                 return -rte_mtr_error_set(error, ENOTSUP,
15572                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15573                                           NULL, "action type not supported");
15574                         }
15575                         if (action_flags & MLX5_FLOW_ACTION_SET_TAG) {
15576                                 /* create modify action if needed. */
15577                                 dev_flow.dv.group = 1;
15578                                 if (flow_dv_modify_hdr_resource_register
15579                                         (dev, mhdr_res, &dev_flow, &flow_err))
15580                                         return -rte_mtr_error_set(error,
15581                                                 ENOTSUP,
15582                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
15583                                                 NULL, "cannot register policy "
15584                                                 "set tag action");
15585                                 act_cnt->modify_hdr =
15586                                         dev_flow.handle->dvh.modify_hdr;
15587                         }
15588                 }
15589         }
15590         return 0;
15591 }
15592
15593 /**
15594  * Create policy action per domain, lock free,
15595  * (mutex should be acquired by caller).
15596  * Dispatcher for action type specific call.
15597  *
15598  * @param[in] dev
15599  *   Pointer to the Ethernet device structure.
15600  * @param[in] mtr_policy
15601  *   Meter policy struct.
15602  * @param[in] action
15603  *   Action specification used to create meter actions.
15604  * @param[out] error
15605  *   Perform verbose error reporting if not NULL. Initialized in case of
15606  *   error only.
15607  *
15608  * @return
15609  *   0 on success, otherwise negative errno value.
15610  */
15611 static int
15612 flow_dv_create_mtr_policy_acts(struct rte_eth_dev *dev,
15613                       struct mlx5_flow_meter_policy *mtr_policy,
15614                       const struct rte_flow_action *actions[RTE_COLORS],
15615                       struct rte_mtr_error *error)
15616 {
15617         int ret, i;
15618         uint16_t sub_policy_num;
15619
15620         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15621                 sub_policy_num = (mtr_policy->sub_policy_num >>
15622                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
15623                         MLX5_MTR_SUB_POLICY_NUM_MASK;
15624                 if (sub_policy_num) {
15625                         ret = __flow_dv_create_domain_policy_acts(dev,
15626                                 mtr_policy, actions,
15627                                 (enum mlx5_meter_domain)i, error);
15628                         /* Cleaning resource is done in the caller level. */
15629                         if (ret)
15630                                 return ret;
15631                 }
15632         }
15633         return 0;
15634 }
15635
15636 /**
15637  * Query a DV flow rule for its statistics via DevX.
15638  *
15639  * @param[in] dev
15640  *   Pointer to Ethernet device.
15641  * @param[in] cnt_idx
15642  *   Index to the flow counter.
15643  * @param[out] data
15644  *   Data retrieved by the query.
15645  * @param[out] error
15646  *   Perform verbose error reporting if not NULL.
15647  *
15648  * @return
15649  *   0 on success, a negative errno value otherwise and rte_errno is set.
15650  */
15651 static int
15652 flow_dv_query_count(struct rte_eth_dev *dev, uint32_t cnt_idx, void *data,
15653                     struct rte_flow_error *error)
15654 {
15655         struct mlx5_priv *priv = dev->data->dev_private;
15656         struct rte_flow_query_count *qc = data;
15657
15658         if (!priv->sh->devx)
15659                 return rte_flow_error_set(error, ENOTSUP,
15660                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15661                                           NULL,
15662                                           "counters are not supported");
15663         if (cnt_idx) {
15664                 uint64_t pkts, bytes;
15665                 struct mlx5_flow_counter *cnt;
15666                 int err = _flow_dv_query_count(dev, cnt_idx, &pkts, &bytes);
15667
15668                 if (err)
15669                         return rte_flow_error_set(error, -err,
15670                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15671                                         NULL, "cannot read counters");
15672                 cnt = flow_dv_counter_get_by_idx(dev, cnt_idx, NULL);
15673                 qc->hits_set = 1;
15674                 qc->bytes_set = 1;
15675                 qc->hits = pkts - cnt->hits;
15676                 qc->bytes = bytes - cnt->bytes;
15677                 if (qc->reset) {
15678                         cnt->hits = pkts;
15679                         cnt->bytes = bytes;
15680                 }
15681                 return 0;
15682         }
15683         return rte_flow_error_set(error, EINVAL,
15684                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15685                                   NULL,
15686                                   "counters are not available");
15687 }
15688
15689 static int
15690 flow_dv_action_query(struct rte_eth_dev *dev,
15691                      const struct rte_flow_action_handle *handle, void *data,
15692                      struct rte_flow_error *error)
15693 {
15694         struct mlx5_age_param *age_param;
15695         struct rte_flow_query_age *resp;
15696         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
15697         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
15698         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
15699         struct mlx5_priv *priv = dev->data->dev_private;
15700         struct mlx5_aso_ct_action *ct;
15701         uint16_t owner;
15702         uint32_t dev_idx;
15703
15704         switch (type) {
15705         case MLX5_INDIRECT_ACTION_TYPE_AGE:
15706                 age_param = &flow_aso_age_get_by_idx(dev, idx)->age_params;
15707                 resp = data;
15708                 resp->aged = __atomic_load_n(&age_param->state,
15709                                               __ATOMIC_RELAXED) == AGE_TMOUT ?
15710                                                                           1 : 0;
15711                 resp->sec_since_last_hit_valid = !resp->aged;
15712                 if (resp->sec_since_last_hit_valid)
15713                         resp->sec_since_last_hit = __atomic_load_n
15714                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
15715                 return 0;
15716         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
15717                 return flow_dv_query_count(dev, idx, data, error);
15718         case MLX5_INDIRECT_ACTION_TYPE_CT:
15719                 owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
15720                 if (owner != PORT_ID(priv))
15721                         return rte_flow_error_set(error, EACCES,
15722                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15723                                         NULL,
15724                                         "CT object owned by another port");
15725                 dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
15726                 ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
15727                 MLX5_ASSERT(ct);
15728                 if (!ct->refcnt)
15729                         return rte_flow_error_set(error, EFAULT,
15730                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15731                                         NULL,
15732                                         "CT object is inactive");
15733                 ((struct rte_flow_action_conntrack *)data)->peer_port =
15734                                                         ct->peer;
15735                 ((struct rte_flow_action_conntrack *)data)->is_original_dir =
15736                                                         ct->is_original;
15737                 if (mlx5_aso_ct_query_by_wqe(priv->sh, ct, data))
15738                         return rte_flow_error_set(error, EIO,
15739                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15740                                         NULL,
15741                                         "Failed to query CT context");
15742                 return 0;
15743         default:
15744                 return rte_flow_error_set(error, ENOTSUP,
15745                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
15746                                           "action type query not supported");
15747         }
15748 }
15749
15750 /**
15751  * Query a flow rule AGE action for aging information.
15752  *
15753  * @param[in] dev
15754  *   Pointer to Ethernet device.
15755  * @param[in] flow
15756  *   Pointer to the sub flow.
15757  * @param[out] data
15758  *   data retrieved by the query.
15759  * @param[out] error
15760  *   Perform verbose error reporting if not NULL.
15761  *
15762  * @return
15763  *   0 on success, a negative errno value otherwise and rte_errno is set.
15764  */
15765 static int
15766 flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow,
15767                   void *data, struct rte_flow_error *error)
15768 {
15769         struct rte_flow_query_age *resp = data;
15770         struct mlx5_age_param *age_param;
15771
15772         if (flow->age) {
15773                 struct mlx5_aso_age_action *act =
15774                                      flow_aso_age_get_by_idx(dev, flow->age);
15775
15776                 age_param = &act->age_params;
15777         } else if (flow->counter) {
15778                 age_param = flow_dv_counter_idx_get_age(dev, flow->counter);
15779
15780                 if (!age_param || !age_param->timeout)
15781                         return rte_flow_error_set
15782                                         (error, EINVAL,
15783                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15784                                          NULL, "cannot read age data");
15785         } else {
15786                 return rte_flow_error_set(error, EINVAL,
15787                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15788                                           NULL, "age data not available");
15789         }
15790         resp->aged = __atomic_load_n(&age_param->state, __ATOMIC_RELAXED) ==
15791                                      AGE_TMOUT ? 1 : 0;
15792         resp->sec_since_last_hit_valid = !resp->aged;
15793         if (resp->sec_since_last_hit_valid)
15794                 resp->sec_since_last_hit = __atomic_load_n
15795                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
15796         return 0;
15797 }
15798
15799 /**
15800  * Query a flow.
15801  *
15802  * @see rte_flow_query()
15803  * @see rte_flow_ops
15804  */
15805 static int
15806 flow_dv_query(struct rte_eth_dev *dev,
15807               struct rte_flow *flow __rte_unused,
15808               const struct rte_flow_action *actions __rte_unused,
15809               void *data __rte_unused,
15810               struct rte_flow_error *error __rte_unused)
15811 {
15812         int ret = -EINVAL;
15813
15814         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
15815                 switch (actions->type) {
15816                 case RTE_FLOW_ACTION_TYPE_VOID:
15817                         break;
15818                 case RTE_FLOW_ACTION_TYPE_COUNT:
15819                         ret = flow_dv_query_count(dev, flow->counter, data,
15820                                                   error);
15821                         break;
15822                 case RTE_FLOW_ACTION_TYPE_AGE:
15823                         ret = flow_dv_query_age(dev, flow, data, error);
15824                         break;
15825                 default:
15826                         return rte_flow_error_set(error, ENOTSUP,
15827                                                   RTE_FLOW_ERROR_TYPE_ACTION,
15828                                                   actions,
15829                                                   "action not supported");
15830                 }
15831         }
15832         return ret;
15833 }
15834
15835 /**
15836  * Destroy the meter table set.
15837  * Lock free, (mutex should be acquired by caller).
15838  *
15839  * @param[in] dev
15840  *   Pointer to Ethernet device.
15841  * @param[in] fm
15842  *   Meter information table.
15843  */
15844 static void
15845 flow_dv_destroy_mtr_tbls(struct rte_eth_dev *dev,
15846                         struct mlx5_flow_meter_info *fm)
15847 {
15848         struct mlx5_priv *priv = dev->data->dev_private;
15849         int i;
15850
15851         if (!fm || !priv->config.dv_flow_en)
15852                 return;
15853         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15854                 if (fm->drop_rule[i]) {
15855                         claim_zero(mlx5_flow_os_destroy_flow(fm->drop_rule[i]));
15856                         fm->drop_rule[i] = NULL;
15857                 }
15858         }
15859 }
15860
15861 static void
15862 flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
15863 {
15864         struct mlx5_priv *priv = dev->data->dev_private;
15865         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
15866         struct mlx5_flow_tbl_data_entry *tbl;
15867         int i, j;
15868
15869         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15870                 if (mtrmng->def_rule[i]) {
15871                         claim_zero(mlx5_flow_os_destroy_flow
15872                                         (mtrmng->def_rule[i]));
15873                         mtrmng->def_rule[i] = NULL;
15874                 }
15875                 if (mtrmng->def_matcher[i]) {
15876                         tbl = container_of(mtrmng->def_matcher[i]->tbl,
15877                                 struct mlx5_flow_tbl_data_entry, tbl);
15878                         mlx5_list_unregister(tbl->matchers,
15879                                              &mtrmng->def_matcher[i]->entry);
15880                         mtrmng->def_matcher[i] = NULL;
15881                 }
15882                 for (j = 0; j < MLX5_REG_BITS; j++) {
15883                         if (mtrmng->drop_matcher[i][j]) {
15884                                 tbl =
15885                                 container_of(mtrmng->drop_matcher[i][j]->tbl,
15886                                              struct mlx5_flow_tbl_data_entry,
15887                                              tbl);
15888                                 mlx5_list_unregister(tbl->matchers,
15889                                             &mtrmng->drop_matcher[i][j]->entry);
15890                                 mtrmng->drop_matcher[i][j] = NULL;
15891                         }
15892                 }
15893                 if (mtrmng->drop_tbl[i]) {
15894                         flow_dv_tbl_resource_release(MLX5_SH(dev),
15895                                 mtrmng->drop_tbl[i]);
15896                         mtrmng->drop_tbl[i] = NULL;
15897                 }
15898         }
15899 }
15900
15901 /* Number of meter flow actions, count and jump or count and drop. */
15902 #define METER_ACTIONS 2
15903
15904 static void
15905 __flow_dv_destroy_domain_def_policy(struct rte_eth_dev *dev,
15906                                     enum mlx5_meter_domain domain)
15907 {
15908         struct mlx5_priv *priv = dev->data->dev_private;
15909         struct mlx5_flow_meter_def_policy *def_policy =
15910                         priv->sh->mtrmng->def_policy[domain];
15911
15912         __flow_dv_destroy_sub_policy_rules(dev, &def_policy->sub_policy);
15913         mlx5_free(def_policy);
15914         priv->sh->mtrmng->def_policy[domain] = NULL;
15915 }
15916
15917 /**
15918  * Destroy the default policy table set.
15919  *
15920  * @param[in] dev
15921  *   Pointer to Ethernet device.
15922  */
15923 static void
15924 flow_dv_destroy_def_policy(struct rte_eth_dev *dev)
15925 {
15926         struct mlx5_priv *priv = dev->data->dev_private;
15927         int i;
15928
15929         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++)
15930                 if (priv->sh->mtrmng->def_policy[i])
15931                         __flow_dv_destroy_domain_def_policy(dev,
15932                                         (enum mlx5_meter_domain)i);
15933         priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
15934 }
15935
15936 static int
15937 __flow_dv_create_policy_flow(struct rte_eth_dev *dev,
15938                         uint32_t color_reg_c_idx,
15939                         enum rte_color color, void *matcher_object,
15940                         int actions_n, void *actions,
15941                         bool match_src_port, const struct rte_flow_item *item,
15942                         void **rule, const struct rte_flow_attr *attr)
15943 {
15944         int ret;
15945         struct mlx5_flow_dv_match_params value = {
15946                 .size = sizeof(value.buf),
15947         };
15948         struct mlx5_flow_dv_match_params matcher = {
15949                 .size = sizeof(matcher.buf),
15950         };
15951         struct mlx5_priv *priv = dev->data->dev_private;
15952         uint8_t misc_mask;
15953
15954         if (match_src_port && (priv->representor || priv->master)) {
15955                 if (flow_dv_translate_item_port_id(dev, matcher.buf,
15956                                                    value.buf, item, attr)) {
15957                         DRV_LOG(ERR, "Failed to create meter policy%d flow's"
15958                                 " value with port.", color);
15959                         return -1;
15960                 }
15961         }
15962         flow_dv_match_meta_reg(matcher.buf, value.buf,
15963                                (enum modify_reg)color_reg_c_idx,
15964                                rte_col_2_mlx5_col(color), UINT32_MAX);
15965         misc_mask = flow_dv_matcher_enable(value.buf);
15966         __flow_dv_adjust_buf_size(&value.size, misc_mask);
15967         ret = mlx5_flow_os_create_flow(matcher_object, (void *)&value,
15968                                        actions_n, actions, rule);
15969         if (ret) {
15970                 DRV_LOG(ERR, "Failed to create meter policy%d flow.", color);
15971                 return -1;
15972         }
15973         return 0;
15974 }
15975
15976 static int
15977 __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
15978                         uint32_t color_reg_c_idx,
15979                         uint16_t priority,
15980                         struct mlx5_flow_meter_sub_policy *sub_policy,
15981                         const struct rte_flow_attr *attr,
15982                         bool match_src_port,
15983                         const struct rte_flow_item *item,
15984                         struct mlx5_flow_dv_matcher **policy_matcher,
15985                         struct rte_flow_error *error)
15986 {
15987         struct mlx5_list_entry *entry;
15988         struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc;
15989         struct mlx5_flow_dv_matcher matcher = {
15990                 .mask = {
15991                         .size = sizeof(matcher.mask.buf),
15992                 },
15993                 .tbl = tbl_rsc,
15994         };
15995         struct mlx5_flow_dv_match_params value = {
15996                 .size = sizeof(value.buf),
15997         };
15998         struct mlx5_flow_cb_ctx ctx = {
15999                 .error = error,
16000                 .data = &matcher,
16001         };
16002         struct mlx5_flow_tbl_data_entry *tbl_data;
16003         struct mlx5_priv *priv = dev->data->dev_private;
16004         const uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1;
16005
16006         if (match_src_port && (priv->representor || priv->master)) {
16007                 if (flow_dv_translate_item_port_id(dev, matcher.mask.buf,
16008                                                    value.buf, item, attr)) {
16009                         DRV_LOG(ERR, "Failed to register meter policy%d matcher"
16010                                 " with port.", priority);
16011                         return -1;
16012                 }
16013         }
16014         tbl_data = container_of(tbl_rsc, struct mlx5_flow_tbl_data_entry, tbl);
16015         if (priority < RTE_COLOR_RED)
16016                 flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16017                         (enum modify_reg)color_reg_c_idx, 0, color_mask);
16018         matcher.priority = priority;
16019         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
16020                                     matcher.mask.size);
16021         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16022         if (!entry) {
16023                 DRV_LOG(ERR, "Failed to register meter drop matcher.");
16024                 return -1;
16025         }
16026         *policy_matcher =
16027                 container_of(entry, struct mlx5_flow_dv_matcher, entry);
16028         return 0;
16029 }
16030
16031 /**
16032  * Create the policy rules per domain.
16033  *
16034  * @param[in] dev
16035  *   Pointer to Ethernet device.
16036  * @param[in] sub_policy
16037  *    Pointer to sub policy table..
16038  * @param[in] egress
16039  *   Direction of the table.
16040  * @param[in] transfer
16041  *   E-Switch or NIC flow.
16042  * @param[in] acts
16043  *   Pointer to policy action list per color.
16044  *
16045  * @return
16046  *   0 on success, -1 otherwise.
16047  */
16048 static int
16049 __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev,
16050                 struct mlx5_flow_meter_sub_policy *sub_policy,
16051                 uint8_t egress, uint8_t transfer, bool match_src_port,
16052                 struct mlx5_meter_policy_acts acts[RTE_COLORS])
16053 {
16054         struct mlx5_priv *priv = dev->data->dev_private;
16055         struct rte_flow_error flow_err;
16056         uint32_t color_reg_c_idx;
16057         struct rte_flow_attr attr = {
16058                 .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
16059                 .priority = 0,
16060                 .ingress = 0,
16061                 .egress = !!egress,
16062                 .transfer = !!transfer,
16063                 .reserved = 0,
16064         };
16065         int i;
16066         int ret = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &flow_err);
16067         struct mlx5_sub_policy_color_rule *color_rule;
16068         bool svport_match;
16069         struct mlx5_sub_policy_color_rule *tmp_rules[RTE_COLORS] = {NULL};
16070
16071         if (ret < 0)
16072                 return -1;
16073         /* Create policy table with POLICY level. */
16074         if (!sub_policy->tbl_rsc)
16075                 sub_policy->tbl_rsc = flow_dv_tbl_resource_get(dev,
16076                                 MLX5_FLOW_TABLE_LEVEL_POLICY,
16077                                 egress, transfer, false, NULL, 0, 0,
16078                                 sub_policy->idx, &flow_err);
16079         if (!sub_policy->tbl_rsc) {
16080                 DRV_LOG(ERR,
16081                         "Failed to create meter sub policy table.");
16082                 return -1;
16083         }
16084         /* Prepare matchers. */
16085         color_reg_c_idx = ret;
16086         for (i = 0; i < RTE_COLORS; i++) {
16087                 TAILQ_INIT(&sub_policy->color_rules[i]);
16088                 if (!acts[i].actions_n)
16089                         continue;
16090                 color_rule = mlx5_malloc(MLX5_MEM_ZERO,
16091                                 sizeof(struct mlx5_sub_policy_color_rule),
16092                                 0, SOCKET_ID_ANY);
16093                 if (!color_rule) {
16094                         DRV_LOG(ERR, "No memory to create color rule.");
16095                         goto err_exit;
16096                 }
16097                 tmp_rules[i] = color_rule;
16098                 TAILQ_INSERT_TAIL(&sub_policy->color_rules[i],
16099                                   color_rule, next_port);
16100                 color_rule->src_port = priv->representor_id;
16101                 /* No use. */
16102                 attr.priority = i;
16103                 /* Create matchers for colors. */
16104                 svport_match = (i != RTE_COLOR_RED) ? match_src_port : false;
16105                 if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx,
16106                                 MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy,
16107                                 &attr, svport_match, NULL,
16108                                 &color_rule->matcher, &flow_err)) {
16109                         DRV_LOG(ERR, "Failed to create color%u matcher.", i);
16110                         goto err_exit;
16111                 }
16112                 /* Create flow, matching color. */
16113                 if (__flow_dv_create_policy_flow(dev,
16114                                 color_reg_c_idx, (enum rte_color)i,
16115                                 color_rule->matcher->matcher_object,
16116                                 acts[i].actions_n, acts[i].dv_actions,
16117                                 svport_match, NULL, &color_rule->rule,
16118                                 &attr)) {
16119                         DRV_LOG(ERR, "Failed to create color%u rule.", i);
16120                         goto err_exit;
16121                 }
16122         }
16123         return 0;
16124 err_exit:
16125         /* All the policy rules will be cleared. */
16126         do {
16127                 color_rule = tmp_rules[i];
16128                 if (color_rule) {
16129                         if (color_rule->rule)
16130                                 mlx5_flow_os_destroy_flow(color_rule->rule);
16131                         if (color_rule->matcher) {
16132                                 struct mlx5_flow_tbl_data_entry *tbl =
16133                                         container_of(color_rule->matcher->tbl,
16134                                                      typeof(*tbl), tbl);
16135                                 mlx5_list_unregister(tbl->matchers,
16136                                                 &color_rule->matcher->entry);
16137                         }
16138                         TAILQ_REMOVE(&sub_policy->color_rules[i],
16139                                      color_rule, next_port);
16140                         mlx5_free(color_rule);
16141                 }
16142         } while (i--);
16143         return -1;
16144 }
16145
16146 static int
16147 __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev,
16148                         struct mlx5_flow_meter_policy *mtr_policy,
16149                         struct mlx5_flow_meter_sub_policy *sub_policy,
16150                         uint32_t domain)
16151 {
16152         struct mlx5_priv *priv = dev->data->dev_private;
16153         struct mlx5_meter_policy_acts acts[RTE_COLORS];
16154         struct mlx5_flow_dv_tag_resource *tag;
16155         struct mlx5_flow_dv_port_id_action_resource *port_action;
16156         struct mlx5_hrxq *hrxq;
16157         struct mlx5_flow_meter_info *next_fm = NULL;
16158         struct mlx5_flow_meter_policy *next_policy;
16159         struct mlx5_flow_meter_sub_policy *next_sub_policy;
16160         struct mlx5_flow_tbl_data_entry *tbl_data;
16161         struct rte_flow_error error;
16162         uint8_t egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16163         uint8_t transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16164         bool mtr_first = egress || (transfer && priv->representor_id != UINT16_MAX);
16165         bool match_src_port = false;
16166         int i;
16167
16168         /* If RSS or Queue, no previous actions / rules is created. */
16169         for (i = 0; i < RTE_COLORS; i++) {
16170                 acts[i].actions_n = 0;
16171                 if (i == RTE_COLOR_RED) {
16172                         /* Only support drop on red. */
16173                         acts[i].dv_actions[0] =
16174                                 mtr_policy->dr_drop_action[domain];
16175                         acts[i].actions_n = 1;
16176                         continue;
16177                 }
16178                 if (i == RTE_COLOR_GREEN &&
16179                     mtr_policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) {
16180                         struct rte_flow_attr attr = {
16181                                 .transfer = transfer
16182                         };
16183
16184                         next_fm = mlx5_flow_meter_find(priv,
16185                                         mtr_policy->act_cnt[i].next_mtr_id,
16186                                         NULL);
16187                         if (!next_fm) {
16188                                 DRV_LOG(ERR,
16189                                         "Failed to get next hierarchy meter.");
16190                                 goto err_exit;
16191                         }
16192                         if (mlx5_flow_meter_attach(priv, next_fm,
16193                                                    &attr, &error)) {
16194                                 DRV_LOG(ERR, "%s", error.message);
16195                                 next_fm = NULL;
16196                                 goto err_exit;
16197                         }
16198                         /* Meter action must be the first for TX. */
16199                         if (mtr_first) {
16200                                 acts[i].dv_actions[acts[i].actions_n] =
16201                                         next_fm->meter_action;
16202                                 acts[i].actions_n++;
16203                         }
16204                 }
16205                 if (mtr_policy->act_cnt[i].rix_mark) {
16206                         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG],
16207                                         mtr_policy->act_cnt[i].rix_mark);
16208                         if (!tag) {
16209                                 DRV_LOG(ERR, "Failed to find "
16210                                 "mark action for policy.");
16211                                 goto err_exit;
16212                         }
16213                         acts[i].dv_actions[acts[i].actions_n] = tag->action;
16214                         acts[i].actions_n++;
16215                 }
16216                 if (mtr_policy->act_cnt[i].modify_hdr) {
16217                         acts[i].dv_actions[acts[i].actions_n] =
16218                                 mtr_policy->act_cnt[i].modify_hdr->action;
16219                         acts[i].actions_n++;
16220                 }
16221                 if (mtr_policy->act_cnt[i].fate_action) {
16222                         switch (mtr_policy->act_cnt[i].fate_action) {
16223                         case MLX5_FLOW_FATE_PORT_ID:
16224                                 port_action = mlx5_ipool_get
16225                                         (priv->sh->ipool[MLX5_IPOOL_PORT_ID],
16226                                 mtr_policy->act_cnt[i].rix_port_id_action);
16227                                 if (!port_action) {
16228                                         DRV_LOG(ERR, "Failed to find "
16229                                                 "port action for policy.");
16230                                         goto err_exit;
16231                                 }
16232                                 acts[i].dv_actions[acts[i].actions_n] =
16233                                         port_action->action;
16234                                 acts[i].actions_n++;
16235                                 mtr_policy->dev = dev;
16236                                 match_src_port = true;
16237                                 break;
16238                         case MLX5_FLOW_FATE_DROP:
16239                         case MLX5_FLOW_FATE_JUMP:
16240                                 acts[i].dv_actions[acts[i].actions_n] =
16241                                 mtr_policy->act_cnt[i].dr_jump_action[domain];
16242                                 acts[i].actions_n++;
16243                                 break;
16244                         case MLX5_FLOW_FATE_SHARED_RSS:
16245                         case MLX5_FLOW_FATE_QUEUE:
16246                                 hrxq = mlx5_ipool_get
16247                                         (priv->sh->ipool[MLX5_IPOOL_HRXQ],
16248                                          sub_policy->rix_hrxq[i]);
16249                                 if (!hrxq) {
16250                                         DRV_LOG(ERR, "Failed to find "
16251                                                 "queue action for policy.");
16252                                         goto err_exit;
16253                                 }
16254                                 acts[i].dv_actions[acts[i].actions_n] =
16255                                         hrxq->action;
16256                                 acts[i].actions_n++;
16257                                 break;
16258                         case MLX5_FLOW_FATE_MTR:
16259                                 if (!next_fm) {
16260                                         DRV_LOG(ERR,
16261                                                 "No next hierarchy meter.");
16262                                         goto err_exit;
16263                                 }
16264                                 if (!mtr_first) {
16265                                         acts[i].dv_actions[acts[i].actions_n] =
16266                                                         next_fm->meter_action;
16267                                         acts[i].actions_n++;
16268                                 }
16269                                 if (mtr_policy->act_cnt[i].next_sub_policy) {
16270                                         next_sub_policy =
16271                                         mtr_policy->act_cnt[i].next_sub_policy;
16272                                 } else {
16273                                         next_policy =
16274                                                 mlx5_flow_meter_policy_find(dev,
16275                                                 next_fm->policy_id, NULL);
16276                                         MLX5_ASSERT(next_policy);
16277                                         next_sub_policy =
16278                                         next_policy->sub_policys[domain][0];
16279                                 }
16280                                 tbl_data =
16281                                         container_of(next_sub_policy->tbl_rsc,
16282                                         struct mlx5_flow_tbl_data_entry, tbl);
16283                                 acts[i].dv_actions[acts[i].actions_n++] =
16284                                                         tbl_data->jump.action;
16285                                 if (mtr_policy->act_cnt[i].modify_hdr)
16286                                         match_src_port = !!transfer;
16287                                 break;
16288                         default:
16289                                 /*Queue action do nothing*/
16290                                 break;
16291                         }
16292                 }
16293         }
16294         if (__flow_dv_create_domain_policy_rules(dev, sub_policy,
16295                                 egress, transfer, match_src_port, acts)) {
16296                 DRV_LOG(ERR,
16297                         "Failed to create policy rules per domain.");
16298                 goto err_exit;
16299         }
16300         return 0;
16301 err_exit:
16302         if (next_fm)
16303                 mlx5_flow_meter_detach(priv, next_fm);
16304         return -1;
16305 }
16306
16307 /**
16308  * Create the policy rules.
16309  *
16310  * @param[in] dev
16311  *   Pointer to Ethernet device.
16312  * @param[in,out] mtr_policy
16313  *   Pointer to meter policy table.
16314  *
16315  * @return
16316  *   0 on success, -1 otherwise.
16317  */
16318 static int
16319 flow_dv_create_policy_rules(struct rte_eth_dev *dev,
16320                              struct mlx5_flow_meter_policy *mtr_policy)
16321 {
16322         int i;
16323         uint16_t sub_policy_num;
16324
16325         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16326                 sub_policy_num = (mtr_policy->sub_policy_num >>
16327                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
16328                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16329                 if (!sub_policy_num)
16330                         continue;
16331                 /* Prepare actions list and create policy rules. */
16332                 if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
16333                         mtr_policy->sub_policys[i][0], i)) {
16334                         DRV_LOG(ERR, "Failed to create policy action "
16335                                 "list per domain.");
16336                         return -1;
16337                 }
16338         }
16339         return 0;
16340 }
16341
16342 static int
16343 __flow_dv_create_domain_def_policy(struct rte_eth_dev *dev, uint32_t domain)
16344 {
16345         struct mlx5_priv *priv = dev->data->dev_private;
16346         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
16347         struct mlx5_flow_meter_def_policy *def_policy;
16348         struct mlx5_flow_tbl_resource *jump_tbl;
16349         struct mlx5_flow_tbl_data_entry *tbl_data;
16350         uint8_t egress, transfer;
16351         struct rte_flow_error error;
16352         struct mlx5_meter_policy_acts acts[RTE_COLORS];
16353         int ret;
16354
16355         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16356         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16357         def_policy = mtrmng->def_policy[domain];
16358         if (!def_policy) {
16359                 def_policy = mlx5_malloc(MLX5_MEM_ZERO,
16360                         sizeof(struct mlx5_flow_meter_def_policy),
16361                         RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
16362                 if (!def_policy) {
16363                         DRV_LOG(ERR, "Failed to alloc default policy table.");
16364                         goto def_policy_error;
16365                 }
16366                 mtrmng->def_policy[domain] = def_policy;
16367                 /* Create the meter suffix table with SUFFIX level. */
16368                 jump_tbl = flow_dv_tbl_resource_get(dev,
16369                                 MLX5_FLOW_TABLE_LEVEL_METER,
16370                                 egress, transfer, false, NULL, 0,
16371                                 0, MLX5_MTR_TABLE_ID_SUFFIX, &error);
16372                 if (!jump_tbl) {
16373                         DRV_LOG(ERR,
16374                                 "Failed to create meter suffix table.");
16375                         goto def_policy_error;
16376                 }
16377                 def_policy->sub_policy.jump_tbl[RTE_COLOR_GREEN] = jump_tbl;
16378                 tbl_data = container_of(jump_tbl,
16379                                         struct mlx5_flow_tbl_data_entry, tbl);
16380                 def_policy->dr_jump_action[RTE_COLOR_GREEN] =
16381                                                 tbl_data->jump.action;
16382                 acts[RTE_COLOR_GREEN].dv_actions[0] = tbl_data->jump.action;
16383                 acts[RTE_COLOR_GREEN].actions_n = 1;
16384                 /*
16385                  * YELLOW has the same default policy as GREEN does.
16386                  * G & Y share the same table and action. The 2nd time of table
16387                  * resource getting is just to update the reference count for
16388                  * the releasing stage.
16389                  */
16390                 jump_tbl = flow_dv_tbl_resource_get(dev,
16391                                 MLX5_FLOW_TABLE_LEVEL_METER,
16392                                 egress, transfer, false, NULL, 0,
16393                                 0, MLX5_MTR_TABLE_ID_SUFFIX, &error);
16394                 if (!jump_tbl) {
16395                         DRV_LOG(ERR,
16396                                 "Failed to get meter suffix table.");
16397                         goto def_policy_error;
16398                 }
16399                 def_policy->sub_policy.jump_tbl[RTE_COLOR_YELLOW] = jump_tbl;
16400                 tbl_data = container_of(jump_tbl,
16401                                         struct mlx5_flow_tbl_data_entry, tbl);
16402                 def_policy->dr_jump_action[RTE_COLOR_YELLOW] =
16403                                                 tbl_data->jump.action;
16404                 acts[RTE_COLOR_YELLOW].dv_actions[0] = tbl_data->jump.action;
16405                 acts[RTE_COLOR_YELLOW].actions_n = 1;
16406                 /* Create jump action to the drop table. */
16407                 if (!mtrmng->drop_tbl[domain]) {
16408                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get
16409                                 (dev, MLX5_FLOW_TABLE_LEVEL_METER,
16410                                  egress, transfer, false, NULL, 0,
16411                                  0, MLX5_MTR_TABLE_ID_DROP, &error);
16412                         if (!mtrmng->drop_tbl[domain]) {
16413                                 DRV_LOG(ERR, "Failed to create meter "
16414                                         "drop table for default policy.");
16415                                 goto def_policy_error;
16416                         }
16417                 }
16418                 /* all RED: unique Drop table for jump action. */
16419                 tbl_data = container_of(mtrmng->drop_tbl[domain],
16420                                         struct mlx5_flow_tbl_data_entry, tbl);
16421                 def_policy->dr_jump_action[RTE_COLOR_RED] =
16422                                                 tbl_data->jump.action;
16423                 acts[RTE_COLOR_RED].dv_actions[0] = tbl_data->jump.action;
16424                 acts[RTE_COLOR_RED].actions_n = 1;
16425                 /* Create default policy rules. */
16426                 ret = __flow_dv_create_domain_policy_rules(dev,
16427                                         &def_policy->sub_policy,
16428                                         egress, transfer, false, acts);
16429                 if (ret) {
16430                         DRV_LOG(ERR, "Failed to create default policy rules.");
16431                         goto def_policy_error;
16432                 }
16433         }
16434         return 0;
16435 def_policy_error:
16436         __flow_dv_destroy_domain_def_policy(dev,
16437                                             (enum mlx5_meter_domain)domain);
16438         return -1;
16439 }
16440
16441 /**
16442  * Create the default policy table set.
16443  *
16444  * @param[in] dev
16445  *   Pointer to Ethernet device.
16446  * @return
16447  *   0 on success, -1 otherwise.
16448  */
16449 static int
16450 flow_dv_create_def_policy(struct rte_eth_dev *dev)
16451 {
16452         struct mlx5_priv *priv = dev->data->dev_private;
16453         int i;
16454
16455         /* Non-termination policy table. */
16456         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16457                 if (!priv->config.dv_esw_en && i == MLX5_MTR_DOMAIN_TRANSFER)
16458                         continue;
16459                 if (__flow_dv_create_domain_def_policy(dev, i)) {
16460                         DRV_LOG(ERR, "Failed to create default policy");
16461                         /* Rollback the created default policies for others. */
16462                         flow_dv_destroy_def_policy(dev);
16463                         return -1;
16464                 }
16465         }
16466         return 0;
16467 }
16468
16469 /**
16470  * Create the needed meter tables.
16471  * Lock free, (mutex should be acquired by caller).
16472  *
16473  * @param[in] dev
16474  *   Pointer to Ethernet device.
16475  * @param[in] fm
16476  *   Meter information table.
16477  * @param[in] mtr_idx
16478  *   Meter index.
16479  * @param[in] domain_bitmap
16480  *   Domain bitmap.
16481  * @return
16482  *   0 on success, -1 otherwise.
16483  */
16484 static int
16485 flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
16486                         struct mlx5_flow_meter_info *fm,
16487                         uint32_t mtr_idx,
16488                         uint8_t domain_bitmap)
16489 {
16490         struct mlx5_priv *priv = dev->data->dev_private;
16491         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
16492         struct rte_flow_error error;
16493         struct mlx5_flow_tbl_data_entry *tbl_data;
16494         uint8_t egress, transfer;
16495         void *actions[METER_ACTIONS];
16496         int domain, ret, i;
16497         struct mlx5_flow_counter *cnt;
16498         struct mlx5_flow_dv_match_params value = {
16499                 .size = sizeof(value.buf),
16500         };
16501         struct mlx5_flow_dv_match_params matcher_para = {
16502                 .size = sizeof(matcher_para.buf),
16503         };
16504         int mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
16505                                                      0, &error);
16506         uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1;
16507         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
16508         struct mlx5_list_entry *entry;
16509         struct mlx5_flow_dv_matcher matcher = {
16510                 .mask = {
16511                         .size = sizeof(matcher.mask.buf),
16512                 },
16513         };
16514         struct mlx5_flow_dv_matcher *drop_matcher;
16515         struct mlx5_flow_cb_ctx ctx = {
16516                 .error = &error,
16517                 .data = &matcher,
16518         };
16519         uint8_t misc_mask;
16520
16521         if (!priv->mtr_en || mtr_id_reg_c < 0) {
16522                 rte_errno = ENOTSUP;
16523                 return -1;
16524         }
16525         for (domain = 0; domain < MLX5_MTR_DOMAIN_MAX; domain++) {
16526                 if (!(domain_bitmap & (1 << domain)) ||
16527                         (mtrmng->def_rule[domain] && !fm->drop_cnt))
16528                         continue;
16529                 egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16530                 transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16531                 /* Create the drop table with METER DROP level. */
16532                 if (!mtrmng->drop_tbl[domain]) {
16533                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get(dev,
16534                                         MLX5_FLOW_TABLE_LEVEL_METER,
16535                                         egress, transfer, false, NULL, 0,
16536                                         0, MLX5_MTR_TABLE_ID_DROP, &error);
16537                         if (!mtrmng->drop_tbl[domain]) {
16538                                 DRV_LOG(ERR, "Failed to create meter drop table.");
16539                                 goto policy_error;
16540                         }
16541                 }
16542                 /* Create default matcher in drop table. */
16543                 matcher.tbl = mtrmng->drop_tbl[domain],
16544                 tbl_data = container_of(mtrmng->drop_tbl[domain],
16545                                 struct mlx5_flow_tbl_data_entry, tbl);
16546                 if (!mtrmng->def_matcher[domain]) {
16547                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16548                                        (enum modify_reg)mtr_id_reg_c,
16549                                        0, 0);
16550                         matcher.priority = MLX5_MTRS_DEFAULT_RULE_PRIORITY;
16551                         matcher.crc = rte_raw_cksum
16552                                         ((const void *)matcher.mask.buf,
16553                                         matcher.mask.size);
16554                         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16555                         if (!entry) {
16556                                 DRV_LOG(ERR, "Failed to register meter "
16557                                 "drop default matcher.");
16558                                 goto policy_error;
16559                         }
16560                         mtrmng->def_matcher[domain] = container_of(entry,
16561                         struct mlx5_flow_dv_matcher, entry);
16562                 }
16563                 /* Create default rule in drop table. */
16564                 if (!mtrmng->def_rule[domain]) {
16565                         i = 0;
16566                         actions[i++] = priv->sh->dr_drop_action;
16567                         flow_dv_match_meta_reg(matcher_para.buf, value.buf,
16568                                 (enum modify_reg)mtr_id_reg_c, 0, 0);
16569                         misc_mask = flow_dv_matcher_enable(value.buf);
16570                         __flow_dv_adjust_buf_size(&value.size, misc_mask);
16571                         ret = mlx5_flow_os_create_flow
16572                                 (mtrmng->def_matcher[domain]->matcher_object,
16573                                 (void *)&value, i, actions,
16574                                 &mtrmng->def_rule[domain]);
16575                         if (ret) {
16576                                 DRV_LOG(ERR, "Failed to create meter "
16577                                 "default drop rule for drop table.");
16578                                 goto policy_error;
16579                         }
16580                 }
16581                 if (!fm->drop_cnt)
16582                         continue;
16583                 MLX5_ASSERT(mtrmng->max_mtr_bits);
16584                 if (!mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1]) {
16585                         /* Create matchers for Drop. */
16586                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16587                                         (enum modify_reg)mtr_id_reg_c, 0,
16588                                         (mtr_id_mask << mtr_id_offset));
16589                         matcher.priority = MLX5_REG_BITS - mtrmng->max_mtr_bits;
16590                         matcher.crc = rte_raw_cksum
16591                                         ((const void *)matcher.mask.buf,
16592                                         matcher.mask.size);
16593                         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16594                         if (!entry) {
16595                                 DRV_LOG(ERR,
16596                                 "Failed to register meter drop matcher.");
16597                                 goto policy_error;
16598                         }
16599                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1] =
16600                                 container_of(entry, struct mlx5_flow_dv_matcher,
16601                                              entry);
16602                 }
16603                 drop_matcher =
16604                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1];
16605                 /* Create drop rule, matching meter_id only. */
16606                 flow_dv_match_meta_reg(matcher_para.buf, value.buf,
16607                                 (enum modify_reg)mtr_id_reg_c,
16608                                 (mtr_idx << mtr_id_offset), UINT32_MAX);
16609                 i = 0;
16610                 cnt = flow_dv_counter_get_by_idx(dev,
16611                                         fm->drop_cnt, NULL);
16612                 actions[i++] = cnt->action;
16613                 actions[i++] = priv->sh->dr_drop_action;
16614                 misc_mask = flow_dv_matcher_enable(value.buf);
16615                 __flow_dv_adjust_buf_size(&value.size, misc_mask);
16616                 ret = mlx5_flow_os_create_flow(drop_matcher->matcher_object,
16617                                                (void *)&value, i, actions,
16618                                                &fm->drop_rule[domain]);
16619                 if (ret) {
16620                         DRV_LOG(ERR, "Failed to create meter "
16621                                 "drop rule for drop table.");
16622                                 goto policy_error;
16623                 }
16624         }
16625         return 0;
16626 policy_error:
16627         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16628                 if (fm->drop_rule[i]) {
16629                         claim_zero(mlx5_flow_os_destroy_flow
16630                                 (fm->drop_rule[i]));
16631                         fm->drop_rule[i] = NULL;
16632                 }
16633         }
16634         return -1;
16635 }
16636
16637 static struct mlx5_flow_meter_sub_policy *
16638 __flow_dv_meter_get_rss_sub_policy(struct rte_eth_dev *dev,
16639                 struct mlx5_flow_meter_policy *mtr_policy,
16640                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS],
16641                 struct mlx5_flow_meter_sub_policy *next_sub_policy,
16642                 bool *is_reuse)
16643 {
16644         struct mlx5_priv *priv = dev->data->dev_private;
16645         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
16646         uint32_t sub_policy_idx = 0;
16647         uint32_t hrxq_idx[MLX5_MTR_RTE_COLORS] = {0};
16648         uint32_t i, j;
16649         struct mlx5_hrxq *hrxq;
16650         struct mlx5_flow_handle dh;
16651         struct mlx5_meter_policy_action_container *act_cnt;
16652         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
16653         uint16_t sub_policy_num;
16654
16655         rte_spinlock_lock(&mtr_policy->sl);
16656         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16657                 if (!rss_desc[i])
16658                         continue;
16659                 hrxq_idx[i] = mlx5_hrxq_get(dev, rss_desc[i]);
16660                 if (!hrxq_idx[i]) {
16661                         rte_spinlock_unlock(&mtr_policy->sl);
16662                         return NULL;
16663                 }
16664         }
16665         sub_policy_num = (mtr_policy->sub_policy_num >>
16666                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16667                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16668         for (j = 0; j < sub_policy_num; j++) {
16669                 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16670                         if (rss_desc[i] &&
16671                             hrxq_idx[i] !=
16672                             mtr_policy->sub_policys[domain][j]->rix_hrxq[i])
16673                                 break;
16674                 }
16675                 if (i >= MLX5_MTR_RTE_COLORS) {
16676                         /*
16677                          * Found the sub policy table with
16678                          * the same queue per color.
16679                          */
16680                         rte_spinlock_unlock(&mtr_policy->sl);
16681                         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++)
16682                                 mlx5_hrxq_release(dev, hrxq_idx[i]);
16683                         *is_reuse = true;
16684                         return mtr_policy->sub_policys[domain][j];
16685                 }
16686         }
16687         /* Create sub policy. */
16688         if (!mtr_policy->sub_policys[domain][0]->rix_hrxq[0]) {
16689                 /* Reuse the first pre-allocated sub_policy. */
16690                 sub_policy = mtr_policy->sub_policys[domain][0];
16691                 sub_policy_idx = sub_policy->idx;
16692         } else {
16693                 sub_policy = mlx5_ipool_zmalloc
16694                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16695                                  &sub_policy_idx);
16696                 if (!sub_policy ||
16697                     sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) {
16698                         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++)
16699                                 mlx5_hrxq_release(dev, hrxq_idx[i]);
16700                         goto rss_sub_policy_error;
16701                 }
16702                 sub_policy->idx = sub_policy_idx;
16703                 sub_policy->main_policy = mtr_policy;
16704         }
16705         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16706                 if (!rss_desc[i])
16707                         continue;
16708                 sub_policy->rix_hrxq[i] = hrxq_idx[i];
16709                 if (mtr_policy->is_hierarchy) {
16710                         act_cnt = &mtr_policy->act_cnt[i];
16711                         act_cnt->next_sub_policy = next_sub_policy;
16712                         mlx5_hrxq_release(dev, hrxq_idx[i]);
16713                 } else {
16714                         /*
16715                          * Overwrite the last action from
16716                          * RSS action to Queue action.
16717                          */
16718                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
16719                                               hrxq_idx[i]);
16720                         if (!hrxq) {
16721                                 DRV_LOG(ERR, "Failed to get policy hrxq");
16722                                 goto rss_sub_policy_error;
16723                         }
16724                         act_cnt = &mtr_policy->act_cnt[i];
16725                         if (act_cnt->rix_mark || act_cnt->modify_hdr) {
16726                                 memset(&dh, 0, sizeof(struct mlx5_flow_handle));
16727                                 if (act_cnt->rix_mark)
16728                                         dh.mark = 1;
16729                                 dh.fate_action = MLX5_FLOW_FATE_QUEUE;
16730                                 dh.rix_hrxq = hrxq_idx[i];
16731                                 flow_drv_rxq_flags_set(dev, &dh);
16732                         }
16733                 }
16734         }
16735         if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
16736                                                sub_policy, domain)) {
16737                 DRV_LOG(ERR, "Failed to create policy "
16738                         "rules for ingress domain.");
16739                 goto rss_sub_policy_error;
16740         }
16741         if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16742                 i = (mtr_policy->sub_policy_num >>
16743                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16744                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16745                 if (i >= MLX5_MTR_RSS_MAX_SUB_POLICY) {
16746                         DRV_LOG(ERR, "No free sub-policy slot.");
16747                         goto rss_sub_policy_error;
16748                 }
16749                 mtr_policy->sub_policys[domain][i] = sub_policy;
16750                 i++;
16751                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16752                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
16753                 mtr_policy->sub_policy_num |=
16754                         (i & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16755                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
16756         }
16757         rte_spinlock_unlock(&mtr_policy->sl);
16758         *is_reuse = false;
16759         return sub_policy;
16760 rss_sub_policy_error:
16761         if (sub_policy) {
16762                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
16763                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16764                         i = (mtr_policy->sub_policy_num >>
16765                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16766                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16767                         mtr_policy->sub_policys[domain][i] = NULL;
16768                         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16769                                         sub_policy->idx);
16770                 }
16771         }
16772         rte_spinlock_unlock(&mtr_policy->sl);
16773         return NULL;
16774 }
16775
16776 /**
16777  * Find the policy table for prefix table with RSS.
16778  *
16779  * @param[in] dev
16780  *   Pointer to Ethernet device.
16781  * @param[in] mtr_policy
16782  *   Pointer to meter policy table.
16783  * @param[in] rss_desc
16784  *   Pointer to rss_desc
16785  * @return
16786  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
16787  */
16788 static struct mlx5_flow_meter_sub_policy *
16789 flow_dv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
16790                 struct mlx5_flow_meter_policy *mtr_policy,
16791                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
16792 {
16793         struct mlx5_priv *priv = dev->data->dev_private;
16794         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
16795         struct mlx5_flow_meter_info *next_fm;
16796         struct mlx5_flow_meter_policy *next_policy;
16797         struct mlx5_flow_meter_sub_policy *next_sub_policy = NULL;
16798         struct mlx5_flow_meter_policy *policies[MLX5_MTR_CHAIN_MAX_NUM];
16799         struct mlx5_flow_meter_sub_policy *sub_policies[MLX5_MTR_CHAIN_MAX_NUM];
16800         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
16801         bool reuse_sub_policy;
16802         uint32_t i = 0;
16803         uint32_t j = 0;
16804
16805         while (true) {
16806                 /* Iterate hierarchy to get all policies in this hierarchy. */
16807                 policies[i++] = mtr_policy;
16808                 if (!mtr_policy->is_hierarchy)
16809                         break;
16810                 if (i >= MLX5_MTR_CHAIN_MAX_NUM) {
16811                         DRV_LOG(ERR, "Exceed max meter number in hierarchy.");
16812                         return NULL;
16813                 }
16814                 next_fm = mlx5_flow_meter_find(priv,
16815                         mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
16816                 if (!next_fm) {
16817                         DRV_LOG(ERR, "Failed to get next meter in hierarchy.");
16818                         return NULL;
16819                 }
16820                 next_policy =
16821                         mlx5_flow_meter_policy_find(dev, next_fm->policy_id,
16822                                                     NULL);
16823                 MLX5_ASSERT(next_policy);
16824                 mtr_policy = next_policy;
16825         }
16826         while (i) {
16827                 /**
16828                  * From last policy to the first one in hierarchy,
16829                  * create / get the sub policy for each of them.
16830                  */
16831                 sub_policy = __flow_dv_meter_get_rss_sub_policy(dev,
16832                                                         policies[--i],
16833                                                         rss_desc,
16834                                                         next_sub_policy,
16835                                                         &reuse_sub_policy);
16836                 if (!sub_policy) {
16837                         DRV_LOG(ERR, "Failed to get the sub policy.");
16838                         goto err_exit;
16839                 }
16840                 if (!reuse_sub_policy)
16841                         sub_policies[j++] = sub_policy;
16842                 next_sub_policy = sub_policy;
16843         }
16844         return sub_policy;
16845 err_exit:
16846         while (j) {
16847                 uint16_t sub_policy_num;
16848
16849                 sub_policy = sub_policies[--j];
16850                 mtr_policy = sub_policy->main_policy;
16851                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
16852                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16853                         sub_policy_num = (mtr_policy->sub_policy_num >>
16854                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16855                                 MLX5_MTR_SUB_POLICY_NUM_MASK;
16856                         mtr_policy->sub_policys[domain][sub_policy_num - 1] =
16857                                                                         NULL;
16858                         sub_policy_num--;
16859                         mtr_policy->sub_policy_num &=
16860                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16861                                   (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
16862                         mtr_policy->sub_policy_num |=
16863                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16864                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
16865                         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16866                                         sub_policy->idx);
16867                 }
16868         }
16869         return NULL;
16870 }
16871
16872 /**
16873  * Create the sub policy tag rule for all meters in hierarchy.
16874  *
16875  * @param[in] dev
16876  *   Pointer to Ethernet device.
16877  * @param[in] fm
16878  *   Meter information table.
16879  * @param[in] src_port
16880  *   The src port this extra rule should use.
16881  * @param[in] item
16882  *   The src port match item.
16883  * @param[out] error
16884  *   Perform verbose error reporting if not NULL.
16885  * @return
16886  *   0 on success, a negative errno value otherwise and rte_errno is set.
16887  */
16888 static int
16889 flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev,
16890                                 struct mlx5_flow_meter_info *fm,
16891                                 int32_t src_port,
16892                                 const struct rte_flow_item *item,
16893                                 struct rte_flow_error *error)
16894 {
16895         struct mlx5_priv *priv = dev->data->dev_private;
16896         struct mlx5_flow_meter_policy *mtr_policy;
16897         struct mlx5_flow_meter_sub_policy *sub_policy;
16898         struct mlx5_flow_meter_info *next_fm = NULL;
16899         struct mlx5_flow_meter_policy *next_policy;
16900         struct mlx5_flow_meter_sub_policy *next_sub_policy;
16901         struct mlx5_flow_tbl_data_entry *tbl_data;
16902         struct mlx5_sub_policy_color_rule *color_rule;
16903         struct mlx5_meter_policy_acts acts;
16904         uint32_t color_reg_c_idx;
16905         bool mtr_first = (src_port != UINT16_MAX) ? true : false;
16906         struct rte_flow_attr attr = {
16907                 .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
16908                 .priority = 0,
16909                 .ingress = 0,
16910                 .egress = 0,
16911                 .transfer = 1,
16912                 .reserved = 0,
16913         };
16914         uint32_t domain = MLX5_MTR_DOMAIN_TRANSFER;
16915         int i;
16916
16917         mtr_policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
16918         MLX5_ASSERT(mtr_policy);
16919         if (!mtr_policy->is_hierarchy)
16920                 return 0;
16921         next_fm = mlx5_flow_meter_find(priv,
16922                         mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
16923         if (!next_fm) {
16924                 return rte_flow_error_set(error, EINVAL,
16925                                 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
16926                                 "Failed to find next meter in hierarchy.");
16927         }
16928         if (!next_fm->drop_cnt)
16929                 goto exit;
16930         color_reg_c_idx = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, error);
16931         sub_policy = mtr_policy->sub_policys[domain][0];
16932         for (i = 0; i < RTE_COLORS; i++) {
16933                 bool rule_exist = false;
16934                 struct mlx5_meter_policy_action_container *act_cnt;
16935
16936                 if (i >= RTE_COLOR_YELLOW)
16937                         break;
16938                 TAILQ_FOREACH(color_rule,
16939                               &sub_policy->color_rules[i], next_port)
16940                         if (color_rule->src_port == src_port) {
16941                                 rule_exist = true;
16942                                 break;
16943                         }
16944                 if (rule_exist)
16945                         continue;
16946                 color_rule = mlx5_malloc(MLX5_MEM_ZERO,
16947                                 sizeof(struct mlx5_sub_policy_color_rule),
16948                                 0, SOCKET_ID_ANY);
16949                 if (!color_rule)
16950                         return rte_flow_error_set(error, ENOMEM,
16951                                 RTE_FLOW_ERROR_TYPE_ACTION,
16952                                 NULL, "No memory to create tag color rule.");
16953                 color_rule->src_port = src_port;
16954                 attr.priority = i;
16955                 next_policy = mlx5_flow_meter_policy_find(dev,
16956                                                 next_fm->policy_id, NULL);
16957                 MLX5_ASSERT(next_policy);
16958                 next_sub_policy = next_policy->sub_policys[domain][0];
16959                 tbl_data = container_of(next_sub_policy->tbl_rsc,
16960                                         struct mlx5_flow_tbl_data_entry, tbl);
16961                 act_cnt = &mtr_policy->act_cnt[i];
16962                 if (mtr_first) {
16963                         acts.dv_actions[0] = next_fm->meter_action;
16964                         acts.dv_actions[1] = act_cnt->modify_hdr->action;
16965                 } else {
16966                         acts.dv_actions[0] = act_cnt->modify_hdr->action;
16967                         acts.dv_actions[1] = next_fm->meter_action;
16968                 }
16969                 acts.dv_actions[2] = tbl_data->jump.action;
16970                 acts.actions_n = 3;
16971                 if (mlx5_flow_meter_attach(priv, next_fm, &attr, error)) {
16972                         next_fm = NULL;
16973                         goto err_exit;
16974                 }
16975                 if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx,
16976                                 MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy,
16977                                 &attr, true, item,
16978                                 &color_rule->matcher, error)) {
16979                         rte_flow_error_set(error, errno,
16980                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
16981                                 "Failed to create hierarchy meter matcher.");
16982                         goto err_exit;
16983                 }
16984                 if (__flow_dv_create_policy_flow(dev, color_reg_c_idx,
16985                                         (enum rte_color)i,
16986                                         color_rule->matcher->matcher_object,
16987                                         acts.actions_n, acts.dv_actions,
16988                                         true, item,
16989                                         &color_rule->rule, &attr)) {
16990                         rte_flow_error_set(error, errno,
16991                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
16992                                 "Failed to create hierarchy meter rule.");
16993                         goto err_exit;
16994                 }
16995                 TAILQ_INSERT_TAIL(&sub_policy->color_rules[i],
16996                                   color_rule, next_port);
16997         }
16998 exit:
16999         /**
17000          * Recursive call to iterate all meters in hierarchy and
17001          * create needed rules.
17002          */
17003         return flow_dv_meter_hierarchy_rule_create(dev, next_fm,
17004                                                 src_port, item, error);
17005 err_exit:
17006         if (color_rule) {
17007                 if (color_rule->rule)
17008                         mlx5_flow_os_destroy_flow(color_rule->rule);
17009                 if (color_rule->matcher) {
17010                         struct mlx5_flow_tbl_data_entry *tbl =
17011                                 container_of(color_rule->matcher->tbl,
17012                                                 typeof(*tbl), tbl);
17013                         mlx5_list_unregister(tbl->matchers,
17014                                                 &color_rule->matcher->entry);
17015                 }
17016                 mlx5_free(color_rule);
17017         }
17018         if (next_fm)
17019                 mlx5_flow_meter_detach(priv, next_fm);
17020         return -rte_errno;
17021 }
17022
17023 /**
17024  * Destroy the sub policy table with RX queue.
17025  *
17026  * @param[in] dev
17027  *   Pointer to Ethernet device.
17028  * @param[in] mtr_policy
17029  *   Pointer to meter policy table.
17030  */
17031 static void
17032 flow_dv_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
17033                                     struct mlx5_flow_meter_policy *mtr_policy)
17034 {
17035         struct mlx5_priv *priv = dev->data->dev_private;
17036         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
17037         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
17038         uint32_t i, j;
17039         uint16_t sub_policy_num, new_policy_num;
17040
17041         rte_spinlock_lock(&mtr_policy->sl);
17042         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
17043                 switch (mtr_policy->act_cnt[i].fate_action) {
17044                 case MLX5_FLOW_FATE_SHARED_RSS:
17045                         sub_policy_num = (mtr_policy->sub_policy_num >>
17046                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
17047                         MLX5_MTR_SUB_POLICY_NUM_MASK;
17048                         new_policy_num = sub_policy_num;
17049                         for (j = 0; j < sub_policy_num; j++) {
17050                                 sub_policy =
17051                                         mtr_policy->sub_policys[domain][j];
17052                                 if (sub_policy) {
17053                                         __flow_dv_destroy_sub_policy_rules(dev,
17054                                                 sub_policy);
17055                                 if (sub_policy !=
17056                                         mtr_policy->sub_policys[domain][0]) {
17057                                         mtr_policy->sub_policys[domain][j] =
17058                                                                 NULL;
17059                                         mlx5_ipool_free
17060                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
17061                                                 sub_policy->idx);
17062                                                 new_policy_num--;
17063                                         }
17064                                 }
17065                         }
17066                         if (new_policy_num != sub_policy_num) {
17067                                 mtr_policy->sub_policy_num &=
17068                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
17069                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
17070                                 mtr_policy->sub_policy_num |=
17071                                 (new_policy_num &
17072                                         MLX5_MTR_SUB_POLICY_NUM_MASK) <<
17073                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
17074                         }
17075                         break;
17076                 case MLX5_FLOW_FATE_QUEUE:
17077                         sub_policy = mtr_policy->sub_policys[domain][0];
17078                         __flow_dv_destroy_sub_policy_rules(dev,
17079                                                            sub_policy);
17080                         break;
17081                 default:
17082                         /*Other actions without queue and do nothing*/
17083                         break;
17084                 }
17085         }
17086         rte_spinlock_unlock(&mtr_policy->sl);
17087 }
17088 /**
17089  * Check whether the DR drop action is supported on the root table or not.
17090  *
17091  * Create a simple flow with DR drop action on root table to validate
17092  * if DR drop action on root table is supported or not.
17093  *
17094  * @param[in] dev
17095  *   Pointer to rte_eth_dev structure.
17096  *
17097  * @return
17098  *   0 on success, a negative errno value otherwise and rte_errno is set.
17099  */
17100 int
17101 mlx5_flow_discover_dr_action_support(struct rte_eth_dev *dev)
17102 {
17103         struct mlx5_priv *priv = dev->data->dev_private;
17104         struct mlx5_dev_ctx_shared *sh = priv->sh;
17105         struct mlx5_flow_dv_match_params mask = {
17106                 .size = sizeof(mask.buf),
17107         };
17108         struct mlx5_flow_dv_match_params value = {
17109                 .size = sizeof(value.buf),
17110         };
17111         struct mlx5dv_flow_matcher_attr dv_attr = {
17112                 .type = IBV_FLOW_ATTR_NORMAL,
17113                 .priority = 0,
17114                 .match_criteria_enable = 0,
17115                 .match_mask = (void *)&mask,
17116         };
17117         struct mlx5_flow_tbl_resource *tbl = NULL;
17118         void *matcher = NULL;
17119         void *flow = NULL;
17120         int ret = -1;
17121
17122         tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL,
17123                                         0, 0, 0, NULL);
17124         if (!tbl)
17125                 goto err;
17126         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
17127         __flow_dv_adjust_buf_size(&mask.size, dv_attr.match_criteria_enable);
17128         ret = mlx5_flow_os_create_flow_matcher(sh->cdev->ctx, &dv_attr,
17129                                                tbl->obj, &matcher);
17130         if (ret)
17131                 goto err;
17132         __flow_dv_adjust_buf_size(&value.size, dv_attr.match_criteria_enable);
17133         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 1,
17134                                        &sh->dr_drop_action, &flow);
17135 err:
17136         /*
17137          * If DR drop action is not supported on root table, flow create will
17138          * be failed with EOPNOTSUPP or EPROTONOSUPPORT.
17139          */
17140         if (!flow) {
17141                 if (matcher &&
17142                     (errno == EPROTONOSUPPORT || errno == EOPNOTSUPP))
17143                         DRV_LOG(INFO, "DR drop action is not supported in root table.");
17144                 else
17145                         DRV_LOG(ERR, "Unexpected error in DR drop action support detection");
17146                 ret = -1;
17147         } else {
17148                 claim_zero(mlx5_flow_os_destroy_flow(flow));
17149         }
17150         if (matcher)
17151                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
17152         if (tbl)
17153                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
17154         return ret;
17155 }
17156
17157 /**
17158  * Validate the batch counter support in root table.
17159  *
17160  * Create a simple flow with invalid counter and drop action on root table to
17161  * validate if batch counter with offset on root table is supported or not.
17162  *
17163  * @param[in] dev
17164  *   Pointer to rte_eth_dev structure.
17165  *
17166  * @return
17167  *   0 on success, a negative errno value otherwise and rte_errno is set.
17168  */
17169 int
17170 mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev)
17171 {
17172         struct mlx5_priv *priv = dev->data->dev_private;
17173         struct mlx5_dev_ctx_shared *sh = priv->sh;
17174         struct mlx5_flow_dv_match_params mask = {
17175                 .size = sizeof(mask.buf),
17176         };
17177         struct mlx5_flow_dv_match_params value = {
17178                 .size = sizeof(value.buf),
17179         };
17180         struct mlx5dv_flow_matcher_attr dv_attr = {
17181                 .type = IBV_FLOW_ATTR_NORMAL | IBV_FLOW_ATTR_FLAGS_EGRESS,
17182                 .priority = 0,
17183                 .match_criteria_enable = 0,
17184                 .match_mask = (void *)&mask,
17185         };
17186         void *actions[2] = { 0 };
17187         struct mlx5_flow_tbl_resource *tbl = NULL;
17188         struct mlx5_devx_obj *dcs = NULL;
17189         void *matcher = NULL;
17190         void *flow = NULL;
17191         int ret = -1;
17192
17193         tbl = flow_dv_tbl_resource_get(dev, 0, 1, 0, false, NULL,
17194                                         0, 0, 0, NULL);
17195         if (!tbl)
17196                 goto err;
17197         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->cdev->ctx, 0x4);
17198         if (!dcs)
17199                 goto err;
17200         ret = mlx5_flow_os_create_flow_action_count(dcs->obj, UINT16_MAX,
17201                                                     &actions[0]);
17202         if (ret)
17203                 goto err;
17204         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
17205         __flow_dv_adjust_buf_size(&mask.size, dv_attr.match_criteria_enable);
17206         ret = mlx5_flow_os_create_flow_matcher(sh->cdev->ctx, &dv_attr,
17207                                                tbl->obj, &matcher);
17208         if (ret)
17209                 goto err;
17210         __flow_dv_adjust_buf_size(&value.size, dv_attr.match_criteria_enable);
17211         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 1,
17212                                        actions, &flow);
17213 err:
17214         /*
17215          * If batch counter with offset is not supported, the driver will not
17216          * validate the invalid offset value, flow create should success.
17217          * In this case, it means batch counter is not supported in root table.
17218          *
17219          * Otherwise, if flow create is failed, counter offset is supported.
17220          */
17221         if (flow) {
17222                 DRV_LOG(INFO, "Batch counter is not supported in root "
17223                               "table. Switch to fallback mode.");
17224                 rte_errno = ENOTSUP;
17225                 ret = -rte_errno;
17226                 claim_zero(mlx5_flow_os_destroy_flow(flow));
17227         } else {
17228                 /* Check matcher to make sure validate fail at flow create. */
17229                 if (!matcher || (matcher && errno != EINVAL))
17230                         DRV_LOG(ERR, "Unexpected error in counter offset "
17231                                      "support detection");
17232                 ret = 0;
17233         }
17234         if (actions[0])
17235                 claim_zero(mlx5_flow_os_destroy_flow_action(actions[0]));
17236         if (matcher)
17237                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
17238         if (tbl)
17239                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
17240         if (dcs)
17241                 claim_zero(mlx5_devx_cmd_destroy(dcs));
17242         return ret;
17243 }
17244
17245 /**
17246  * Query a devx counter.
17247  *
17248  * @param[in] dev
17249  *   Pointer to the Ethernet device structure.
17250  * @param[in] cnt
17251  *   Index to the flow counter.
17252  * @param[in] clear
17253  *   Set to clear the counter statistics.
17254  * @param[out] pkts
17255  *   The statistics value of packets.
17256  * @param[out] bytes
17257  *   The statistics value of bytes.
17258  *
17259  * @return
17260  *   0 on success, otherwise return -1.
17261  */
17262 static int
17263 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
17264                       uint64_t *pkts, uint64_t *bytes)
17265 {
17266         struct mlx5_priv *priv = dev->data->dev_private;
17267         struct mlx5_flow_counter *cnt;
17268         uint64_t inn_pkts, inn_bytes;
17269         int ret;
17270
17271         if (!priv->sh->devx)
17272                 return -1;
17273
17274         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
17275         if (ret)
17276                 return -1;
17277         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
17278         *pkts = inn_pkts - cnt->hits;
17279         *bytes = inn_bytes - cnt->bytes;
17280         if (clear) {
17281                 cnt->hits = inn_pkts;
17282                 cnt->bytes = inn_bytes;
17283         }
17284         return 0;
17285 }
17286
17287 /**
17288  * Get aged-out flows.
17289  *
17290  * @param[in] dev
17291  *   Pointer to the Ethernet device structure.
17292  * @param[in] context
17293  *   The address of an array of pointers to the aged-out flows contexts.
17294  * @param[in] nb_contexts
17295  *   The length of context array pointers.
17296  * @param[out] error
17297  *   Perform verbose error reporting if not NULL. Initialized in case of
17298  *   error only.
17299  *
17300  * @return
17301  *   how many contexts get in success, otherwise negative errno value.
17302  *   if nb_contexts is 0, return the amount of all aged contexts.
17303  *   if nb_contexts is not 0 , return the amount of aged flows reported
17304  *   in the context array.
17305  * @note: only stub for now
17306  */
17307 static int
17308 flow_dv_get_aged_flows(struct rte_eth_dev *dev,
17309                     void **context,
17310                     uint32_t nb_contexts,
17311                     struct rte_flow_error *error)
17312 {
17313         struct mlx5_priv *priv = dev->data->dev_private;
17314         struct mlx5_age_info *age_info;
17315         struct mlx5_age_param *age_param;
17316         struct mlx5_flow_counter *counter;
17317         struct mlx5_aso_age_action *act;
17318         int nb_flows = 0;
17319
17320         if (nb_contexts && !context)
17321                 return rte_flow_error_set(error, EINVAL,
17322                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
17323                                           NULL, "empty context");
17324         age_info = GET_PORT_AGE_INFO(priv);
17325         rte_spinlock_lock(&age_info->aged_sl);
17326         LIST_FOREACH(act, &age_info->aged_aso, next) {
17327                 nb_flows++;
17328                 if (nb_contexts) {
17329                         context[nb_flows - 1] =
17330                                                 act->age_params.context;
17331                         if (!(--nb_contexts))
17332                                 break;
17333                 }
17334         }
17335         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
17336                 nb_flows++;
17337                 if (nb_contexts) {
17338                         age_param = MLX5_CNT_TO_AGE(counter);
17339                         context[nb_flows - 1] = age_param->context;
17340                         if (!(--nb_contexts))
17341                                 break;
17342                 }
17343         }
17344         rte_spinlock_unlock(&age_info->aged_sl);
17345         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
17346         return nb_flows;
17347 }
17348
17349 /*
17350  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
17351  */
17352 static uint32_t
17353 flow_dv_counter_allocate(struct rte_eth_dev *dev)
17354 {
17355         return flow_dv_counter_alloc(dev, 0);
17356 }
17357
17358 /**
17359  * Validate indirect action.
17360  * Dispatcher for action type specific validation.
17361  *
17362  * @param[in] dev
17363  *   Pointer to the Ethernet device structure.
17364  * @param[in] conf
17365  *   Indirect action configuration.
17366  * @param[in] action
17367  *   The indirect action object to validate.
17368  * @param[out] error
17369  *   Perform verbose error reporting if not NULL. Initialized in case of
17370  *   error only.
17371  *
17372  * @return
17373  *   0 on success, otherwise negative errno value.
17374  */
17375 static int
17376 flow_dv_action_validate(struct rte_eth_dev *dev,
17377                         const struct rte_flow_indir_action_conf *conf,
17378                         const struct rte_flow_action *action,
17379                         struct rte_flow_error *err)
17380 {
17381         struct mlx5_priv *priv = dev->data->dev_private;
17382
17383         RTE_SET_USED(conf);
17384         switch (action->type) {
17385         case RTE_FLOW_ACTION_TYPE_RSS:
17386                 /*
17387                  * priv->obj_ops is set according to driver capabilities.
17388                  * When DevX capabilities are
17389                  * sufficient, it is set to devx_obj_ops.
17390                  * Otherwise, it is set to ibv_obj_ops.
17391                  * ibv_obj_ops doesn't support ind_table_modify operation.
17392                  * In this case the indirect RSS action can't be used.
17393                  */
17394                 if (priv->obj_ops.ind_table_modify == NULL)
17395                         return rte_flow_error_set
17396                                         (err, ENOTSUP,
17397                                          RTE_FLOW_ERROR_TYPE_ACTION,
17398                                          NULL,
17399                                          "Indirect RSS action not supported");
17400                 return mlx5_validate_action_rss(dev, action, err);
17401         case RTE_FLOW_ACTION_TYPE_AGE:
17402                 if (!priv->sh->aso_age_mng)
17403                         return rte_flow_error_set(err, ENOTSUP,
17404                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
17405                                                 NULL,
17406                                                 "Indirect age action not supported");
17407                 return flow_dv_validate_action_age(0, action, dev, err);
17408         case RTE_FLOW_ACTION_TYPE_COUNT:
17409                 return flow_dv_validate_action_count(dev, true, 0, err);
17410         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
17411                 if (!priv->sh->ct_aso_en)
17412                         return rte_flow_error_set(err, ENOTSUP,
17413                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
17414                                         "ASO CT is not supported");
17415                 return mlx5_validate_action_ct(dev, action->conf, err);
17416         default:
17417                 return rte_flow_error_set(err, ENOTSUP,
17418                                           RTE_FLOW_ERROR_TYPE_ACTION,
17419                                           NULL,
17420                                           "action type not supported");
17421         }
17422 }
17423
17424 /*
17425  * Check if the RSS configurations for colors of a meter policy match
17426  * each other, except the queues.
17427  *
17428  * @param[in] r1
17429  *   Pointer to the first RSS flow action.
17430  * @param[in] r2
17431  *   Pointer to the second RSS flow action.
17432  *
17433  * @return
17434  *   0 on match, 1 on conflict.
17435  */
17436 static inline int
17437 flow_dv_mtr_policy_rss_compare(const struct rte_flow_action_rss *r1,
17438                                const struct rte_flow_action_rss *r2)
17439 {
17440         if (!r1 || !r2)
17441                 return 0;
17442         if (r1->func != r2->func || r1->level != r2->level ||
17443             r1->types != r2->types || r1->key_len != r2->key_len ||
17444             memcmp(r1->key, r2->key, r1->key_len))
17445                 return 1;
17446         return 0;
17447 }
17448
17449 /**
17450  * Validate the meter hierarchy chain for meter policy.
17451  *
17452  * @param[in] dev
17453  *   Pointer to the Ethernet device structure.
17454  * @param[in] meter_id
17455  *   Meter id.
17456  * @param[in] action_flags
17457  *   Holds the actions detected until now.
17458  * @param[out] is_rss
17459  *   Is RSS or not.
17460  * @param[out] hierarchy_domain
17461  *   The domain bitmap for hierarchy policy.
17462  * @param[out] error
17463  *   Perform verbose error reporting if not NULL. Initialized in case of
17464  *   error only.
17465  *
17466  * @return
17467  *   0 on success, otherwise negative errno value with error set.
17468  */
17469 static int
17470 flow_dv_validate_policy_mtr_hierarchy(struct rte_eth_dev *dev,
17471                                   uint32_t meter_id,
17472                                   uint64_t action_flags,
17473                                   bool *is_rss,
17474                                   uint8_t *hierarchy_domain,
17475                                   struct rte_mtr_error *error)
17476 {
17477         struct mlx5_priv *priv = dev->data->dev_private;
17478         struct mlx5_flow_meter_info *fm;
17479         struct mlx5_flow_meter_policy *policy;
17480         uint8_t cnt = 1;
17481
17482         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
17483                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
17484                 return -rte_mtr_error_set(error, EINVAL,
17485                                         RTE_MTR_ERROR_TYPE_POLICER_ACTION_GREEN,
17486                                         NULL,
17487                                         "Multiple fate actions not supported.");
17488         *hierarchy_domain = 0;
17489         while (true) {
17490                 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
17491                 if (!fm)
17492                         return -rte_mtr_error_set(error, EINVAL,
17493                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
17494                                         "Meter not found in meter hierarchy.");
17495                 if (fm->def_policy)
17496                         return -rte_mtr_error_set(error, EINVAL,
17497                                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
17498                         "Non termination meter not supported in hierarchy.");
17499                 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
17500                 MLX5_ASSERT(policy);
17501                 /**
17502                  * Only inherit the supported domains of the first meter in
17503                  * hierarchy.
17504                  * One meter supports at least one domain.
17505                  */
17506                 if (!*hierarchy_domain) {
17507                         if (policy->transfer)
17508                                 *hierarchy_domain |=
17509                                                 MLX5_MTR_DOMAIN_TRANSFER_BIT;
17510                         if (policy->ingress)
17511                                 *hierarchy_domain |=
17512                                                 MLX5_MTR_DOMAIN_INGRESS_BIT;
17513                         if (policy->egress)
17514                                 *hierarchy_domain |= MLX5_MTR_DOMAIN_EGRESS_BIT;
17515                 }
17516                 if (!policy->is_hierarchy) {
17517                         *is_rss = policy->is_rss;
17518                         break;
17519                 }
17520                 meter_id = policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id;
17521                 if (++cnt >= MLX5_MTR_CHAIN_MAX_NUM)
17522                         return -rte_mtr_error_set(error, EINVAL,
17523                                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
17524                                         "Exceed max hierarchy meter number.");
17525         }
17526         return 0;
17527 }
17528
17529 /**
17530  * Validate meter policy actions.
17531  * Dispatcher for action type specific validation.
17532  *
17533  * @param[in] dev
17534  *   Pointer to the Ethernet device structure.
17535  * @param[in] action
17536  *   The meter policy action object to validate.
17537  * @param[in] attr
17538  *   Attributes of flow to determine steering domain.
17539  * @param[out] error
17540  *   Perform verbose error reporting if not NULL. Initialized in case of
17541  *   error only.
17542  *
17543  * @return
17544  *   0 on success, otherwise negative errno value.
17545  */
17546 static int
17547 flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev,
17548                         const struct rte_flow_action *actions[RTE_COLORS],
17549                         struct rte_flow_attr *attr,
17550                         bool *is_rss,
17551                         uint8_t *domain_bitmap,
17552                         uint8_t *policy_mode,
17553                         struct rte_mtr_error *error)
17554 {
17555         struct mlx5_priv *priv = dev->data->dev_private;
17556         struct mlx5_dev_config *dev_conf = &priv->config;
17557         const struct rte_flow_action *act;
17558         uint64_t action_flags[RTE_COLORS] = {0};
17559         int actions_n;
17560         int i, ret;
17561         struct rte_flow_error flow_err;
17562         uint8_t domain_color[RTE_COLORS] = {0};
17563         uint8_t def_domain = MLX5_MTR_ALL_DOMAIN_BIT;
17564         uint8_t hierarchy_domain = 0;
17565         const struct rte_flow_action_meter *mtr;
17566         bool def_green = false;
17567         bool def_yellow = false;
17568         const struct rte_flow_action_rss *rss_color[RTE_COLORS] = {NULL};
17569
17570         if (!priv->config.dv_esw_en)
17571                 def_domain &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
17572         *domain_bitmap = def_domain;
17573         /* Red color could only support DROP action. */
17574         if (!actions[RTE_COLOR_RED] ||
17575             actions[RTE_COLOR_RED]->type != RTE_FLOW_ACTION_TYPE_DROP)
17576                 return -rte_mtr_error_set(error, ENOTSUP,
17577                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17578                                 NULL, "Red color only supports drop action.");
17579         /*
17580          * Check default policy actions:
17581          * Green / Yellow: no action, Red: drop action
17582          * Either G or Y will trigger default policy actions to be created.
17583          */
17584         if (!actions[RTE_COLOR_GREEN] ||
17585             actions[RTE_COLOR_GREEN]->type == RTE_FLOW_ACTION_TYPE_END)
17586                 def_green = true;
17587         if (!actions[RTE_COLOR_YELLOW] ||
17588             actions[RTE_COLOR_YELLOW]->type == RTE_FLOW_ACTION_TYPE_END)
17589                 def_yellow = true;
17590         if (def_green && def_yellow) {
17591                 *policy_mode = MLX5_MTR_POLICY_MODE_DEF;
17592                 return 0;
17593         } else if (!def_green && def_yellow) {
17594                 *policy_mode = MLX5_MTR_POLICY_MODE_OG;
17595         } else if (def_green && !def_yellow) {
17596                 *policy_mode = MLX5_MTR_POLICY_MODE_OY;
17597         }
17598         /* Set to empty string in case of NULL pointer access by user. */
17599         flow_err.message = "";
17600         for (i = 0; i < RTE_COLORS; i++) {
17601                 act = actions[i];
17602                 for (action_flags[i] = 0, actions_n = 0;
17603                      act && act->type != RTE_FLOW_ACTION_TYPE_END;
17604                      act++) {
17605                         if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
17606                                 return -rte_mtr_error_set(error, ENOTSUP,
17607                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17608                                           NULL, "too many actions");
17609                         switch (act->type) {
17610                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
17611                         case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
17612                                 if (!priv->config.dv_esw_en)
17613                                         return -rte_mtr_error_set(error,
17614                                         ENOTSUP,
17615                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17616                                         NULL, "PORT action validate check"
17617                                         " fail for ESW disable");
17618                                 ret = flow_dv_validate_action_port_id(dev,
17619                                                 action_flags[i],
17620                                                 act, attr, &flow_err);
17621                                 if (ret)
17622                                         return -rte_mtr_error_set(error,
17623                                         ENOTSUP,
17624                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17625                                         NULL, flow_err.message ?
17626                                         flow_err.message :
17627                                         "PORT action validate check fail");
17628                                 ++actions_n;
17629                                 action_flags[i] |= MLX5_FLOW_ACTION_PORT_ID;
17630                                 break;
17631                         case RTE_FLOW_ACTION_TYPE_MARK:
17632                                 ret = flow_dv_validate_action_mark(dev, act,
17633                                                            action_flags[i],
17634                                                            attr, &flow_err);
17635                                 if (ret < 0)
17636                                         return -rte_mtr_error_set(error,
17637                                         ENOTSUP,
17638                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17639                                         NULL, flow_err.message ?
17640                                         flow_err.message :
17641                                         "Mark action validate check fail");
17642                                 if (dev_conf->dv_xmeta_en !=
17643                                         MLX5_XMETA_MODE_LEGACY)
17644                                         return -rte_mtr_error_set(error,
17645                                         ENOTSUP,
17646                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17647                                         NULL, "Extend MARK action is "
17648                                         "not supported. Please try use "
17649                                         "default policy for meter.");
17650                                 action_flags[i] |= MLX5_FLOW_ACTION_MARK;
17651                                 ++actions_n;
17652                                 break;
17653                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
17654                                 ret = flow_dv_validate_action_set_tag(dev,
17655                                                         act, action_flags[i],
17656                                                         attr, &flow_err);
17657                                 if (ret)
17658                                         return -rte_mtr_error_set(error,
17659                                         ENOTSUP,
17660                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17661                                         NULL, flow_err.message ?
17662                                         flow_err.message :
17663                                         "Set tag action validate check fail");
17664                                 action_flags[i] |= MLX5_FLOW_ACTION_SET_TAG;
17665                                 ++actions_n;
17666                                 break;
17667                         case RTE_FLOW_ACTION_TYPE_DROP:
17668                                 ret = mlx5_flow_validate_action_drop
17669                                         (action_flags[i], attr, &flow_err);
17670                                 if (ret < 0)
17671                                         return -rte_mtr_error_set(error,
17672                                         ENOTSUP,
17673                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17674                                         NULL, flow_err.message ?
17675                                         flow_err.message :
17676                                         "Drop action validate check fail");
17677                                 action_flags[i] |= MLX5_FLOW_ACTION_DROP;
17678                                 ++actions_n;
17679                                 break;
17680                         case RTE_FLOW_ACTION_TYPE_QUEUE:
17681                                 /*
17682                                  * Check whether extensive
17683                                  * metadata feature is engaged.
17684                                  */
17685                                 if (dev_conf->dv_flow_en &&
17686                                     (dev_conf->dv_xmeta_en !=
17687                                      MLX5_XMETA_MODE_LEGACY) &&
17688                                     mlx5_flow_ext_mreg_supported(dev))
17689                                         return -rte_mtr_error_set(error,
17690                                           ENOTSUP,
17691                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17692                                           NULL, "Queue action with meta "
17693                                           "is not supported. Please try use "
17694                                           "default policy for meter.");
17695                                 ret = mlx5_flow_validate_action_queue(act,
17696                                                         action_flags[i], dev,
17697                                                         attr, &flow_err);
17698                                 if (ret < 0)
17699                                         return -rte_mtr_error_set(error,
17700                                           ENOTSUP,
17701                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17702                                           NULL, flow_err.message ?
17703                                           flow_err.message :
17704                                           "Queue action validate check fail");
17705                                 action_flags[i] |= MLX5_FLOW_ACTION_QUEUE;
17706                                 ++actions_n;
17707                                 break;
17708                         case RTE_FLOW_ACTION_TYPE_RSS:
17709                                 if (dev_conf->dv_flow_en &&
17710                                     (dev_conf->dv_xmeta_en !=
17711                                      MLX5_XMETA_MODE_LEGACY) &&
17712                                     mlx5_flow_ext_mreg_supported(dev))
17713                                         return -rte_mtr_error_set(error,
17714                                           ENOTSUP,
17715                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17716                                           NULL, "RSS action with meta "
17717                                           "is not supported. Please try use "
17718                                           "default policy for meter.");
17719                                 ret = mlx5_validate_action_rss(dev, act,
17720                                                                &flow_err);
17721                                 if (ret < 0)
17722                                         return -rte_mtr_error_set(error,
17723                                           ENOTSUP,
17724                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17725                                           NULL, flow_err.message ?
17726                                           flow_err.message :
17727                                           "RSS action validate check fail");
17728                                 action_flags[i] |= MLX5_FLOW_ACTION_RSS;
17729                                 ++actions_n;
17730                                 /* Either G or Y will set the RSS. */
17731                                 rss_color[i] = act->conf;
17732                                 break;
17733                         case RTE_FLOW_ACTION_TYPE_JUMP:
17734                                 ret = flow_dv_validate_action_jump(dev,
17735                                         NULL, act, action_flags[i],
17736                                         attr, true, &flow_err);
17737                                 if (ret)
17738                                         return -rte_mtr_error_set(error,
17739                                           ENOTSUP,
17740                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17741                                           NULL, flow_err.message ?
17742                                           flow_err.message :
17743                                           "Jump action validate check fail");
17744                                 ++actions_n;
17745                                 action_flags[i] |= MLX5_FLOW_ACTION_JUMP;
17746                                 break;
17747                         /*
17748                          * Only the last meter in the hierarchy will support
17749                          * the YELLOW color steering. Then in the meter policy
17750                          * actions list, there should be no other meter inside.
17751                          */
17752                         case RTE_FLOW_ACTION_TYPE_METER:
17753                                 if (i != RTE_COLOR_GREEN)
17754                                         return -rte_mtr_error_set(error,
17755                                                 ENOTSUP,
17756                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17757                                                 NULL,
17758                                                 "Meter hierarchy only supports GREEN color.");
17759                                 if (*policy_mode != MLX5_MTR_POLICY_MODE_OG)
17760                                         return -rte_mtr_error_set(error,
17761                                                 ENOTSUP,
17762                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17763                                                 NULL,
17764                                                 "No yellow policy should be provided in meter hierarchy.");
17765                                 mtr = act->conf;
17766                                 ret = flow_dv_validate_policy_mtr_hierarchy(dev,
17767                                                         mtr->mtr_id,
17768                                                         action_flags[i],
17769                                                         is_rss,
17770                                                         &hierarchy_domain,
17771                                                         error);
17772                                 if (ret)
17773                                         return ret;
17774                                 ++actions_n;
17775                                 action_flags[i] |=
17776                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
17777                                 break;
17778                         default:
17779                                 return -rte_mtr_error_set(error, ENOTSUP,
17780                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17781                                         NULL,
17782                                         "Doesn't support optional action");
17783                         }
17784                 }
17785                 if (action_flags[i] & MLX5_FLOW_ACTION_PORT_ID) {
17786                         domain_color[i] = MLX5_MTR_DOMAIN_TRANSFER_BIT;
17787                 } else if ((action_flags[i] &
17788                           (MLX5_FLOW_ACTION_RSS | MLX5_FLOW_ACTION_QUEUE)) ||
17789                           (action_flags[i] & MLX5_FLOW_ACTION_MARK)) {
17790                         /*
17791                          * Only support MLX5_XMETA_MODE_LEGACY
17792                          * so MARK action is only in ingress domain.
17793                          */
17794                         domain_color[i] = MLX5_MTR_DOMAIN_INGRESS_BIT;
17795                 } else {
17796                         domain_color[i] = def_domain;
17797                         if (action_flags[i] &&
17798                             !(action_flags[i] & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
17799                                 domain_color[i] &=
17800                                 ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
17801                 }
17802                 if (action_flags[i] &
17803                     MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
17804                         domain_color[i] &= hierarchy_domain;
17805                 /*
17806                  * Non-termination actions only support NIC Tx domain.
17807                  * The adjustion should be skipped when there is no
17808                  * action or only END is provided. The default domains
17809                  * bit-mask is set to find the MIN intersection.
17810                  * The action flags checking should also be skipped.
17811                  */
17812                 if ((def_green && i == RTE_COLOR_GREEN) ||
17813                     (def_yellow && i == RTE_COLOR_YELLOW))
17814                         continue;
17815                 /*
17816                  * Validate the drop action mutual exclusion
17817                  * with other actions. Drop action is mutually-exclusive
17818                  * with any other action, except for Count action.
17819                  */
17820                 if ((action_flags[i] & MLX5_FLOW_ACTION_DROP) &&
17821                     (action_flags[i] & ~MLX5_FLOW_ACTION_DROP)) {
17822                         return -rte_mtr_error_set(error, ENOTSUP,
17823                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17824                                 NULL, "Drop action is mutually-exclusive "
17825                                 "with any other action");
17826                 }
17827                 /* Eswitch has few restrictions on using items and actions */
17828                 if (domain_color[i] & MLX5_MTR_DOMAIN_TRANSFER_BIT) {
17829                         if (!mlx5_flow_ext_mreg_supported(dev) &&
17830                             action_flags[i] & MLX5_FLOW_ACTION_MARK)
17831                                 return -rte_mtr_error_set(error, ENOTSUP,
17832                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17833                                         NULL, "unsupported action MARK");
17834                         if (action_flags[i] & MLX5_FLOW_ACTION_QUEUE)
17835                                 return -rte_mtr_error_set(error, ENOTSUP,
17836                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17837                                         NULL, "unsupported action QUEUE");
17838                         if (action_flags[i] & MLX5_FLOW_ACTION_RSS)
17839                                 return -rte_mtr_error_set(error, ENOTSUP,
17840                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17841                                         NULL, "unsupported action RSS");
17842                         if (!(action_flags[i] & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
17843                                 return -rte_mtr_error_set(error, ENOTSUP,
17844                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17845                                         NULL, "no fate action is found");
17846                 } else {
17847                         if (!(action_flags[i] & MLX5_FLOW_FATE_ACTIONS) &&
17848                             (domain_color[i] & MLX5_MTR_DOMAIN_INGRESS_BIT)) {
17849                                 if ((domain_color[i] &
17850                                      MLX5_MTR_DOMAIN_EGRESS_BIT))
17851                                         domain_color[i] =
17852                                                 MLX5_MTR_DOMAIN_EGRESS_BIT;
17853                                 else
17854                                         return -rte_mtr_error_set(error,
17855                                                 ENOTSUP,
17856                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17857                                                 NULL,
17858                                                 "no fate action is found");
17859                         }
17860                 }
17861         }
17862         /* If both colors have RSS, the attributes should be the same. */
17863         if (flow_dv_mtr_policy_rss_compare(rss_color[RTE_COLOR_GREEN],
17864                                            rss_color[RTE_COLOR_YELLOW]))
17865                 return -rte_mtr_error_set(error, EINVAL,
17866                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17867                                           NULL, "policy RSS attr conflict");
17868         if (rss_color[RTE_COLOR_GREEN] || rss_color[RTE_COLOR_YELLOW])
17869                 *is_rss = true;
17870         /* "domain_color[C]" is non-zero for each color, default is ALL. */
17871         if (!def_green && !def_yellow &&
17872             domain_color[RTE_COLOR_GREEN] != domain_color[RTE_COLOR_YELLOW] &&
17873             !(action_flags[RTE_COLOR_GREEN] & MLX5_FLOW_ACTION_DROP) &&
17874             !(action_flags[RTE_COLOR_YELLOW] & MLX5_FLOW_ACTION_DROP))
17875                 return -rte_mtr_error_set(error, EINVAL,
17876                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17877                                           NULL, "policy domains conflict");
17878         /*
17879          * At least one color policy is listed in the actions, the domains
17880          * to be supported should be the intersection.
17881          */
17882         *domain_bitmap = domain_color[RTE_COLOR_GREEN] &
17883                          domain_color[RTE_COLOR_YELLOW];
17884         return 0;
17885 }
17886
17887 static int
17888 flow_dv_sync_domain(struct rte_eth_dev *dev, uint32_t domains, uint32_t flags)
17889 {
17890         struct mlx5_priv *priv = dev->data->dev_private;
17891         int ret = 0;
17892
17893         if ((domains & MLX5_DOMAIN_BIT_NIC_RX) && priv->sh->rx_domain != NULL) {
17894                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->rx_domain,
17895                                                 flags);
17896                 if (ret != 0)
17897                         return ret;
17898         }
17899         if ((domains & MLX5_DOMAIN_BIT_NIC_TX) && priv->sh->tx_domain != NULL) {
17900                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->tx_domain, flags);
17901                 if (ret != 0)
17902                         return ret;
17903         }
17904         if ((domains & MLX5_DOMAIN_BIT_FDB) && priv->sh->fdb_domain != NULL) {
17905                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->fdb_domain, flags);
17906                 if (ret != 0)
17907                         return ret;
17908         }
17909         return 0;
17910 }
17911
17912 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
17913         .validate = flow_dv_validate,
17914         .prepare = flow_dv_prepare,
17915         .translate = flow_dv_translate,
17916         .apply = flow_dv_apply,
17917         .remove = flow_dv_remove,
17918         .destroy = flow_dv_destroy,
17919         .query = flow_dv_query,
17920         .create_mtr_tbls = flow_dv_create_mtr_tbls,
17921         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbls,
17922         .destroy_mtr_drop_tbls = flow_dv_destroy_mtr_drop_tbls,
17923         .create_meter = flow_dv_mtr_alloc,
17924         .free_meter = flow_dv_aso_mtr_release_to_pool,
17925         .validate_mtr_acts = flow_dv_validate_mtr_policy_acts,
17926         .create_mtr_acts = flow_dv_create_mtr_policy_acts,
17927         .destroy_mtr_acts = flow_dv_destroy_mtr_policy_acts,
17928         .create_policy_rules = flow_dv_create_policy_rules,
17929         .destroy_policy_rules = flow_dv_destroy_policy_rules,
17930         .create_def_policy = flow_dv_create_def_policy,
17931         .destroy_def_policy = flow_dv_destroy_def_policy,
17932         .meter_sub_policy_rss_prepare = flow_dv_meter_sub_policy_rss_prepare,
17933         .meter_hierarchy_rule_create = flow_dv_meter_hierarchy_rule_create,
17934         .destroy_sub_policy_with_rxq = flow_dv_destroy_sub_policy_with_rxq,
17935         .counter_alloc = flow_dv_counter_allocate,
17936         .counter_free = flow_dv_counter_free,
17937         .counter_query = flow_dv_counter_query,
17938         .get_aged_flows = flow_dv_get_aged_flows,
17939         .action_validate = flow_dv_action_validate,
17940         .action_create = flow_dv_action_create,
17941         .action_destroy = flow_dv_action_destroy,
17942         .action_update = flow_dv_action_update,
17943         .action_query = flow_dv_action_query,
17944         .sync_domain = flow_dv_sync_domain,
17945 };
17946
17947 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
17948