drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_ip.h>
  19 #include <rte_gre.h>
  20 #include <rte_vxlan.h>
  21 #include <rte_gtp.h>
  22 #include <rte_eal_paging.h>
  23 #include <rte_mpls.h>
  24 #include <rte_mtr.h>
  25 #include <rte_mtr_driver.h>
  26 #include <rte_tailq.h>
  27
  28 #include <mlx5_glue.h>
  29 #include <mlx5_devx_cmds.h>
  30 #include <mlx5_prm.h>
  31 #include <mlx5_malloc.h>
  32
  33 #include "mlx5_defs.h"
  34 #include "mlx5.h"
  35 #include "mlx5_common_os.h"
  36 #include "mlx5_flow.h"
  37 #include "mlx5_flow_os.h"
  38 #include "mlx5_rx.h"
  39 #include "mlx5_tx.h"
  40 #include "rte_pmd_mlx5.h"
  41
  42 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  43
  44 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  45 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  46 #endif
  47
  48 #ifndef HAVE_MLX5DV_DR_ESWITCH
  49 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  50 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  51 #endif
  52 #endif
  53
  54 #ifndef HAVE_MLX5DV_DR
  55 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  56 #endif
  57
  58 /* VLAN header definitions */
  59 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  60 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  61 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  62 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  63 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  64
  65 union flow_dv_attr {
  66         struct {
  67                 uint32_t valid:1;
  68                 uint32_t ipv4:1;
  69                 uint32_t ipv6:1;
  70                 uint32_t tcp:1;
  71                 uint32_t udp:1;
  72                 uint32_t reserved:27;
  73         };
  74         uint32_t attr;
  75 };
  76
  77 static int
  78 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
  79                              struct mlx5_flow_tbl_resource *tbl);
  80
  81 static int
  82 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  83                                      uint32_t encap_decap_idx);
  84
  85 static int
  86 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  87                                         uint32_t port_id);
  88 static void
  89 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss);
  90
  91 static int
  92 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
  93                                   uint32_t rix_jump);
  94
  95 /**
  96  * Initialize flow attributes structure according to flow items' types.
  97  *
  98  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  99  * mode. For tunnel mode, the items to be modified are the outermost ones.
 100  *
 101  * @param[in] item
 102  *   Pointer to item specification.
 103  * @param[out] attr
 104  *   Pointer to flow attributes structure.
 105  * @param[in] dev_flow
 106  *   Pointer to the sub flow.
 107  * @param[in] tunnel_decap
 108  *   Whether action is after tunnel decapsulation.
 109  */
 110 static void
 111 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 112                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 113 {
 114         uint64_t layers = dev_flow->handle->layers;
 115
 116         /*
 117          * If layers is already initialized, it means this dev_flow is the
 118          * suffix flow, the layers flags is set by the prefix flow. Need to
 119          * use the layer flags from prefix flow as the suffix flow may not
 120          * have the user defined items as the flow is split.
 121          */
 122         if (layers) {
 123                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 124                         attr->ipv4 = 1;
 125                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 126                         attr->ipv6 = 1;
 127                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 128                         attr->tcp = 1;
 129                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 130                         attr->udp = 1;
 131                 attr->valid = 1;
 132                 return;
 133         }
 134         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 135                 uint8_t next_protocol = 0xff;
 136                 switch (item->type) {
 137                 case RTE_FLOW_ITEM_TYPE_GRE:
 138                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 139                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 140                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 141                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 142                 case RTE_FLOW_ITEM_TYPE_MPLS:
 143                         if (tunnel_decap)
 144                                 attr->attr = 0;
 145                         break;
 146                 case RTE_FLOW_ITEM_TYPE_IPV4:
 147                         if (!attr->ipv6)
 148                                 attr->ipv4 = 1;
 149                         if (item->mask != NULL &&
 150                             ((const struct rte_flow_item_ipv4 *)
 151                             item->mask)->hdr.next_proto_id)
 152                                 next_protocol =
 153                                     ((const struct rte_flow_item_ipv4 *)
 154                                       (item->spec))->hdr.next_proto_id &
 155                                     ((const struct rte_flow_item_ipv4 *)
 156                                       (item->mask))->hdr.next_proto_id;
 157                         if ((next_protocol == IPPROTO_IPIP ||
 158                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 159                                 attr->attr = 0;
 160                         break;
 161                 case RTE_FLOW_ITEM_TYPE_IPV6:
 162                         if (!attr->ipv4)
 163                                 attr->ipv6 = 1;
 164                         if (item->mask != NULL &&
 165                             ((const struct rte_flow_item_ipv6 *)
 166                             item->mask)->hdr.proto)
 167                                 next_protocol =
 168                                     ((const struct rte_flow_item_ipv6 *)
 169                                       (item->spec))->hdr.proto &
 170                                     ((const struct rte_flow_item_ipv6 *)
 171                                       (item->mask))->hdr.proto;
 172                         if ((next_protocol == IPPROTO_IPIP ||
 173                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 174                                 attr->attr = 0;
 175                         break;
 176                 case RTE_FLOW_ITEM_TYPE_UDP:
 177                         if (!attr->tcp)
 178                                 attr->udp = 1;
 179                         break;
 180                 case RTE_FLOW_ITEM_TYPE_TCP:
 181                         if (!attr->udp)
 182                                 attr->tcp = 1;
 183                         break;
 184                 default:
 185                         break;
 186                 }
 187         }
 188         attr->valid = 1;
 189 }
 190
 191 /*
 192  * Convert rte_mtr_color to mlx5 color.
 193  *
 194  * @param[in] rcol
 195  *   rte_mtr_color.
 196  *
 197  * @return
 198  *   mlx5 color.
 199  */
 200 static inline int
 201 rte_col_2_mlx5_col(enum rte_color rcol)
 202 {
 203         switch (rcol) {
 204         case RTE_COLOR_GREEN:
 205                 return MLX5_FLOW_COLOR_GREEN;
 206         case RTE_COLOR_YELLOW:
 207                 return MLX5_FLOW_COLOR_YELLOW;
 208         case RTE_COLOR_RED:
 209                 return MLX5_FLOW_COLOR_RED;
 210         default:
 211                 break;
 212         }
 213         return MLX5_FLOW_COLOR_UNDEFINED;
 214 }
 215
 216 struct field_modify_info {
 217         uint32_t size; /* Size of field in protocol header, in bytes. */
 218         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 219         enum mlx5_modification_field id;
 220 };
 221
 222 struct field_modify_info modify_eth[] = {
 223         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 224         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 225         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 226         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 227         {0, 0, 0},
 228 };
 229
 230 struct field_modify_info modify_vlan_out_first_vid[] = {
 231         /* Size in bits !!! */
 232         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 233         {0, 0, 0},
 234 };
 235
 236 struct field_modify_info modify_ipv4[] = {
 237         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 238         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 239         {4, 12, MLX5_MODI_OUT_SIPV4},
 240         {4, 16, MLX5_MODI_OUT_DIPV4},
 241         {0, 0, 0},
 242 };
 243
 244 struct field_modify_info modify_ipv6[] = {
 245         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 246         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 247         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 248         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 249         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 250         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 251         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 252         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 253         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 254         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 255         {0, 0, 0},
 256 };
 257
 258 struct field_modify_info modify_udp[] = {
 259         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 260         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 261         {0, 0, 0},
 262 };
 263
 264 struct field_modify_info modify_tcp[] = {
 265         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 266         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 267         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 268         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 269         {0, 0, 0},
 270 };
 271
 272 static const struct rte_flow_item *
 273 mlx5_flow_find_tunnel_item(const struct rte_flow_item *item)
 274 {
 275         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 276                 switch (item->type) {
 277                 default:
 278                         break;
 279                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 280                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 281                 case RTE_FLOW_ITEM_TYPE_GRE:
 282                 case RTE_FLOW_ITEM_TYPE_MPLS:
 283                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 284                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 285                         return item;
 286                 case RTE_FLOW_ITEM_TYPE_IPV4:
 287                 case RTE_FLOW_ITEM_TYPE_IPV6:
 288                         if (item[1].type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 289                             item[1].type == RTE_FLOW_ITEM_TYPE_IPV6)
 290                                 return item;
 291                         break;
 292                 }
 293         }
 294         return NULL;
 295 }
 296
 297 static void
 298 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 299                           uint8_t next_protocol, uint64_t *item_flags,
 300                           int *tunnel)
 301 {
 302         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 303                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 304         if (next_protocol == IPPROTO_IPIP) {
 305                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 306                 *tunnel = 1;
 307         }
 308         if (next_protocol == IPPROTO_IPV6) {
 309                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 310                 *tunnel = 1;
 311         }
 312 }
 313
 314 static inline struct mlx5_hlist *
 315 flow_dv_hlist_prepare(struct mlx5_dev_ctx_shared *sh, struct mlx5_hlist **phl,
 316                      const char *name, uint32_t size, bool direct_key,
 317                      bool lcores_share, void *ctx,
 318                      mlx5_list_create_cb cb_create,
 319                      mlx5_list_match_cb cb_match,
 320                      mlx5_list_remove_cb cb_remove,
 321                      mlx5_list_clone_cb cb_clone,
 322                      mlx5_list_clone_free_cb cb_clone_free)
 323 {
 324         struct mlx5_hlist *hl;
 325         struct mlx5_hlist *expected = NULL;
 326         char s[MLX5_NAME_SIZE];
 327
 328         hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
 329         if (likely(hl))
 330                 return hl;
 331         snprintf(s, sizeof(s), "%s_%s", sh->ibdev_name, name);
 332         hl = mlx5_hlist_create(s, size, direct_key, lcores_share,
 333                         ctx, cb_create, cb_match, cb_remove, cb_clone,
 334                         cb_clone_free);
 335         if (!hl) {
 336                 DRV_LOG(ERR, "%s hash creation failed", name);
 337                 rte_errno = ENOMEM;
 338                 return NULL;
 339         }
 340         if (!__atomic_compare_exchange_n(phl, &expected, hl, false,
 341                                          __ATOMIC_SEQ_CST,
 342                                          __ATOMIC_SEQ_CST)) {
 343                 mlx5_hlist_destroy(hl);
 344                 hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
 345         }
 346         return hl;
 347 }
 348
 349 /* Update VLAN's VID/PCP based on input rte_flow_action.
 350  *
 351  * @param[in] action
 352  *   Pointer to struct rte_flow_action.
 353  * @param[out] vlan
 354  *   Pointer to struct rte_vlan_hdr.
 355  */
 356 static void
 357 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 358                          struct rte_vlan_hdr *vlan)
 359 {
 360         uint16_t vlan_tci;
 361         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 362                 vlan_tci =
 363                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 364                                                action->conf)->vlan_pcp;
 365                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 366                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 367                 vlan->vlan_tci |= vlan_tci;
 368         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 369                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 370                 vlan->vlan_tci |= rte_be_to_cpu_16
 371                     (((const struct rte_flow_action_of_set_vlan_vid *)
 372                                              action->conf)->vlan_vid);
 373         }
 374 }
 375
 376 /**
 377  * Fetch 1, 2, 3 or 4 byte field from the byte array
 378  * and return as unsigned integer in host-endian format.
 379  *
 380  * @param[in] data
 381  *   Pointer to data array.
 382  * @param[in] size
 383  *   Size of field to extract.
 384  *
 385  * @return
 386  *   converted field in host endian format.
 387  */
 388 static inline uint32_t
 389 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 390 {
 391         uint32_t ret;
 392
 393         switch (size) {
 394         case 1:
 395                 ret = *data;
 396                 break;
 397         case 2:
 398                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 399                 break;
 400         case 3:
 401                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 402                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 403                 break;
 404         case 4:
 405                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 406                 break;
 407         default:
 408                 MLX5_ASSERT(false);
 409                 ret = 0;
 410                 break;
 411         }
 412         return ret;
 413 }
 414
 415 /**
 416  * Convert modify-header action to DV specification.
 417  *
 418  * Data length of each action is determined by provided field description
 419  * and the item mask. Data bit offset and width of each action is determined
 420  * by provided item mask.
 421  *
 422  * @param[in] item
 423  *   Pointer to item specification.
 424  * @param[in] field
 425  *   Pointer to field modification information.
 426  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 427  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 428  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 429  * @param[in] dcopy
 430  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 431  *   Negative offset value sets the same offset as source offset.
 432  *   size field is ignored, value is taken from source field.
 433  * @param[in,out] resource
 434  *   Pointer to the modify-header resource.
 435  * @param[in] type
 436  *   Type of modification.
 437  * @param[out] error
 438  *   Pointer to the error structure.
 439  *
 440  * @return
 441  *   0 on success, a negative errno value otherwise and rte_errno is set.
 442  */
 443 static int
 444 flow_dv_convert_modify_action(struct rte_flow_item *item,
 445                               struct field_modify_info *field,
 446                               struct field_modify_info *dcopy,
 447                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 448                               uint32_t type, struct rte_flow_error *error)
 449 {
 450         uint32_t i = resource->actions_num;
 451         struct mlx5_modification_cmd *actions = resource->actions;
 452         uint32_t carry_b = 0;
 453
 454         /*
 455          * The item and mask are provided in big-endian format.
 456          * The fields should be presented as in big-endian format either.
 457          * Mask must be always present, it defines the actual field width.
 458          */
 459         MLX5_ASSERT(item->mask);
 460         MLX5_ASSERT(field->size);
 461         do {
 462                 uint32_t size_b;
 463                 uint32_t off_b;
 464                 uint32_t mask;
 465                 uint32_t data;
 466                 bool next_field = true;
 467                 bool next_dcopy = true;
 468
 469                 if (i >= MLX5_MAX_MODIFY_NUM)
 470                         return rte_flow_error_set(error, EINVAL,
 471                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 472                                  "too many items to modify");
 473                 /* Fetch variable byte size mask from the array. */
 474                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 475                                            field->offset, field->size);
 476                 if (!mask) {
 477                         ++field;
 478                         continue;
 479                 }
 480                 /* Deduce actual data width in bits from mask value. */
 481                 off_b = rte_bsf32(mask) + carry_b;
 482                 size_b = sizeof(uint32_t) * CHAR_BIT -
 483                          off_b - __builtin_clz(mask);
 484                 MLX5_ASSERT(size_b);
 485                 actions[i] = (struct mlx5_modification_cmd) {
 486                         .action_type = type,
 487                         .field = field->id,
 488                         .offset = off_b,
 489                         .length = (size_b == sizeof(uint32_t) * CHAR_BIT) ?
 490                                 0 : size_b,
 491                 };
 492                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 493                         MLX5_ASSERT(dcopy);
 494                         actions[i].dst_field = dcopy->id;
 495                         actions[i].dst_offset =
 496                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 497                         /* Convert entire record to big-endian format. */
 498                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 499                         /*
 500                          * Destination field overflow. Copy leftovers of
 501                          * a source field to the next destination field.
 502                          */
 503                         carry_b = 0;
 504                         if ((size_b > dcopy->size * CHAR_BIT - dcopy->offset) &&
 505                             dcopy->size != 0) {
 506                                 actions[i].length =
 507                                         dcopy->size * CHAR_BIT - dcopy->offset;
 508                                 carry_b = actions[i].length;
 509                                 next_field = false;
 510                         }
 511                         /*
 512                          * Not enough bits in a source filed to fill a
 513                          * destination field. Switch to the next source.
 514                          */
 515                         if ((size_b < dcopy->size * CHAR_BIT - dcopy->offset) &&
 516                             (size_b == field->size * CHAR_BIT - off_b)) {
 517                                 actions[i].length =
 518                                         field->size * CHAR_BIT - off_b;
 519                                 dcopy->offset += actions[i].length;
 520                                 next_dcopy = false;
 521                         }
 522                         if (next_dcopy)
 523                                 ++dcopy;
 524                 } else {
 525                         MLX5_ASSERT(item->spec);
 526                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 527                                                    field->offset, field->size);
 528                         /* Shift out the trailing masked bits from data. */
 529                         data = (data & mask) >> off_b;
 530                         actions[i].data1 = rte_cpu_to_be_32(data);
 531                 }
 532                 /* Convert entire record to expected big-endian format. */
 533                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 534                 if (next_field)
 535                         ++field;
 536                 ++i;
 537         } while (field->size);
 538         if (resource->actions_num == i)
 539                 return rte_flow_error_set(error, EINVAL,
 540                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 541                                           "invalid modification flow item");
 542         resource->actions_num = i;
 543         return 0;
 544 }
 545
 546 /**
 547  * Convert modify-header set IPv4 address action to DV specification.
 548  *
 549  * @param[in,out] resource
 550  *   Pointer to the modify-header resource.
 551  * @param[in] action
 552  *   Pointer to action specification.
 553  * @param[out] error
 554  *   Pointer to the error structure.
 555  *
 556  * @return
 557  *   0 on success, a negative errno value otherwise and rte_errno is set.
 558  */
 559 static int
 560 flow_dv_convert_action_modify_ipv4
 561                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 562                          const struct rte_flow_action *action,
 563                          struct rte_flow_error *error)
 564 {
 565         const struct rte_flow_action_set_ipv4 *conf =
 566                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 567         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 568         struct rte_flow_item_ipv4 ipv4;
 569         struct rte_flow_item_ipv4 ipv4_mask;
 570
 571         memset(&ipv4, 0, sizeof(ipv4));
 572         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 573         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 574                 ipv4.hdr.src_addr = conf->ipv4_addr;
 575                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 576         } else {
 577                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 578                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 579         }
 580         item.spec = &ipv4;
 581         item.mask = &ipv4_mask;
 582         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 583                                              MLX5_MODIFICATION_TYPE_SET, error);
 584 }
 585
 586 /**
 587  * Convert modify-header set IPv6 address action to DV specification.
 588  *
 589  * @param[in,out] resource
 590  *   Pointer to the modify-header resource.
 591  * @param[in] action
 592  *   Pointer to action specification.
 593  * @param[out] error
 594  *   Pointer to the error structure.
 595  *
 596  * @return
 597  *   0 on success, a negative errno value otherwise and rte_errno is set.
 598  */
 599 static int
 600 flow_dv_convert_action_modify_ipv6
 601                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 602                          const struct rte_flow_action *action,
 603                          struct rte_flow_error *error)
 604 {
 605         const struct rte_flow_action_set_ipv6 *conf =
 606                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 607         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 608         struct rte_flow_item_ipv6 ipv6;
 609         struct rte_flow_item_ipv6 ipv6_mask;
 610
 611         memset(&ipv6, 0, sizeof(ipv6));
 612         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 613         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 614                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 615                        sizeof(ipv6.hdr.src_addr));
 616                 memcpy(&ipv6_mask.hdr.src_addr,
 617                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 618                        sizeof(ipv6.hdr.src_addr));
 619         } else {
 620                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 621                        sizeof(ipv6.hdr.dst_addr));
 622                 memcpy(&ipv6_mask.hdr.dst_addr,
 623                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 624                        sizeof(ipv6.hdr.dst_addr));
 625         }
 626         item.spec = &ipv6;
 627         item.mask = &ipv6_mask;
 628         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 629                                              MLX5_MODIFICATION_TYPE_SET, error);
 630 }
 631
 632 /**
 633  * Convert modify-header set MAC address action to DV specification.
 634  *
 635  * @param[in,out] resource
 636  *   Pointer to the modify-header resource.
 637  * @param[in] action
 638  *   Pointer to action specification.
 639  * @param[out] error
 640  *   Pointer to the error structure.
 641  *
 642  * @return
 643  *   0 on success, a negative errno value otherwise and rte_errno is set.
 644  */
 645 static int
 646 flow_dv_convert_action_modify_mac
 647                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 648                          const struct rte_flow_action *action,
 649                          struct rte_flow_error *error)
 650 {
 651         const struct rte_flow_action_set_mac *conf =
 652                 (const struct rte_flow_action_set_mac *)(action->conf);
 653         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 654         struct rte_flow_item_eth eth;
 655         struct rte_flow_item_eth eth_mask;
 656
 657         memset(&eth, 0, sizeof(eth));
 658         memset(&eth_mask, 0, sizeof(eth_mask));
 659         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 660                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 661                        sizeof(eth.src.addr_bytes));
 662                 memcpy(&eth_mask.src.addr_bytes,
 663                        &rte_flow_item_eth_mask.src.addr_bytes,
 664                        sizeof(eth_mask.src.addr_bytes));
 665         } else {
 666                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 667                        sizeof(eth.dst.addr_bytes));
 668                 memcpy(&eth_mask.dst.addr_bytes,
 669                        &rte_flow_item_eth_mask.dst.addr_bytes,
 670                        sizeof(eth_mask.dst.addr_bytes));
 671         }
 672         item.spec = &eth;
 673         item.mask = &eth_mask;
 674         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 675                                              MLX5_MODIFICATION_TYPE_SET, error);
 676 }
 677
 678 /**
 679  * Convert modify-header set VLAN VID action to DV specification.
 680  *
 681  * @param[in,out] resource
 682  *   Pointer to the modify-header resource.
 683  * @param[in] action
 684  *   Pointer to action specification.
 685  * @param[out] error
 686  *   Pointer to the error structure.
 687  *
 688  * @return
 689  *   0 on success, a negative errno value otherwise and rte_errno is set.
 690  */
 691 static int
 692 flow_dv_convert_action_modify_vlan_vid
 693                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 694                          const struct rte_flow_action *action,
 695                          struct rte_flow_error *error)
 696 {
 697         const struct rte_flow_action_of_set_vlan_vid *conf =
 698                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 699         int i = resource->actions_num;
 700         struct mlx5_modification_cmd *actions = resource->actions;
 701         struct field_modify_info *field = modify_vlan_out_first_vid;
 702
 703         if (i >= MLX5_MAX_MODIFY_NUM)
 704                 return rte_flow_error_set(error, EINVAL,
 705                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 706                          "too many items to modify");
 707         actions[i] = (struct mlx5_modification_cmd) {
 708                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 709                 .field = field->id,
 710                 .length = field->size,
 711                 .offset = field->offset,
 712         };
 713         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 714         actions[i].data1 = conf->vlan_vid;
 715         actions[i].data1 = actions[i].data1 << 16;
 716         resource->actions_num = ++i;
 717         return 0;
 718 }
 719
 720 /**
 721  * Convert modify-header set TP action to DV specification.
 722  *
 723  * @param[in,out] resource
 724  *   Pointer to the modify-header resource.
 725  * @param[in] action
 726  *   Pointer to action specification.
 727  * @param[in] items
 728  *   Pointer to rte_flow_item objects list.
 729  * @param[in] attr
 730  *   Pointer to flow attributes structure.
 731  * @param[in] dev_flow
 732  *   Pointer to the sub flow.
 733  * @param[in] tunnel_decap
 734  *   Whether action is after tunnel decapsulation.
 735  * @param[out] error
 736  *   Pointer to the error structure.
 737  *
 738  * @return
 739  *   0 on success, a negative errno value otherwise and rte_errno is set.
 740  */
 741 static int
 742 flow_dv_convert_action_modify_tp
 743                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 744                          const struct rte_flow_action *action,
 745                          const struct rte_flow_item *items,
 746                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 747                          bool tunnel_decap, struct rte_flow_error *error)
 748 {
 749         const struct rte_flow_action_set_tp *conf =
 750                 (const struct rte_flow_action_set_tp *)(action->conf);
 751         struct rte_flow_item item;
 752         struct rte_flow_item_udp udp;
 753         struct rte_flow_item_udp udp_mask;
 754         struct rte_flow_item_tcp tcp;
 755         struct rte_flow_item_tcp tcp_mask;
 756         struct field_modify_info *field;
 757
 758         if (!attr->valid)
 759                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 760         if (attr->udp) {
 761                 memset(&udp, 0, sizeof(udp));
 762                 memset(&udp_mask, 0, sizeof(udp_mask));
 763                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 764                         udp.hdr.src_port = conf->port;
 765                         udp_mask.hdr.src_port =
 766                                         rte_flow_item_udp_mask.hdr.src_port;
 767                 } else {
 768                         udp.hdr.dst_port = conf->port;
 769                         udp_mask.hdr.dst_port =
 770                                         rte_flow_item_udp_mask.hdr.dst_port;
 771                 }
 772                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 773                 item.spec = &udp;
 774                 item.mask = &udp_mask;
 775                 field = modify_udp;
 776         } else {
 777                 MLX5_ASSERT(attr->tcp);
 778                 memset(&tcp, 0, sizeof(tcp));
 779                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 780                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 781                         tcp.hdr.src_port = conf->port;
 782                         tcp_mask.hdr.src_port =
 783                                         rte_flow_item_tcp_mask.hdr.src_port;
 784                 } else {
 785                         tcp.hdr.dst_port = conf->port;
 786                         tcp_mask.hdr.dst_port =
 787                                         rte_flow_item_tcp_mask.hdr.dst_port;
 788                 }
 789                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 790                 item.spec = &tcp;
 791                 item.mask = &tcp_mask;
 792                 field = modify_tcp;
 793         }
 794         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 795                                              MLX5_MODIFICATION_TYPE_SET, error);
 796 }
 797
 798 /**
 799  * Convert modify-header set TTL action to DV specification.
 800  *
 801  * @param[in,out] resource
 802  *   Pointer to the modify-header resource.
 803  * @param[in] action
 804  *   Pointer to action specification.
 805  * @param[in] items
 806  *   Pointer to rte_flow_item objects list.
 807  * @param[in] attr
 808  *   Pointer to flow attributes structure.
 809  * @param[in] dev_flow
 810  *   Pointer to the sub flow.
 811  * @param[in] tunnel_decap
 812  *   Whether action is after tunnel decapsulation.
 813  * @param[out] error
 814  *   Pointer to the error structure.
 815  *
 816  * @return
 817  *   0 on success, a negative errno value otherwise and rte_errno is set.
 818  */
 819 static int
 820 flow_dv_convert_action_modify_ttl
 821                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 822                          const struct rte_flow_action *action,
 823                          const struct rte_flow_item *items,
 824                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 825                          bool tunnel_decap, struct rte_flow_error *error)
 826 {
 827         const struct rte_flow_action_set_ttl *conf =
 828                 (const struct rte_flow_action_set_ttl *)(action->conf);
 829         struct rte_flow_item item;
 830         struct rte_flow_item_ipv4 ipv4;
 831         struct rte_flow_item_ipv4 ipv4_mask;
 832         struct rte_flow_item_ipv6 ipv6;
 833         struct rte_flow_item_ipv6 ipv6_mask;
 834         struct field_modify_info *field;
 835
 836         if (!attr->valid)
 837                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 838         if (attr->ipv4) {
 839                 memset(&ipv4, 0, sizeof(ipv4));
 840                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 841                 ipv4.hdr.time_to_live = conf->ttl_value;
 842                 ipv4_mask.hdr.time_to_live = 0xFF;
 843                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 844                 item.spec = &ipv4;
 845                 item.mask = &ipv4_mask;
 846                 field = modify_ipv4;
 847         } else {
 848                 MLX5_ASSERT(attr->ipv6);
 849                 memset(&ipv6, 0, sizeof(ipv6));
 850                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 851                 ipv6.hdr.hop_limits = conf->ttl_value;
 852                 ipv6_mask.hdr.hop_limits = 0xFF;
 853                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 854                 item.spec = &ipv6;
 855                 item.mask = &ipv6_mask;
 856                 field = modify_ipv6;
 857         }
 858         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 859                                              MLX5_MODIFICATION_TYPE_SET, error);
 860 }
 861
 862 /**
 863  * Convert modify-header decrement TTL action to DV specification.
 864  *
 865  * @param[in,out] resource
 866  *   Pointer to the modify-header resource.
 867  * @param[in] action
 868  *   Pointer to action specification.
 869  * @param[in] items
 870  *   Pointer to rte_flow_item objects list.
 871  * @param[in] attr
 872  *   Pointer to flow attributes structure.
 873  * @param[in] dev_flow
 874  *   Pointer to the sub flow.
 875  * @param[in] tunnel_decap
 876  *   Whether action is after tunnel decapsulation.
 877  * @param[out] error
 878  *   Pointer to the error structure.
 879  *
 880  * @return
 881  *   0 on success, a negative errno value otherwise and rte_errno is set.
 882  */
 883 static int
 884 flow_dv_convert_action_modify_dec_ttl
 885                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 886                          const struct rte_flow_item *items,
 887                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 888                          bool tunnel_decap, struct rte_flow_error *error)
 889 {
 890         struct rte_flow_item item;
 891         struct rte_flow_item_ipv4 ipv4;
 892         struct rte_flow_item_ipv4 ipv4_mask;
 893         struct rte_flow_item_ipv6 ipv6;
 894         struct rte_flow_item_ipv6 ipv6_mask;
 895         struct field_modify_info *field;
 896
 897         if (!attr->valid)
 898                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 899         if (attr->ipv4) {
 900                 memset(&ipv4, 0, sizeof(ipv4));
 901                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 902                 ipv4.hdr.time_to_live = 0xFF;
 903                 ipv4_mask.hdr.time_to_live = 0xFF;
 904                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 905                 item.spec = &ipv4;
 906                 item.mask = &ipv4_mask;
 907                 field = modify_ipv4;
 908         } else {
 909                 MLX5_ASSERT(attr->ipv6);
 910                 memset(&ipv6, 0, sizeof(ipv6));
 911                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 912                 ipv6.hdr.hop_limits = 0xFF;
 913                 ipv6_mask.hdr.hop_limits = 0xFF;
 914                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 915                 item.spec = &ipv6;
 916                 item.mask = &ipv6_mask;
 917                 field = modify_ipv6;
 918         }
 919         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 920                                              MLX5_MODIFICATION_TYPE_ADD, error);
 921 }
 922
 923 /**
 924  * Convert modify-header increment/decrement TCP Sequence number
 925  * to DV specification.
 926  *
 927  * @param[in,out] resource
 928  *   Pointer to the modify-header resource.
 929  * @param[in] action
 930  *   Pointer to action specification.
 931  * @param[out] error
 932  *   Pointer to the error structure.
 933  *
 934  * @return
 935  *   0 on success, a negative errno value otherwise and rte_errno is set.
 936  */
 937 static int
 938 flow_dv_convert_action_modify_tcp_seq
 939                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 940                          const struct rte_flow_action *action,
 941                          struct rte_flow_error *error)
 942 {
 943         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 944         uint64_t value = rte_be_to_cpu_32(*conf);
 945         struct rte_flow_item item;
 946         struct rte_flow_item_tcp tcp;
 947         struct rte_flow_item_tcp tcp_mask;
 948
 949         memset(&tcp, 0, sizeof(tcp));
 950         memset(&tcp_mask, 0, sizeof(tcp_mask));
 951         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 952                 /*
 953                  * The HW has no decrement operation, only increment operation.
 954                  * To simulate decrement X from Y using increment operation
 955                  * we need to add UINT32_MAX X times to Y.
 956                  * Each adding of UINT32_MAX decrements Y by 1.
 957                  */
 958                 value *= UINT32_MAX;
 959         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 960         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 961         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 962         item.spec = &tcp;
 963         item.mask = &tcp_mask;
 964         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 965                                              MLX5_MODIFICATION_TYPE_ADD, error);
 966 }
 967
 968 /**
 969  * Convert modify-header increment/decrement TCP Acknowledgment number
 970  * to DV specification.
 971  *
 972  * @param[in,out] resource
 973  *   Pointer to the modify-header resource.
 974  * @param[in] action
 975  *   Pointer to action specification.
 976  * @param[out] error
 977  *   Pointer to the error structure.
 978  *
 979  * @return
 980  *   0 on success, a negative errno value otherwise and rte_errno is set.
 981  */
 982 static int
 983 flow_dv_convert_action_modify_tcp_ack
 984                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 985                          const struct rte_flow_action *action,
 986                          struct rte_flow_error *error)
 987 {
 988         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 989         uint64_t value = rte_be_to_cpu_32(*conf);
 990         struct rte_flow_item item;
 991         struct rte_flow_item_tcp tcp;
 992         struct rte_flow_item_tcp tcp_mask;
 993
 994         memset(&tcp, 0, sizeof(tcp));
 995         memset(&tcp_mask, 0, sizeof(tcp_mask));
 996         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 997                 /*
 998                  * The HW has no decrement operation, only increment operation.
 999                  * To simulate decrement X from Y using increment operation
1000                  * we need to add UINT32_MAX X times to Y.
1001                  * Each adding of UINT32_MAX decrements Y by 1.
1002                  */
1003                 value *= UINT32_MAX;
1004         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
1005         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
1006         item.type = RTE_FLOW_ITEM_TYPE_TCP;
1007         item.spec = &tcp;
1008         item.mask = &tcp_mask;
1009         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
1010                                              MLX5_MODIFICATION_TYPE_ADD, error);
1011 }
1012
1013 static enum mlx5_modification_field reg_to_field[] = {
1014         [REG_NON] = MLX5_MODI_OUT_NONE,
1015         [REG_A] = MLX5_MODI_META_DATA_REG_A,
1016         [REG_B] = MLX5_MODI_META_DATA_REG_B,
1017         [REG_C_0] = MLX5_MODI_META_REG_C_0,
1018         [REG_C_1] = MLX5_MODI_META_REG_C_1,
1019         [REG_C_2] = MLX5_MODI_META_REG_C_2,
1020         [REG_C_3] = MLX5_MODI_META_REG_C_3,
1021         [REG_C_4] = MLX5_MODI_META_REG_C_4,
1022         [REG_C_5] = MLX5_MODI_META_REG_C_5,
1023         [REG_C_6] = MLX5_MODI_META_REG_C_6,
1024         [REG_C_7] = MLX5_MODI_META_REG_C_7,
1025 };
1026
1027 /**
1028  * Convert register set to DV specification.
1029  *
1030  * @param[in,out] resource
1031  *   Pointer to the modify-header resource.
1032  * @param[in] action
1033  *   Pointer to action specification.
1034  * @param[out] error
1035  *   Pointer to the error structure.
1036  *
1037  * @return
1038  *   0 on success, a negative errno value otherwise and rte_errno is set.
1039  */
1040 static int
1041 flow_dv_convert_action_set_reg
1042                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1043                          const struct rte_flow_action *action,
1044                          struct rte_flow_error *error)
1045 {
1046         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
1047         struct mlx5_modification_cmd *actions = resource->actions;
1048         uint32_t i = resource->actions_num;
1049
1050         if (i >= MLX5_MAX_MODIFY_NUM)
1051                 return rte_flow_error_set(error, EINVAL,
1052                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
1053                                           "too many items to modify");
1054         MLX5_ASSERT(conf->id != REG_NON);
1055         MLX5_ASSERT(conf->id < (enum modify_reg)RTE_DIM(reg_to_field));
1056         actions[i] = (struct mlx5_modification_cmd) {
1057                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1058                 .field = reg_to_field[conf->id],
1059                 .offset = conf->offset,
1060                 .length = conf->length,
1061         };
1062         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1063         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1064         ++i;
1065         resource->actions_num = i;
1066         return 0;
1067 }
1068
1069 /**
1070  * Convert SET_TAG action to DV specification.
1071  *
1072  * @param[in] dev
1073  *   Pointer to the rte_eth_dev structure.
1074  * @param[in,out] resource
1075  *   Pointer to the modify-header resource.
1076  * @param[in] conf
1077  *   Pointer to action specification.
1078  * @param[out] error
1079  *   Pointer to the error structure.
1080  *
1081  * @return
1082  *   0 on success, a negative errno value otherwise and rte_errno is set.
1083  */
1084 static int
1085 flow_dv_convert_action_set_tag
1086                         (struct rte_eth_dev *dev,
1087                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1088                          const struct rte_flow_action_set_tag *conf,
1089                          struct rte_flow_error *error)
1090 {
1091         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1092         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1093         struct rte_flow_item item = {
1094                 .spec = &data,
1095                 .mask = &mask,
1096         };
1097         struct field_modify_info reg_c_x[] = {
1098                 [1] = {0, 0, 0},
1099         };
1100         enum mlx5_modification_field reg_type;
1101         int ret;
1102
1103         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1104         if (ret < 0)
1105                 return ret;
1106         MLX5_ASSERT(ret != REG_NON);
1107         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1108         reg_type = reg_to_field[ret];
1109         MLX5_ASSERT(reg_type > 0);
1110         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1111         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1112                                              MLX5_MODIFICATION_TYPE_SET, error);
1113 }
1114
1115 /**
1116  * Convert internal COPY_REG action to DV specification.
1117  *
1118  * @param[in] dev
1119  *   Pointer to the rte_eth_dev structure.
1120  * @param[in,out] res
1121  *   Pointer to the modify-header resource.
1122  * @param[in] action
1123  *   Pointer to action specification.
1124  * @param[out] error
1125  *   Pointer to the error structure.
1126  *
1127  * @return
1128  *   0 on success, a negative errno value otherwise and rte_errno is set.
1129  */
1130 static int
1131 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1132                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1133                                  const struct rte_flow_action *action,
1134                                  struct rte_flow_error *error)
1135 {
1136         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1137         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1138         struct rte_flow_item item = {
1139                 .spec = NULL,
1140                 .mask = &mask,
1141         };
1142         struct field_modify_info reg_src[] = {
1143                 {4, 0, reg_to_field[conf->src]},
1144                 {0, 0, 0},
1145         };
1146         struct field_modify_info reg_dst = {
1147                 .offset = 0,
1148                 .id = reg_to_field[conf->dst],
1149         };
1150         /* Adjust reg_c[0] usage according to reported mask. */
1151         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1152                 struct mlx5_priv *priv = dev->data->dev_private;
1153                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1154
1155                 MLX5_ASSERT(reg_c0);
1156                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1157                 if (conf->dst == REG_C_0) {
1158                         /* Copy to reg_c[0], within mask only. */
1159                         reg_dst.offset = rte_bsf32(reg_c0);
1160                         /*
1161                          * Mask is ignoring the enianness, because
1162                          * there is no conversion in datapath.
1163                          */
1164 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1165                         /* Copy from destination lower bits to reg_c[0]. */
1166                         mask = reg_c0 >> reg_dst.offset;
1167 #else
1168                         /* Copy from destination upper bits to reg_c[0]. */
1169                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1170                                           rte_fls_u32(reg_c0));
1171 #endif
1172                 } else {
1173                         mask = rte_cpu_to_be_32(reg_c0);
1174 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1175                         /* Copy from reg_c[0] to destination lower bits. */
1176                         reg_dst.offset = 0;
1177 #else
1178                         /* Copy from reg_c[0] to destination upper bits. */
1179                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1180                                          (rte_fls_u32(reg_c0) -
1181                                           rte_bsf32(reg_c0));
1182 #endif
1183                 }
1184         }
1185         return flow_dv_convert_modify_action(&item,
1186                                              reg_src, &reg_dst, res,
1187                                              MLX5_MODIFICATION_TYPE_COPY,
1188                                              error);
1189 }
1190
1191 /**
1192  * Convert MARK action to DV specification. This routine is used
1193  * in extensive metadata only and requires metadata register to be
1194  * handled. In legacy mode hardware tag resource is engaged.
1195  *
1196  * @param[in] dev
1197  *   Pointer to the rte_eth_dev structure.
1198  * @param[in] conf
1199  *   Pointer to MARK action specification.
1200  * @param[in,out] resource
1201  *   Pointer to the modify-header resource.
1202  * @param[out] error
1203  *   Pointer to the error structure.
1204  *
1205  * @return
1206  *   0 on success, a negative errno value otherwise and rte_errno is set.
1207  */
1208 static int
1209 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1210                             const struct rte_flow_action_mark *conf,
1211                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1212                             struct rte_flow_error *error)
1213 {
1214         struct mlx5_priv *priv = dev->data->dev_private;
1215         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1216                                            priv->sh->dv_mark_mask);
1217         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1218         struct rte_flow_item item = {
1219                 .spec = &data,
1220                 .mask = &mask,
1221         };
1222         struct field_modify_info reg_c_x[] = {
1223                 [1] = {0, 0, 0},
1224         };
1225         int reg;
1226
1227         if (!mask)
1228                 return rte_flow_error_set(error, EINVAL,
1229                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1230                                           NULL, "zero mark action mask");
1231         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1232         if (reg < 0)
1233                 return reg;
1234         MLX5_ASSERT(reg > 0);
1235         if (reg == REG_C_0) {
1236                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1237                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1238
1239                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1240                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1241                 mask = rte_cpu_to_be_32(mask << shl_c0);
1242         }
1243         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1244         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1245                                              MLX5_MODIFICATION_TYPE_SET, error);
1246 }
1247
1248 /**
1249  * Get metadata register index for specified steering domain.
1250  *
1251  * @param[in] dev
1252  *   Pointer to the rte_eth_dev structure.
1253  * @param[in] attr
1254  *   Attributes of flow to determine steering domain.
1255  * @param[out] error
1256  *   Pointer to the error structure.
1257  *
1258  * @return
1259  *   positive index on success, a negative errno value otherwise
1260  *   and rte_errno is set.
1261  */
1262 static enum modify_reg
1263 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1264                          const struct rte_flow_attr *attr,
1265                          struct rte_flow_error *error)
1266 {
1267         int reg =
1268                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1269                                           MLX5_METADATA_FDB :
1270                                             attr->egress ?
1271                                             MLX5_METADATA_TX :
1272                                             MLX5_METADATA_RX, 0, error);
1273         if (reg < 0)
1274                 return rte_flow_error_set(error,
1275                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1276                                           NULL, "unavailable "
1277                                           "metadata register");
1278         return reg;
1279 }
1280
1281 /**
1282  * Convert SET_META action to DV specification.
1283  *
1284  * @param[in] dev
1285  *   Pointer to the rte_eth_dev structure.
1286  * @param[in,out] resource
1287  *   Pointer to the modify-header resource.
1288  * @param[in] attr
1289  *   Attributes of flow that includes this item.
1290  * @param[in] conf
1291  *   Pointer to action specification.
1292  * @param[out] error
1293  *   Pointer to the error structure.
1294  *
1295  * @return
1296  *   0 on success, a negative errno value otherwise and rte_errno is set.
1297  */
1298 static int
1299 flow_dv_convert_action_set_meta
1300                         (struct rte_eth_dev *dev,
1301                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1302                          const struct rte_flow_attr *attr,
1303                          const struct rte_flow_action_set_meta *conf,
1304                          struct rte_flow_error *error)
1305 {
1306         uint32_t mask = rte_cpu_to_be_32(conf->mask);
1307         uint32_t data = rte_cpu_to_be_32(conf->data) & mask;
1308         struct rte_flow_item item = {
1309                 .spec = &data,
1310                 .mask = &mask,
1311         };
1312         struct field_modify_info reg_c_x[] = {
1313                 [1] = {0, 0, 0},
1314         };
1315         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1316
1317         if (reg < 0)
1318                 return reg;
1319         MLX5_ASSERT(reg != REG_NON);
1320         if (reg == REG_C_0) {
1321                 struct mlx5_priv *priv = dev->data->dev_private;
1322                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1323                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1324
1325                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1326                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1327                 mask = rte_cpu_to_be_32(mask << shl_c0);
1328         }
1329         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1330         /* The routine expects parameters in memory as big-endian ones. */
1331         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1332                                              MLX5_MODIFICATION_TYPE_SET, error);
1333 }
1334
1335 /**
1336  * Convert modify-header set IPv4 DSCP action to DV specification.
1337  *
1338  * @param[in,out] resource
1339  *   Pointer to the modify-header resource.
1340  * @param[in] action
1341  *   Pointer to action specification.
1342  * @param[out] error
1343  *   Pointer to the error structure.
1344  *
1345  * @return
1346  *   0 on success, a negative errno value otherwise and rte_errno is set.
1347  */
1348 static int
1349 flow_dv_convert_action_modify_ipv4_dscp
1350                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1351                          const struct rte_flow_action *action,
1352                          struct rte_flow_error *error)
1353 {
1354         const struct rte_flow_action_set_dscp *conf =
1355                 (const struct rte_flow_action_set_dscp *)(action->conf);
1356         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1357         struct rte_flow_item_ipv4 ipv4;
1358         struct rte_flow_item_ipv4 ipv4_mask;
1359
1360         memset(&ipv4, 0, sizeof(ipv4));
1361         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1362         ipv4.hdr.type_of_service = conf->dscp;
1363         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1364         item.spec = &ipv4;
1365         item.mask = &ipv4_mask;
1366         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1367                                              MLX5_MODIFICATION_TYPE_SET, error);
1368 }
1369
1370 /**
1371  * Convert modify-header set IPv6 DSCP action to DV specification.
1372  *
1373  * @param[in,out] resource
1374  *   Pointer to the modify-header resource.
1375  * @param[in] action
1376  *   Pointer to action specification.
1377  * @param[out] error
1378  *   Pointer to the error structure.
1379  *
1380  * @return
1381  *   0 on success, a negative errno value otherwise and rte_errno is set.
1382  */
1383 static int
1384 flow_dv_convert_action_modify_ipv6_dscp
1385                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1386                          const struct rte_flow_action *action,
1387                          struct rte_flow_error *error)
1388 {
1389         const struct rte_flow_action_set_dscp *conf =
1390                 (const struct rte_flow_action_set_dscp *)(action->conf);
1391         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1392         struct rte_flow_item_ipv6 ipv6;
1393         struct rte_flow_item_ipv6 ipv6_mask;
1394
1395         memset(&ipv6, 0, sizeof(ipv6));
1396         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1397         /*
1398          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1399          * rdma-core only accept the DSCP bits byte aligned start from
1400          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1401          * bits in IPv6 case as rdma-core requires byte aligned value.
1402          */
1403         ipv6.hdr.vtc_flow = conf->dscp;
1404         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1405         item.spec = &ipv6;
1406         item.mask = &ipv6_mask;
1407         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1408                                              MLX5_MODIFICATION_TYPE_SET, error);
1409 }
1410
1411 static int
1412 mlx5_flow_item_field_width(struct mlx5_dev_config *config,
1413                            enum rte_flow_field_id field)
1414 {
1415         switch (field) {
1416         case RTE_FLOW_FIELD_START:
1417                 return 32;
1418         case RTE_FLOW_FIELD_MAC_DST:
1419         case RTE_FLOW_FIELD_MAC_SRC:
1420                 return 48;
1421         case RTE_FLOW_FIELD_VLAN_TYPE:
1422                 return 16;
1423         case RTE_FLOW_FIELD_VLAN_ID:
1424                 return 12;
1425         case RTE_FLOW_FIELD_MAC_TYPE:
1426                 return 16;
1427         case RTE_FLOW_FIELD_IPV4_DSCP:
1428                 return 6;
1429         case RTE_FLOW_FIELD_IPV4_TTL:
1430                 return 8;
1431         case RTE_FLOW_FIELD_IPV4_SRC:
1432         case RTE_FLOW_FIELD_IPV4_DST:
1433                 return 32;
1434         case RTE_FLOW_FIELD_IPV6_DSCP:
1435                 return 6;
1436         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1437                 return 8;
1438         case RTE_FLOW_FIELD_IPV6_SRC:
1439         case RTE_FLOW_FIELD_IPV6_DST:
1440                 return 128;
1441         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1442         case RTE_FLOW_FIELD_TCP_PORT_DST:
1443                 return 16;
1444         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1445         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1446                 return 32;
1447         case RTE_FLOW_FIELD_TCP_FLAGS:
1448                 return 9;
1449         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1450         case RTE_FLOW_FIELD_UDP_PORT_DST:
1451                 return 16;
1452         case RTE_FLOW_FIELD_VXLAN_VNI:
1453         case RTE_FLOW_FIELD_GENEVE_VNI:
1454                 return 24;
1455         case RTE_FLOW_FIELD_GTP_TEID:
1456         case RTE_FLOW_FIELD_TAG:
1457                 return 32;
1458         case RTE_FLOW_FIELD_MARK:
1459                 return 24;
1460         case RTE_FLOW_FIELD_META:
1461                 if (config->dv_xmeta_en == MLX5_XMETA_MODE_META16)
1462                         return 16;
1463                 else if (config->dv_xmeta_en == MLX5_XMETA_MODE_META32)
1464                         return 32;
1465                 else
1466                         return 0;
1467         case RTE_FLOW_FIELD_POINTER:
1468         case RTE_FLOW_FIELD_VALUE:
1469                 return 64;
1470         default:
1471                 MLX5_ASSERT(false);
1472         }
1473         return 0;
1474 }
1475
1476 static void
1477 mlx5_flow_field_id_to_modify_info
1478                 (const struct rte_flow_action_modify_data *data,
1479                  struct field_modify_info *info,
1480                  uint32_t *mask, uint32_t *value,
1481                  uint32_t width, uint32_t dst_width,
1482                  struct rte_eth_dev *dev,
1483                  const struct rte_flow_attr *attr,
1484                  struct rte_flow_error *error)
1485 {
1486         struct mlx5_priv *priv = dev->data->dev_private;
1487         struct mlx5_dev_config *config = &priv->config;
1488         uint32_t idx = 0;
1489         uint32_t off = 0;
1490         uint64_t val = 0;
1491         switch (data->field) {
1492         case RTE_FLOW_FIELD_START:
1493                 /* not supported yet */
1494                 MLX5_ASSERT(false);
1495                 break;
1496         case RTE_FLOW_FIELD_MAC_DST:
1497                 off = data->offset > 16 ? data->offset - 16 : 0;
1498                 if (mask) {
1499                         if (data->offset < 16) {
1500                                 info[idx] = (struct field_modify_info){2, 0,
1501                                                 MLX5_MODI_OUT_DMAC_15_0};
1502                                 if (width < 16) {
1503                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1504                                                                  (16 - width));
1505                                         width = 0;
1506                                 } else {
1507                                         mask[idx] = RTE_BE16(0xffff);
1508                                         width -= 16;
1509                                 }
1510                                 if (!width)
1511                                         break;
1512                                 ++idx;
1513                         }
1514                         info[idx] = (struct field_modify_info){4, 4 * idx,
1515                                                 MLX5_MODI_OUT_DMAC_47_16};
1516                         mask[idx] = rte_cpu_to_be_32((0xffffffff >>
1517                                                       (32 - width)) << off);
1518                 } else {
1519                         if (data->offset < 16)
1520                                 info[idx++] = (struct field_modify_info){2, 0,
1521                                                 MLX5_MODI_OUT_DMAC_15_0};
1522                         info[idx] = (struct field_modify_info){4, off,
1523                                                 MLX5_MODI_OUT_DMAC_47_16};
1524                 }
1525                 break;
1526         case RTE_FLOW_FIELD_MAC_SRC:
1527                 off = data->offset > 16 ? data->offset - 16 : 0;
1528                 if (mask) {
1529                         if (data->offset < 16) {
1530                                 info[idx] = (struct field_modify_info){2, 0,
1531                                                 MLX5_MODI_OUT_SMAC_15_0};
1532                                 if (width < 16) {
1533                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1534                                                                  (16 - width));
1535                                         width = 0;
1536                                 } else {
1537                                         mask[idx] = RTE_BE16(0xffff);
1538                                         width -= 16;
1539                                 }
1540                                 if (!width)
1541                                         break;
1542                                 ++idx;
1543                         }
1544                         info[idx] = (struct field_modify_info){4, 4 * idx,
1545                                                 MLX5_MODI_OUT_SMAC_47_16};
1546                         mask[idx] = rte_cpu_to_be_32((0xffffffff >>
1547                                                       (32 - width)) << off);
1548                 } else {
1549                         if (data->offset < 16)
1550                                 info[idx++] = (struct field_modify_info){2, 0,
1551                                                 MLX5_MODI_OUT_SMAC_15_0};
1552                         info[idx] = (struct field_modify_info){4, off,
1553                                                 MLX5_MODI_OUT_SMAC_47_16};
1554                 }
1555                 break;
1556         case RTE_FLOW_FIELD_VLAN_TYPE:
1557                 /* not supported yet */
1558                 break;
1559         case RTE_FLOW_FIELD_VLAN_ID:
1560                 info[idx] = (struct field_modify_info){2, 0,
1561                                         MLX5_MODI_OUT_FIRST_VID};
1562                 if (mask)
1563                         mask[idx] = rte_cpu_to_be_16(0x0fff >> (12 - width));
1564                 break;
1565         case RTE_FLOW_FIELD_MAC_TYPE:
1566                 info[idx] = (struct field_modify_info){2, 0,
1567                                         MLX5_MODI_OUT_ETHERTYPE};
1568                 if (mask)
1569                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1570                 break;
1571         case RTE_FLOW_FIELD_IPV4_DSCP:
1572                 info[idx] = (struct field_modify_info){1, 0,
1573                                         MLX5_MODI_OUT_IP_DSCP};
1574                 if (mask)
1575                         mask[idx] = 0x3f >> (6 - width);
1576                 break;
1577         case RTE_FLOW_FIELD_IPV4_TTL:
1578                 info[idx] = (struct field_modify_info){1, 0,
1579                                         MLX5_MODI_OUT_IPV4_TTL};
1580                 if (mask)
1581                         mask[idx] = 0xff >> (8 - width);
1582                 break;
1583         case RTE_FLOW_FIELD_IPV4_SRC:
1584                 info[idx] = (struct field_modify_info){4, 0,
1585                                         MLX5_MODI_OUT_SIPV4};
1586                 if (mask)
1587                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1588                                                      (32 - width));
1589                 break;
1590         case RTE_FLOW_FIELD_IPV4_DST:
1591                 info[idx] = (struct field_modify_info){4, 0,
1592                                         MLX5_MODI_OUT_DIPV4};
1593                 if (mask)
1594                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1595                                                      (32 - width));
1596                 break;
1597         case RTE_FLOW_FIELD_IPV6_DSCP:
1598                 info[idx] = (struct field_modify_info){1, 0,
1599                                         MLX5_MODI_OUT_IP_DSCP};
1600                 if (mask)
1601                         mask[idx] = 0x3f >> (6 - width);
1602                 break;
1603         case RTE_FLOW_FIELD_IPV6_HOPLIMIT:
1604                 info[idx] = (struct field_modify_info){1, 0,
1605                                         MLX5_MODI_OUT_IPV6_HOPLIMIT};
1606                 if (mask)
1607                         mask[idx] = 0xff >> (8 - width);
1608                 break;
1609         case RTE_FLOW_FIELD_IPV6_SRC:
1610                 if (mask) {
1611                         if (data->offset < 32) {
1612                                 info[idx] = (struct field_modify_info){4,
1613                                                 4 * idx,
1614                                                 MLX5_MODI_OUT_SIPV6_31_0};
1615                                 if (width < 32) {
1616                                         mask[idx] =
1617                                                 rte_cpu_to_be_32(0xffffffff >>
1618                                                                  (32 - width));
1619                                         width = 0;
1620                                 } else {
1621                                         mask[idx] = RTE_BE32(0xffffffff);
1622                                         width -= 32;
1623                                 }
1624                                 if (!width)
1625                                         break;
1626                                 ++idx;
1627                         }
1628                         if (data->offset < 64) {
1629                                 info[idx] = (struct field_modify_info){4,
1630                                                 4 * idx,
1631                                                 MLX5_MODI_OUT_SIPV6_63_32};
1632                                 if (width < 32) {
1633                                         mask[idx] =
1634                                                 rte_cpu_to_be_32(0xffffffff >>
1635                                                                  (32 - width));
1636                                         width = 0;
1637                                 } else {
1638                                         mask[idx] = RTE_BE32(0xffffffff);
1639                                         width -= 32;
1640                                 }
1641                                 if (!width)
1642                                         break;
1643                                 ++idx;
1644                         }
1645                         if (data->offset < 96) {
1646                                 info[idx] = (struct field_modify_info){4,
1647                                                 4 * idx,
1648                                                 MLX5_MODI_OUT_SIPV6_95_64};
1649                                 if (width < 32) {
1650                                         mask[idx] =
1651                                                 rte_cpu_to_be_32(0xffffffff >>
1652                                                                  (32 - width));
1653                                         width = 0;
1654                                 } else {
1655                                         mask[idx] = RTE_BE32(0xffffffff);
1656                                         width -= 32;
1657                                 }
1658                                 if (!width)
1659                                         break;
1660                                 ++idx;
1661                         }
1662                         info[idx] = (struct field_modify_info){4, 4 * idx,
1663                                                 MLX5_MODI_OUT_SIPV6_127_96};
1664                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1665                                                      (32 - width));
1666                 } else {
1667                         if (data->offset < 32)
1668                                 info[idx++] = (struct field_modify_info){4, 0,
1669                                                 MLX5_MODI_OUT_SIPV6_31_0};
1670                         if (data->offset < 64)
1671                                 info[idx++] = (struct field_modify_info){4, 0,
1672                                                 MLX5_MODI_OUT_SIPV6_63_32};
1673                         if (data->offset < 96)
1674                                 info[idx++] = (struct field_modify_info){4, 0,
1675                                                 MLX5_MODI_OUT_SIPV6_95_64};
1676                         if (data->offset < 128)
1677                                 info[idx++] = (struct field_modify_info){4, 0,
1678                                                 MLX5_MODI_OUT_SIPV6_127_96};
1679                 }
1680                 break;
1681         case RTE_FLOW_FIELD_IPV6_DST:
1682                 if (mask) {
1683                         if (data->offset < 32) {
1684                                 info[idx] = (struct field_modify_info){4,
1685                                                 4 * idx,
1686                                                 MLX5_MODI_OUT_DIPV6_31_0};
1687                                 if (width < 32) {
1688                                         mask[idx] =
1689                                                 rte_cpu_to_be_32(0xffffffff >>
1690                                                                  (32 - width));
1691                                         width = 0;
1692                                 } else {
1693                                         mask[idx] = RTE_BE32(0xffffffff);
1694                                         width -= 32;
1695                                 }
1696                                 if (!width)
1697                                         break;
1698                                 ++idx;
1699                         }
1700                         if (data->offset < 64) {
1701                                 info[idx] = (struct field_modify_info){4,
1702                                                 4 * idx,
1703                                                 MLX5_MODI_OUT_DIPV6_63_32};
1704                                 if (width < 32) {
1705                                         mask[idx] =
1706                                                 rte_cpu_to_be_32(0xffffffff >>
1707                                                                  (32 - width));
1708                                         width = 0;
1709                                 } else {
1710                                         mask[idx] = RTE_BE32(0xffffffff);
1711                                         width -= 32;
1712                                 }
1713                                 if (!width)
1714                                         break;
1715                                 ++idx;
1716                         }
1717                         if (data->offset < 96) {
1718                                 info[idx] = (struct field_modify_info){4,
1719                                                 4 * idx,
1720                                                 MLX5_MODI_OUT_DIPV6_95_64};
1721                                 if (width < 32) {
1722                                         mask[idx] =
1723                                                 rte_cpu_to_be_32(0xffffffff >>
1724                                                                  (32 - width));
1725                                         width = 0;
1726                                 } else {
1727                                         mask[idx] = RTE_BE32(0xffffffff);
1728                                         width -= 32;
1729                                 }
1730                                 if (!width)
1731                                         break;
1732                                 ++idx;
1733                         }
1734                         info[idx] = (struct field_modify_info){4, 4 * idx,
1735                                                 MLX5_MODI_OUT_DIPV6_127_96};
1736                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1737                                                      (32 - width));
1738                 } else {
1739                         if (data->offset < 32)
1740                                 info[idx++] = (struct field_modify_info){4, 0,
1741                                                 MLX5_MODI_OUT_DIPV6_31_0};
1742                         if (data->offset < 64)
1743                                 info[idx++] = (struct field_modify_info){4, 0,
1744                                                 MLX5_MODI_OUT_DIPV6_63_32};
1745                         if (data->offset < 96)
1746                                 info[idx++] = (struct field_modify_info){4, 0,
1747                                                 MLX5_MODI_OUT_DIPV6_95_64};
1748                         if (data->offset < 128)
1749                                 info[idx++] = (struct field_modify_info){4, 0,
1750                                                 MLX5_MODI_OUT_DIPV6_127_96};
1751                 }
1752                 break;
1753         case RTE_FLOW_FIELD_TCP_PORT_SRC:
1754                 info[idx] = (struct field_modify_info){2, 0,
1755                                         MLX5_MODI_OUT_TCP_SPORT};
1756                 if (mask)
1757                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1758                 break;
1759         case RTE_FLOW_FIELD_TCP_PORT_DST:
1760                 info[idx] = (struct field_modify_info){2, 0,
1761                                         MLX5_MODI_OUT_TCP_DPORT};
1762                 if (mask)
1763                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1764                 break;
1765         case RTE_FLOW_FIELD_TCP_SEQ_NUM:
1766                 info[idx] = (struct field_modify_info){4, 0,
1767                                         MLX5_MODI_OUT_TCP_SEQ_NUM};
1768                 if (mask)
1769                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1770                                                      (32 - width));
1771                 break;
1772         case RTE_FLOW_FIELD_TCP_ACK_NUM:
1773                 info[idx] = (struct field_modify_info){4, 0,
1774                                         MLX5_MODI_OUT_TCP_ACK_NUM};
1775                 if (mask)
1776                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1777                                                      (32 - width));
1778                 break;
1779         case RTE_FLOW_FIELD_TCP_FLAGS:
1780                 info[idx] = (struct field_modify_info){2, 0,
1781                                         MLX5_MODI_OUT_TCP_FLAGS};
1782                 if (mask)
1783                         mask[idx] = rte_cpu_to_be_16(0x1ff >> (9 - width));
1784                 break;
1785         case RTE_FLOW_FIELD_UDP_PORT_SRC:
1786                 info[idx] = (struct field_modify_info){2, 0,
1787                                         MLX5_MODI_OUT_UDP_SPORT};
1788                 if (mask)
1789                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1790                 break;
1791         case RTE_FLOW_FIELD_UDP_PORT_DST:
1792                 info[idx] = (struct field_modify_info){2, 0,
1793                                         MLX5_MODI_OUT_UDP_DPORT};
1794                 if (mask)
1795                         mask[idx] = rte_cpu_to_be_16(0xffff >> (16 - width));
1796                 break;
1797         case RTE_FLOW_FIELD_VXLAN_VNI:
1798                 /* not supported yet */
1799                 break;
1800         case RTE_FLOW_FIELD_GENEVE_VNI:
1801                 /* not supported yet*/
1802                 break;
1803         case RTE_FLOW_FIELD_GTP_TEID:
1804                 info[idx] = (struct field_modify_info){4, 0,
1805                                         MLX5_MODI_GTP_TEID};
1806                 if (mask)
1807                         mask[idx] = rte_cpu_to_be_32(0xffffffff >>
1808                                                      (32 - width));
1809                 break;
1810         case RTE_FLOW_FIELD_TAG:
1811                 {
1812                         int reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG,
1813                                                    data->level, error);
1814                         if (reg < 0)
1815                                 return;
1816                         MLX5_ASSERT(reg != REG_NON);
1817                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1818                         info[idx] = (struct field_modify_info){4, 0,
1819                                                 reg_to_field[reg]};
1820                         if (mask)
1821                                 mask[idx] =
1822                                         rte_cpu_to_be_32(0xffffffff >>
1823                                                          (32 - width));
1824                 }
1825                 break;
1826         case RTE_FLOW_FIELD_MARK:
1827                 {
1828                         int reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK,
1829                                                        0, error);
1830                         if (reg < 0)
1831                                 return;
1832                         MLX5_ASSERT(reg != REG_NON);
1833                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1834                         info[idx] = (struct field_modify_info){4, 0,
1835                                                 reg_to_field[reg]};
1836                         if (mask)
1837                                 mask[idx] =
1838                                         rte_cpu_to_be_32(0xffffffff >>
1839                                                          (32 - width));
1840                 }
1841                 break;
1842         case RTE_FLOW_FIELD_META:
1843                 {
1844                         unsigned int xmeta = config->dv_xmeta_en;
1845                         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1846                         if (reg < 0)
1847                                 return;
1848                         MLX5_ASSERT(reg != REG_NON);
1849                         MLX5_ASSERT((unsigned int)reg < RTE_DIM(reg_to_field));
1850                         if (xmeta == MLX5_XMETA_MODE_META16) {
1851                                 info[idx] = (struct field_modify_info){2, 0,
1852                                                         reg_to_field[reg]};
1853                                 if (mask)
1854                                         mask[idx] = rte_cpu_to_be_16(0xffff >>
1855                                                                 (16 - width));
1856                         } else if (xmeta == MLX5_XMETA_MODE_META32) {
1857                                 info[idx] = (struct field_modify_info){4, 0,
1858                                                         reg_to_field[reg]};
1859                                 if (mask)
1860                                         mask[idx] =
1861                                                 rte_cpu_to_be_32(0xffffffff >>
1862                                                                 (32 - width));
1863                         } else {
1864                                 MLX5_ASSERT(false);
1865                         }
1866                 }
1867                 break;
1868         case RTE_FLOW_FIELD_POINTER:
1869         case RTE_FLOW_FIELD_VALUE:
1870                 if (data->field == RTE_FLOW_FIELD_POINTER)
1871                         memcpy(&val, (void *)(uintptr_t)data->value,
1872                                sizeof(uint64_t));
1873                 else
1874                         val = data->value;
1875                 for (idx = 0; idx < MLX5_ACT_MAX_MOD_FIELDS; idx++) {
1876                         if (mask[idx]) {
1877                                 if (dst_width == 48) {
1878                                         /*special case for MAC addresses */
1879                                         value[idx] = rte_cpu_to_be_16(val);
1880                                         val >>= 16;
1881                                         dst_width -= 16;
1882                                 } else if (dst_width > 16) {
1883                                         value[idx] = rte_cpu_to_be_32(val);
1884                                         val >>= 32;
1885                                 } else if (dst_width > 8) {
1886                                         value[idx] = rte_cpu_to_be_16(val);
1887                                         val >>= 16;
1888                                 } else {
1889                                         value[idx] = (uint8_t)val;
1890                                         val >>= 8;
1891                                 }
1892                                 if (!val)
1893                                         break;
1894                         }
1895                 }
1896                 break;
1897         default:
1898                 MLX5_ASSERT(false);
1899                 break;
1900         }
1901 }
1902
1903 /**
1904  * Convert modify_field action to DV specification.
1905  *
1906  * @param[in] dev
1907  *   Pointer to the rte_eth_dev structure.
1908  * @param[in,out] resource
1909  *   Pointer to the modify-header resource.
1910  * @param[in] action
1911  *   Pointer to action specification.
1912  * @param[in] attr
1913  *   Attributes of flow that includes this item.
1914  * @param[out] error
1915  *   Pointer to the error structure.
1916  *
1917  * @return
1918  *   0 on success, a negative errno value otherwise and rte_errno is set.
1919  */
1920 static int
1921 flow_dv_convert_action_modify_field
1922                         (struct rte_eth_dev *dev,
1923                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1924                          const struct rte_flow_action *action,
1925                          const struct rte_flow_attr *attr,
1926                          struct rte_flow_error *error)
1927 {
1928         struct mlx5_priv *priv = dev->data->dev_private;
1929         struct mlx5_dev_config *config = &priv->config;
1930         const struct rte_flow_action_modify_field *conf =
1931                 (const struct rte_flow_action_modify_field *)(action->conf);
1932         struct rte_flow_item item;
1933         struct field_modify_info field[MLX5_ACT_MAX_MOD_FIELDS] = {
1934                                                                 {0, 0, 0} };
1935         struct field_modify_info dcopy[MLX5_ACT_MAX_MOD_FIELDS] = {
1936                                                                 {0, 0, 0} };
1937         uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0};
1938         uint32_t value[MLX5_ACT_MAX_MOD_FIELDS] = {0, 0, 0, 0, 0};
1939         uint32_t type;
1940         uint32_t dst_width = mlx5_flow_item_field_width(config,
1941                                                         conf->dst.field);
1942
1943         if (conf->src.field == RTE_FLOW_FIELD_POINTER ||
1944                 conf->src.field == RTE_FLOW_FIELD_VALUE) {
1945                 type = MLX5_MODIFICATION_TYPE_SET;
1946                 /** For SET fill the destination field (field) first. */
1947                 mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask,
1948                         value, conf->width, dst_width, dev, attr, error);
1949                 /** Then copy immediate value from source as per mask. */
1950                 mlx5_flow_field_id_to_modify_info(&conf->src, dcopy, mask,
1951                         value, conf->width, dst_width, dev, attr, error);
1952                 item.spec = &value;
1953         } else {
1954                 type = MLX5_MODIFICATION_TYPE_COPY;
1955                 /** For COPY fill the destination field (dcopy) without mask. */
1956                 mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL,
1957                         value, conf->width, dst_width, dev, attr, error);
1958                 /** Then construct the source field (field) with mask. */
1959                 mlx5_flow_field_id_to_modify_info(&conf->src, field, mask,
1960                         value, conf->width, dst_width, dev, attr, error);
1961         }
1962         item.mask = &mask;
1963         return flow_dv_convert_modify_action(&item,
1964                         field, dcopy, resource, type, error);
1965 }
1966
1967 /**
1968  * Validate MARK item.
1969  *
1970  * @param[in] dev
1971  *   Pointer to the rte_eth_dev structure.
1972  * @param[in] item
1973  *   Item specification.
1974  * @param[in] attr
1975  *   Attributes of flow that includes this item.
1976  * @param[out] error
1977  *   Pointer to error structure.
1978  *
1979  * @return
1980  *   0 on success, a negative errno value otherwise and rte_errno is set.
1981  */
1982 static int
1983 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1984                            const struct rte_flow_item *item,
1985                            const struct rte_flow_attr *attr __rte_unused,
1986                            struct rte_flow_error *error)
1987 {
1988         struct mlx5_priv *priv = dev->data->dev_private;
1989         struct mlx5_dev_config *config = &priv->config;
1990         const struct rte_flow_item_mark *spec = item->spec;
1991         const struct rte_flow_item_mark *mask = item->mask;
1992         const struct rte_flow_item_mark nic_mask = {
1993                 .id = priv->sh->dv_mark_mask,
1994         };
1995         int ret;
1996
1997         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1998                 return rte_flow_error_set(error, ENOTSUP,
1999                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2000                                           "extended metadata feature"
2001                                           " isn't enabled");
2002         if (!mlx5_flow_ext_mreg_supported(dev))
2003                 return rte_flow_error_set(error, ENOTSUP,
2004                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2005                                           "extended metadata register"
2006                                           " isn't supported");
2007         if (!nic_mask.id)
2008                 return rte_flow_error_set(error, ENOTSUP,
2009                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2010                                           "extended metadata register"
2011                                           " isn't available");
2012         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2013         if (ret < 0)
2014                 return ret;
2015         if (!spec)
2016                 return rte_flow_error_set(error, EINVAL,
2017                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2018                                           item->spec,
2019                                           "data cannot be empty");
2020         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
2021                 return rte_flow_error_set(error, EINVAL,
2022                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2023                                           &spec->id,
2024                                           "mark id exceeds the limit");
2025         if (!mask)
2026                 mask = &nic_mask;
2027         if (!mask->id)
2028                 return rte_flow_error_set(error, EINVAL,
2029                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2030                                         "mask cannot be zero");
2031
2032         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2033                                         (const uint8_t *)&nic_mask,
2034                                         sizeof(struct rte_flow_item_mark),
2035                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2036         if (ret < 0)
2037                 return ret;
2038         return 0;
2039 }
2040
2041 /**
2042  * Validate META item.
2043  *
2044  * @param[in] dev
2045  *   Pointer to the rte_eth_dev structure.
2046  * @param[in] item
2047  *   Item specification.
2048  * @param[in] attr
2049  *   Attributes of flow that includes this item.
2050  * @param[out] error
2051  *   Pointer to error structure.
2052  *
2053  * @return
2054  *   0 on success, a negative errno value otherwise and rte_errno is set.
2055  */
2056 static int
2057 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
2058                            const struct rte_flow_item *item,
2059                            const struct rte_flow_attr *attr,
2060                            struct rte_flow_error *error)
2061 {
2062         struct mlx5_priv *priv = dev->data->dev_private;
2063         struct mlx5_dev_config *config = &priv->config;
2064         const struct rte_flow_item_meta *spec = item->spec;
2065         const struct rte_flow_item_meta *mask = item->mask;
2066         struct rte_flow_item_meta nic_mask = {
2067                 .data = UINT32_MAX
2068         };
2069         int reg;
2070         int ret;
2071
2072         if (!spec)
2073                 return rte_flow_error_set(error, EINVAL,
2074                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2075                                           item->spec,
2076                                           "data cannot be empty");
2077         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
2078                 if (!mlx5_flow_ext_mreg_supported(dev))
2079                         return rte_flow_error_set(error, ENOTSUP,
2080                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2081                                           "extended metadata register"
2082                                           " isn't supported");
2083                 reg = flow_dv_get_metadata_reg(dev, attr, error);
2084                 if (reg < 0)
2085                         return reg;
2086                 if (reg == REG_NON)
2087                         return rte_flow_error_set(error, ENOTSUP,
2088                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2089                                         "unavalable extended metadata register");
2090                 if (reg == REG_B)
2091                         return rte_flow_error_set(error, ENOTSUP,
2092                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2093                                           "match on reg_b "
2094                                           "isn't supported");
2095                 if (reg != REG_A)
2096                         nic_mask.data = priv->sh->dv_meta_mask;
2097         } else {
2098                 if (attr->transfer)
2099                         return rte_flow_error_set(error, ENOTSUP,
2100                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2101                                         "extended metadata feature "
2102                                         "should be enabled when "
2103                                         "meta item is requested "
2104                                         "with e-switch mode ");
2105                 if (attr->ingress)
2106                         return rte_flow_error_set(error, ENOTSUP,
2107                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
2108                                         "match on metadata for ingress "
2109                                         "is not supported in legacy "
2110                                         "metadata mode");
2111         }
2112         if (!mask)
2113                 mask = &rte_flow_item_meta_mask;
2114         if (!mask->data)
2115                 return rte_flow_error_set(error, EINVAL,
2116                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2117                                         "mask cannot be zero");
2118
2119         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2120                                         (const uint8_t *)&nic_mask,
2121                                         sizeof(struct rte_flow_item_meta),
2122                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2123         return ret;
2124 }
2125
2126 /**
2127  * Validate TAG item.
2128  *
2129  * @param[in] dev
2130  *   Pointer to the rte_eth_dev structure.
2131  * @param[in] item
2132  *   Item specification.
2133  * @param[in] attr
2134  *   Attributes of flow that includes this item.
2135  * @param[out] error
2136  *   Pointer to error structure.
2137  *
2138  * @return
2139  *   0 on success, a negative errno value otherwise and rte_errno is set.
2140  */
2141 static int
2142 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
2143                           const struct rte_flow_item *item,
2144                           const struct rte_flow_attr *attr __rte_unused,
2145                           struct rte_flow_error *error)
2146 {
2147         const struct rte_flow_item_tag *spec = item->spec;
2148         const struct rte_flow_item_tag *mask = item->mask;
2149         const struct rte_flow_item_tag nic_mask = {
2150                 .data = RTE_BE32(UINT32_MAX),
2151                 .index = 0xff,
2152         };
2153         int ret;
2154
2155         if (!mlx5_flow_ext_mreg_supported(dev))
2156                 return rte_flow_error_set(error, ENOTSUP,
2157                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2158                                           "extensive metadata register"
2159                                           " isn't supported");
2160         if (!spec)
2161                 return rte_flow_error_set(error, EINVAL,
2162                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
2163                                           item->spec,
2164                                           "data cannot be empty");
2165         if (!mask)
2166                 mask = &rte_flow_item_tag_mask;
2167         if (!mask->data)
2168                 return rte_flow_error_set(error, EINVAL,
2169                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2170                                         "mask cannot be zero");
2171
2172         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2173                                         (const uint8_t *)&nic_mask,
2174                                         sizeof(struct rte_flow_item_tag),
2175                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2176         if (ret < 0)
2177                 return ret;
2178         if (mask->index != 0xff)
2179                 return rte_flow_error_set(error, EINVAL,
2180                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
2181                                           "partial mask for tag index"
2182                                           " is not supported");
2183         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
2184         if (ret < 0)
2185                 return ret;
2186         MLX5_ASSERT(ret != REG_NON);
2187         return 0;
2188 }
2189
2190 /**
2191  * Validate vport item.
2192  *
2193  * @param[in] dev
2194  *   Pointer to the rte_eth_dev structure.
2195  * @param[in] item
2196  *   Item specification.
2197  * @param[in] attr
2198  *   Attributes of flow that includes this item.
2199  * @param[in] item_flags
2200  *   Bit-fields that holds the items detected until now.
2201  * @param[out] error
2202  *   Pointer to error structure.
2203  *
2204  * @return
2205  *   0 on success, a negative errno value otherwise and rte_errno is set.
2206  */
2207 static int
2208 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
2209                               const struct rte_flow_item *item,
2210                               const struct rte_flow_attr *attr,
2211                               uint64_t item_flags,
2212                               struct rte_flow_error *error)
2213 {
2214         const struct rte_flow_item_port_id *spec = item->spec;
2215         const struct rte_flow_item_port_id *mask = item->mask;
2216         const struct rte_flow_item_port_id switch_mask = {
2217                         .id = 0xffffffff,
2218         };
2219         struct mlx5_priv *esw_priv;
2220         struct mlx5_priv *dev_priv;
2221         int ret;
2222
2223         if (!attr->transfer)
2224                 return rte_flow_error_set(error, EINVAL,
2225                                           RTE_FLOW_ERROR_TYPE_ITEM,
2226                                           NULL,
2227                                           "match on port id is valid only"
2228                                           " when transfer flag is enabled");
2229         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
2230                 return rte_flow_error_set(error, ENOTSUP,
2231                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2232                                           "multiple source ports are not"
2233                                           " supported");
2234         if (!mask)
2235                 mask = &switch_mask;
2236         if (mask->id != 0xffffffff)
2237                 return rte_flow_error_set(error, ENOTSUP,
2238                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2239                                            mask,
2240                                            "no support for partial mask on"
2241                                            " \"id\" field");
2242         ret = mlx5_flow_item_acceptable
2243                                 (item, (const uint8_t *)mask,
2244                                  (const uint8_t *)&rte_flow_item_port_id_mask,
2245                                  sizeof(struct rte_flow_item_port_id),
2246                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2247         if (ret)
2248                 return ret;
2249         if (!spec)
2250                 return 0;
2251         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
2252         if (!esw_priv)
2253                 return rte_flow_error_set(error, rte_errno,
2254                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2255                                           "failed to obtain E-Switch info for"
2256                                           " port");
2257         dev_priv = mlx5_dev_to_eswitch_info(dev);
2258         if (!dev_priv)
2259                 return rte_flow_error_set(error, rte_errno,
2260                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2261                                           NULL,
2262                                           "failed to obtain E-Switch info");
2263         if (esw_priv->domain_id != dev_priv->domain_id)
2264                 return rte_flow_error_set(error, EINVAL,
2265                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
2266                                           "cannot match on a port from a"
2267                                           " different E-Switch");
2268         return 0;
2269 }
2270
2271 /**
2272  * Validate VLAN item.
2273  *
2274  * @param[in] item
2275  *   Item specification.
2276  * @param[in] item_flags
2277  *   Bit-fields that holds the items detected until now.
2278  * @param[in] dev
2279  *   Ethernet device flow is being created on.
2280  * @param[out] error
2281  *   Pointer to error structure.
2282  *
2283  * @return
2284  *   0 on success, a negative errno value otherwise and rte_errno is set.
2285  */
2286 static int
2287 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
2288                            uint64_t item_flags,
2289                            struct rte_eth_dev *dev,
2290                            struct rte_flow_error *error)
2291 {
2292         const struct rte_flow_item_vlan *mask = item->mask;
2293         const struct rte_flow_item_vlan nic_mask = {
2294                 .tci = RTE_BE16(UINT16_MAX),
2295                 .inner_type = RTE_BE16(UINT16_MAX),
2296                 .has_more_vlan = 1,
2297         };
2298         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2299         int ret;
2300         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
2301                                         MLX5_FLOW_LAYER_INNER_L4) :
2302                                        (MLX5_FLOW_LAYER_OUTER_L3 |
2303                                         MLX5_FLOW_LAYER_OUTER_L4);
2304         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
2305                                         MLX5_FLOW_LAYER_OUTER_VLAN;
2306
2307         if (item_flags & vlanm)
2308                 return rte_flow_error_set(error, EINVAL,
2309                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2310                                           "multiple VLAN layers not supported");
2311         else if ((item_flags & l34m) != 0)
2312                 return rte_flow_error_set(error, EINVAL,
2313                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2314                                           "VLAN cannot follow L3/L4 layer");
2315         if (!mask)
2316                 mask = &rte_flow_item_vlan_mask;
2317         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2318                                         (const uint8_t *)&nic_mask,
2319                                         sizeof(struct rte_flow_item_vlan),
2320                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2321         if (ret)
2322                 return ret;
2323         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
2324                 struct mlx5_priv *priv = dev->data->dev_private;
2325
2326                 if (priv->vmwa_context) {
2327                         /*
2328                          * Non-NULL context means we have a virtual machine
2329                          * and SR-IOV enabled, we have to create VLAN interface
2330                          * to make hypervisor to setup E-Switch vport
2331                          * context correctly. We avoid creating the multiple
2332                          * VLAN interfaces, so we cannot support VLAN tag mask.
2333                          */
2334                         return rte_flow_error_set(error, EINVAL,
2335                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2336                                                   item,
2337                                                   "VLAN tag mask is not"
2338                                                   " supported in virtual"
2339                                                   " environment");
2340                 }
2341         }
2342         return 0;
2343 }
2344
2345 /*
2346  * GTP flags are contained in 1 byte of the format:
2347  * -------------------------------------------
2348  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
2349  * |-----------------------------------------|
2350  * | value | Version | PT | Res | E | S | PN |
2351  * -------------------------------------------
2352  *
2353  * Matching is supported only for GTP flags E, S, PN.
2354  */
2355 #define MLX5_GTP_FLAGS_MASK     0x07
2356
2357 /**
2358  * Validate GTP item.
2359  *
2360  * @param[in] dev
2361  *   Pointer to the rte_eth_dev structure.
2362  * @param[in] item
2363  *   Item specification.
2364  * @param[in] item_flags
2365  *   Bit-fields that holds the items detected until now.
2366  * @param[out] error
2367  *   Pointer to error structure.
2368  *
2369  * @return
2370  *   0 on success, a negative errno value otherwise and rte_errno is set.
2371  */
2372 static int
2373 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
2374                           const struct rte_flow_item *item,
2375                           uint64_t item_flags,
2376                           struct rte_flow_error *error)
2377 {
2378         struct mlx5_priv *priv = dev->data->dev_private;
2379         const struct rte_flow_item_gtp *spec = item->spec;
2380         const struct rte_flow_item_gtp *mask = item->mask;
2381         const struct rte_flow_item_gtp nic_mask = {
2382                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
2383                 .msg_type = 0xff,
2384                 .teid = RTE_BE32(0xffffffff),
2385         };
2386
2387         if (!priv->config.hca_attr.tunnel_stateless_gtp)
2388                 return rte_flow_error_set(error, ENOTSUP,
2389                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2390                                           "GTP support is not enabled");
2391         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
2392                 return rte_flow_error_set(error, ENOTSUP,
2393                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2394                                           "multiple tunnel layers not"
2395                                           " supported");
2396         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
2397                 return rte_flow_error_set(error, EINVAL,
2398                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2399                                           "no outer UDP layer found");
2400         if (!mask)
2401                 mask = &rte_flow_item_gtp_mask;
2402         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
2403                 return rte_flow_error_set(error, ENOTSUP,
2404                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2405                                           "Match is supported for GTP"
2406                                           " flags only");
2407         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2408                                          (const uint8_t *)&nic_mask,
2409                                          sizeof(struct rte_flow_item_gtp),
2410                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2411 }
2412
2413 /**
2414  * Validate GTP PSC item.
2415  *
2416  * @param[in] item
2417  *   Item specification.
2418  * @param[in] last_item
2419  *   Previous validated item in the pattern items.
2420  * @param[in] gtp_item
2421  *   Previous GTP item specification.
2422  * @param[in] attr
2423  *   Pointer to flow attributes.
2424  * @param[out] error
2425  *   Pointer to error structure.
2426  *
2427  * @return
2428  *   0 on success, a negative errno value otherwise and rte_errno is set.
2429  */
2430 static int
2431 flow_dv_validate_item_gtp_psc(const struct rte_flow_item *item,
2432                               uint64_t last_item,
2433                               const struct rte_flow_item *gtp_item,
2434                               const struct rte_flow_attr *attr,
2435                               struct rte_flow_error *error)
2436 {
2437         const struct rte_flow_item_gtp *gtp_spec;
2438         const struct rte_flow_item_gtp *gtp_mask;
2439         const struct rte_flow_item_gtp_psc *spec;
2440         const struct rte_flow_item_gtp_psc *mask;
2441         const struct rte_flow_item_gtp_psc nic_mask = {
2442                 .pdu_type = 0xFF,
2443                 .qfi = 0xFF,
2444         };
2445
2446         if (!gtp_item || !(last_item & MLX5_FLOW_LAYER_GTP))
2447                 return rte_flow_error_set
2448                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2449                          "GTP PSC item must be preceded with GTP item");
2450         gtp_spec = gtp_item->spec;
2451         gtp_mask = gtp_item->mask ? gtp_item->mask : &rte_flow_item_gtp_mask;
2452         /* GTP spec and E flag is requested to match zero. */
2453         if (gtp_spec &&
2454                 (gtp_mask->v_pt_rsv_flags &
2455                 ~gtp_spec->v_pt_rsv_flags & MLX5_GTP_EXT_HEADER_FLAG))
2456                 return rte_flow_error_set
2457                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2458                          "GTP E flag must be 1 to match GTP PSC");
2459         /* Check the flow is not created in group zero. */
2460         if (!attr->transfer && !attr->group)
2461                 return rte_flow_error_set
2462                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2463                          "GTP PSC is not supported for group 0");
2464         /* GTP spec is here and E flag is requested to match zero. */
2465         if (!item->spec)
2466                 return 0;
2467         spec = item->spec;
2468         mask = item->mask ? item->mask : &rte_flow_item_gtp_psc_mask;
2469         if (spec->pdu_type > MLX5_GTP_EXT_MAX_PDU_TYPE)
2470                 return rte_flow_error_set
2471                         (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item,
2472                          "PDU type should be smaller than 16");
2473         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
2474                                          (const uint8_t *)&nic_mask,
2475                                          sizeof(struct rte_flow_item_gtp_psc),
2476                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
2477 }
2478
2479 /**
2480  * Validate IPV4 item.
2481  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
2482  * add specific validation of fragment_offset field,
2483  *
2484  * @param[in] item
2485  *   Item specification.
2486  * @param[in] item_flags
2487  *   Bit-fields that holds the items detected until now.
2488  * @param[out] error
2489  *   Pointer to error structure.
2490  *
2491  * @return
2492  *   0 on success, a negative errno value otherwise and rte_errno is set.
2493  */
2494 static int
2495 flow_dv_validate_item_ipv4(struct rte_eth_dev *dev,
2496                            const struct rte_flow_item *item,
2497                            uint64_t item_flags, uint64_t last_item,
2498                            uint16_t ether_type, struct rte_flow_error *error)
2499 {
2500         int ret;
2501         struct mlx5_priv *priv = dev->data->dev_private;
2502         const struct rte_flow_item_ipv4 *spec = item->spec;
2503         const struct rte_flow_item_ipv4 *last = item->last;
2504         const struct rte_flow_item_ipv4 *mask = item->mask;
2505         rte_be16_t fragment_offset_spec = 0;
2506         rte_be16_t fragment_offset_last = 0;
2507         struct rte_flow_item_ipv4 nic_ipv4_mask = {
2508                 .hdr = {
2509                         .src_addr = RTE_BE32(0xffffffff),
2510                         .dst_addr = RTE_BE32(0xffffffff),
2511                         .type_of_service = 0xff,
2512                         .fragment_offset = RTE_BE16(0xffff),
2513                         .next_proto_id = 0xff,
2514                         .time_to_live = 0xff,
2515                 },
2516         };
2517
2518         if (mask && (mask->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK)) {
2519                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2520                 bool ihl_cap = !tunnel ? priv->config.hca_attr.outer_ipv4_ihl :
2521                                priv->config.hca_attr.inner_ipv4_ihl;
2522                 if (!ihl_cap)
2523                         return rte_flow_error_set(error, ENOTSUP,
2524                                                   RTE_FLOW_ERROR_TYPE_ITEM,
2525                                                   item,
2526                                                   "IPV4 ihl offload not supported");
2527                 nic_ipv4_mask.hdr.version_ihl = mask->hdr.version_ihl;
2528         }
2529         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
2530                                            ether_type, &nic_ipv4_mask,
2531                                            MLX5_ITEM_RANGE_ACCEPTED, error);
2532         if (ret < 0)
2533                 return ret;
2534         if (spec && mask)
2535                 fragment_offset_spec = spec->hdr.fragment_offset &
2536                                        mask->hdr.fragment_offset;
2537         if (!fragment_offset_spec)
2538                 return 0;
2539         /*
2540          * spec and mask are valid, enforce using full mask to make sure the
2541          * complete value is used correctly.
2542          */
2543         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2544                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2545                 return rte_flow_error_set(error, EINVAL,
2546                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2547                                           item, "must use full mask for"
2548                                           " fragment_offset");
2549         /*
2550          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
2551          * indicating this is 1st fragment of fragmented packet.
2552          * This is not yet supported in MLX5, return appropriate error message.
2553          */
2554         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
2555                 return rte_flow_error_set(error, ENOTSUP,
2556                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2557                                           "match on first fragment not "
2558                                           "supported");
2559         if (fragment_offset_spec && !last)
2560                 return rte_flow_error_set(error, ENOTSUP,
2561                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2562                                           "specified value not supported");
2563         /* spec and last are valid, validate the specified range. */
2564         fragment_offset_last = last->hdr.fragment_offset &
2565                                mask->hdr.fragment_offset;
2566         /*
2567          * Match on fragment_offset spec 0x2001 and last 0x3fff
2568          * means MF is 1 and frag-offset is > 0.
2569          * This packet is fragment 2nd and onward, excluding last.
2570          * This is not yet supported in MLX5, return appropriate
2571          * error message.
2572          */
2573         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
2574             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
2575                 return rte_flow_error_set(error, ENOTSUP,
2576                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2577                                           last, "match on following "
2578                                           "fragments not supported");
2579         /*
2580          * Match on fragment_offset spec 0x0001 and last 0x1fff
2581          * means MF is 0 and frag-offset is > 0.
2582          * This packet is last fragment of fragmented packet.
2583          * This is not yet supported in MLX5, return appropriate
2584          * error message.
2585          */
2586         if (fragment_offset_spec == RTE_BE16(1) &&
2587             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
2588                 return rte_flow_error_set(error, ENOTSUP,
2589                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2590                                           last, "match on last "
2591                                           "fragment not supported");
2592         /*
2593          * Match on fragment_offset spec 0x0001 and last 0x3fff
2594          * means MF and/or frag-offset is not 0.
2595          * This is a fragmented packet.
2596          * Other range values are invalid and rejected.
2597          */
2598         if (!(fragment_offset_spec == RTE_BE16(1) &&
2599               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
2600                 return rte_flow_error_set(error, ENOTSUP,
2601                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2602                                           "specified range not supported");
2603         return 0;
2604 }
2605
2606 /**
2607  * Validate IPV6 fragment extension item.
2608  *
2609  * @param[in] item
2610  *   Item specification.
2611  * @param[in] item_flags
2612  *   Bit-fields that holds the items detected until now.
2613  * @param[out] error
2614  *   Pointer to error structure.
2615  *
2616  * @return
2617  *   0 on success, a negative errno value otherwise and rte_errno is set.
2618  */
2619 static int
2620 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
2621                                     uint64_t item_flags,
2622                                     struct rte_flow_error *error)
2623 {
2624         const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
2625         const struct rte_flow_item_ipv6_frag_ext *last = item->last;
2626         const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
2627         rte_be16_t frag_data_spec = 0;
2628         rte_be16_t frag_data_last = 0;
2629         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
2630         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
2631                                       MLX5_FLOW_LAYER_OUTER_L4;
2632         int ret = 0;
2633         struct rte_flow_item_ipv6_frag_ext nic_mask = {
2634                 .hdr = {
2635                         .next_header = 0xff,
2636                         .frag_data = RTE_BE16(0xffff),
2637                 },
2638         };
2639
2640         if (item_flags & l4m)
2641                 return rte_flow_error_set(error, EINVAL,
2642                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2643                                           "ipv6 fragment extension item cannot "
2644                                           "follow L4 item.");
2645         if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
2646             (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
2647                 return rte_flow_error_set(error, EINVAL,
2648                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2649                                           "ipv6 fragment extension item must "
2650                                           "follow ipv6 item");
2651         if (spec && mask)
2652                 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
2653         if (!frag_data_spec)
2654                 return 0;
2655         /*
2656          * spec and mask are valid, enforce using full mask to make sure the
2657          * complete value is used correctly.
2658          */
2659         if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
2660                                 RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2661                 return rte_flow_error_set(error, EINVAL,
2662                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
2663                                           item, "must use full mask for"
2664                                           " frag_data");
2665         /*
2666          * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
2667          * This is 1st fragment of fragmented packet.
2668          */
2669         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
2670                 return rte_flow_error_set(error, ENOTSUP,
2671                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2672                                           "match on first fragment not "
2673                                           "supported");
2674         if (frag_data_spec && !last)
2675                 return rte_flow_error_set(error, EINVAL,
2676                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
2677                                           "specified value not supported");
2678         ret = mlx5_flow_item_acceptable
2679                                 (item, (const uint8_t *)mask,
2680                                  (const uint8_t *)&nic_mask,
2681                                  sizeof(struct rte_flow_item_ipv6_frag_ext),
2682                                  MLX5_ITEM_RANGE_ACCEPTED, error);
2683         if (ret)
2684                 return ret;
2685         /* spec and last are valid, validate the specified range. */
2686         frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
2687         /*
2688          * Match on frag_data spec 0x0009 and last 0xfff9
2689          * means M is 1 and frag-offset is > 0.
2690          * This packet is fragment 2nd and onward, excluding last.
2691          * This is not yet supported in MLX5, return appropriate
2692          * error message.
2693          */
2694         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
2695                                        RTE_IPV6_EHDR_MF_MASK) &&
2696             frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2697                 return rte_flow_error_set(error, ENOTSUP,
2698                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2699                                           last, "match on following "
2700                                           "fragments not supported");
2701         /*
2702          * Match on frag_data spec 0x0008 and last 0xfff8
2703          * means M is 0 and frag-offset is > 0.
2704          * This packet is last fragment of fragmented packet.
2705          * This is not yet supported in MLX5, return appropriate
2706          * error message.
2707          */
2708         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
2709             frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
2710                 return rte_flow_error_set(error, ENOTSUP,
2711                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2712                                           last, "match on last "
2713                                           "fragment not supported");
2714         /* Other range values are invalid and rejected. */
2715         return rte_flow_error_set(error, EINVAL,
2716                                   RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2717                                   "specified range not supported");
2718 }
2719
2720 /*
2721  * Validate ASO CT item.
2722  *
2723  * @param[in] dev
2724  *   Pointer to the rte_eth_dev structure.
2725  * @param[in] item
2726  *   Item specification.
2727  * @param[in] item_flags
2728  *   Pointer to bit-fields that holds the items detected until now.
2729  * @param[out] error
2730  *   Pointer to error structure.
2731  *
2732  * @return
2733  *   0 on success, a negative errno value otherwise and rte_errno is set.
2734  */
2735 static int
2736 flow_dv_validate_item_aso_ct(struct rte_eth_dev *dev,
2737                              const struct rte_flow_item *item,
2738                              uint64_t *item_flags,
2739                              struct rte_flow_error *error)
2740 {
2741         const struct rte_flow_item_conntrack *spec = item->spec;
2742         const struct rte_flow_item_conntrack *mask = item->mask;
2743         RTE_SET_USED(dev);
2744         uint32_t flags;
2745
2746         if (*item_flags & MLX5_FLOW_LAYER_ASO_CT)
2747                 return rte_flow_error_set(error, EINVAL,
2748                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2749                                           "Only one CT is supported");
2750         if (!mask)
2751                 mask = &rte_flow_item_conntrack_mask;
2752         flags = spec->flags & mask->flags;
2753         if ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID) &&
2754             ((flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID) ||
2755              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD) ||
2756              (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)))
2757                 return rte_flow_error_set(error, EINVAL,
2758                                           RTE_FLOW_ERROR_TYPE_ITEM, NULL,
2759                                           "Conflict status bits");
2760         /* State change also needs to be considered. */
2761         *item_flags |= MLX5_FLOW_LAYER_ASO_CT;
2762         return 0;
2763 }
2764
2765 /**
2766  * Validate the pop VLAN action.
2767  *
2768  * @param[in] dev
2769  *   Pointer to the rte_eth_dev structure.
2770  * @param[in] action_flags
2771  *   Holds the actions detected until now.
2772  * @param[in] action
2773  *   Pointer to the pop vlan action.
2774  * @param[in] item_flags
2775  *   The items found in this flow rule.
2776  * @param[in] attr
2777  *   Pointer to flow attributes.
2778  * @param[out] error
2779  *   Pointer to error structure.
2780  *
2781  * @return
2782  *   0 on success, a negative errno value otherwise and rte_errno is set.
2783  */
2784 static int
2785 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
2786                                  uint64_t action_flags,
2787                                  const struct rte_flow_action *action,
2788                                  uint64_t item_flags,
2789                                  const struct rte_flow_attr *attr,
2790                                  struct rte_flow_error *error)
2791 {
2792         const struct mlx5_priv *priv = dev->data->dev_private;
2793         struct mlx5_dev_ctx_shared *sh = priv->sh;
2794         bool direction_error = false;
2795
2796         if (!priv->sh->pop_vlan_action)
2797                 return rte_flow_error_set(error, ENOTSUP,
2798                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2799                                           NULL,
2800                                           "pop vlan action is not supported");
2801         /* Pop VLAN is not supported in egress except for CX6 FDB mode. */
2802         if (attr->transfer) {
2803                 bool fdb_tx = priv->representor_id != UINT16_MAX;
2804                 bool is_cx5 = sh->steering_format_version ==
2805                     MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5;
2806
2807                 if (fdb_tx && is_cx5)
2808                         direction_error = true;
2809         } else if (attr->egress) {
2810                 direction_error = true;
2811         }
2812         if (direction_error)
2813                 return rte_flow_error_set(error, ENOTSUP,
2814                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2815                                           NULL,
2816                                           "pop vlan action not supported for egress");
2817         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
2818                 return rte_flow_error_set(error, ENOTSUP,
2819                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2820                                           "no support for multiple VLAN "
2821                                           "actions");
2822         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
2823         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
2824             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
2825                 return rte_flow_error_set(error, ENOTSUP,
2826                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2827                                           NULL,
2828                                           "cannot pop vlan after decap without "
2829                                           "match on inner vlan in the flow");
2830         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
2831         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
2832             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2833                 return rte_flow_error_set(error, ENOTSUP,
2834                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2835                                           NULL,
2836                                           "cannot pop vlan without a "
2837                                           "match on (outer) vlan in the flow");
2838         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2839                 return rte_flow_error_set(error, EINVAL,
2840                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2841                                           "wrong action order, port_id should "
2842                                           "be after pop VLAN action");
2843         if (!attr->transfer && priv->representor)
2844                 return rte_flow_error_set(error, ENOTSUP,
2845                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2846                                           "pop vlan action for VF representor "
2847                                           "not supported on NIC table");
2848         return 0;
2849 }
2850
2851 /**
2852  * Get VLAN default info from vlan match info.
2853  *
2854  * @param[in] items
2855  *   the list of item specifications.
2856  * @param[out] vlan
2857  *   pointer VLAN info to fill to.
2858  *
2859  * @return
2860  *   0 on success, a negative errno value otherwise and rte_errno is set.
2861  */
2862 static void
2863 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2864                                   struct rte_vlan_hdr *vlan)
2865 {
2866         const struct rte_flow_item_vlan nic_mask = {
2867                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2868                                 MLX5DV_FLOW_VLAN_VID_MASK),
2869                 .inner_type = RTE_BE16(0xffff),
2870         };
2871
2872         if (items == NULL)
2873                 return;
2874         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2875                 int type = items->type;
2876
2877                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2878                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2879                         break;
2880         }
2881         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2882                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2883                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2884
2885                 /* If VLAN item in pattern doesn't contain data, return here. */
2886                 if (!vlan_v)
2887                         return;
2888                 if (!vlan_m)
2889                         vlan_m = &nic_mask;
2890                 /* Only full match values are accepted */
2891                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2892                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2893                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2894                         vlan->vlan_tci |=
2895                                 rte_be_to_cpu_16(vlan_v->tci &
2896                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2897                 }
2898                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2899                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2900                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2901                         vlan->vlan_tci |=
2902                                 rte_be_to_cpu_16(vlan_v->tci &
2903                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2904                 }
2905                 if (vlan_m->inner_type == nic_mask.inner_type)
2906                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2907                                                            vlan_m->inner_type);
2908         }
2909 }
2910
2911 /**
2912  * Validate the push VLAN action.
2913  *
2914  * @param[in] dev
2915  *   Pointer to the rte_eth_dev structure.
2916  * @param[in] action_flags
2917  *   Holds the actions detected until now.
2918  * @param[in] item_flags
2919  *   The items found in this flow rule.
2920  * @param[in] action
2921  *   Pointer to the action structure.
2922  * @param[in] attr
2923  *   Pointer to flow attributes
2924  * @param[out] error
2925  *   Pointer to error structure.
2926  *
2927  * @return
2928  *   0 on success, a negative errno value otherwise and rte_errno is set.
2929  */
2930 static int
2931 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2932                                   uint64_t action_flags,
2933                                   const struct rte_flow_item_vlan *vlan_m,
2934                                   const struct rte_flow_action *action,
2935                                   const struct rte_flow_attr *attr,
2936                                   struct rte_flow_error *error)
2937 {
2938         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2939         const struct mlx5_priv *priv = dev->data->dev_private;
2940         struct mlx5_dev_ctx_shared *sh = priv->sh;
2941         bool direction_error = false;
2942
2943         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2944             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2945                 return rte_flow_error_set(error, EINVAL,
2946                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2947                                           "invalid vlan ethertype");
2948         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2949                 return rte_flow_error_set(error, EINVAL,
2950                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2951                                           "wrong action order, port_id should "
2952                                           "be after push VLAN");
2953         /* Push VLAN is not supported in ingress except for CX6 FDB mode. */
2954         if (attr->transfer) {
2955                 bool fdb_tx = priv->representor_id != UINT16_MAX;
2956                 bool is_cx5 = sh->steering_format_version ==
2957                     MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5;
2958
2959                 if (!fdb_tx && is_cx5)
2960                         direction_error = true;
2961         } else if (attr->ingress) {
2962                 direction_error = true;
2963         }
2964         if (direction_error)
2965                 return rte_flow_error_set(error, ENOTSUP,
2966                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
2967                                           NULL,
2968                                           "push vlan action not supported for ingress");
2969         if (!attr->transfer && priv->representor)
2970                 return rte_flow_error_set(error, ENOTSUP,
2971                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2972                                           "push vlan action for VF representor "
2973                                           "not supported on NIC table");
2974         if (vlan_m &&
2975             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2976             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2977                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2978             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2979             !(mlx5_flow_find_action
2980                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2981                 return rte_flow_error_set(error, EINVAL,
2982                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2983                                           "not full match mask on VLAN PCP and "
2984                                           "there is no of_set_vlan_pcp action, "
2985                                           "push VLAN action cannot figure out "
2986                                           "PCP value");
2987         if (vlan_m &&
2988             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2989             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2990                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2991             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2992             !(mlx5_flow_find_action
2993                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2994                 return rte_flow_error_set(error, EINVAL,
2995                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2996                                           "not full match mask on VLAN VID and "
2997                                           "there is no of_set_vlan_vid action, "
2998                                           "push VLAN action cannot figure out "
2999                                           "VID value");
3000         (void)attr;
3001         return 0;
3002 }
3003
3004 /**
3005  * Validate the set VLAN PCP.
3006  *
3007  * @param[in] action_flags
3008  *   Holds the actions detected until now.
3009  * @param[in] actions
3010  *   Pointer to the list of actions remaining in the flow rule.
3011  * @param[out] error
3012  *   Pointer to error structure.
3013  *
3014  * @return
3015  *   0 on success, a negative errno value otherwise and rte_errno is set.
3016  */
3017 static int
3018 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
3019                                      const struct rte_flow_action actions[],
3020                                      struct rte_flow_error *error)
3021 {
3022         const struct rte_flow_action *action = actions;
3023         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
3024
3025         if (conf->vlan_pcp > 7)
3026                 return rte_flow_error_set(error, EINVAL,
3027                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3028                                           "VLAN PCP value is too big");
3029         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
3030                 return rte_flow_error_set(error, ENOTSUP,
3031                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3032                                           "set VLAN PCP action must follow "
3033                                           "the push VLAN action");
3034         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
3035                 return rte_flow_error_set(error, ENOTSUP,
3036                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3037                                           "Multiple VLAN PCP modification are "
3038                                           "not supported");
3039         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
3040                 return rte_flow_error_set(error, EINVAL,
3041                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3042                                           "wrong action order, port_id should "
3043                                           "be after set VLAN PCP");
3044         return 0;
3045 }
3046
3047 /**
3048  * Validate the set VLAN VID.
3049  *
3050  * @param[in] item_flags
3051  *   Holds the items detected in this rule.
3052  * @param[in] action_flags
3053  *   Holds the actions detected until now.
3054  * @param[in] actions
3055  *   Pointer to the list of actions remaining in the flow rule.
3056  * @param[out] error
3057  *   Pointer to error structure.
3058  *
3059  * @return
3060  *   0 on success, a negative errno value otherwise and rte_errno is set.
3061  */
3062 static int
3063 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
3064                                      uint64_t action_flags,
3065                                      const struct rte_flow_action actions[],
3066                                      struct rte_flow_error *error)
3067 {
3068         const struct rte_flow_action *action = actions;
3069         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
3070
3071         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
3072                 return rte_flow_error_set(error, EINVAL,
3073                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3074                                           "VLAN VID value is too big");
3075         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
3076             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
3077                 return rte_flow_error_set(error, ENOTSUP,
3078                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3079                                           "set VLAN VID action must follow push"
3080                                           " VLAN action or match on VLAN item");
3081         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
3082                 return rte_flow_error_set(error, ENOTSUP,
3083                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3084                                           "Multiple VLAN VID modifications are "
3085                                           "not supported");
3086         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
3087                 return rte_flow_error_set(error, EINVAL,
3088                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3089                                           "wrong action order, port_id should "
3090                                           "be after set VLAN VID");
3091         return 0;
3092 }
3093
3094 /*
3095  * Validate the FLAG action.
3096  *
3097  * @param[in] dev
3098  *   Pointer to the rte_eth_dev structure.
3099  * @param[in] action_flags
3100  *   Holds the actions detected until now.
3101  * @param[in] attr
3102  *   Pointer to flow attributes
3103  * @param[out] error
3104  *   Pointer to error structure.
3105  *
3106  * @return
3107  *   0 on success, a negative errno value otherwise and rte_errno is set.
3108  */
3109 static int
3110 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
3111                              uint64_t action_flags,
3112                              const struct rte_flow_attr *attr,
3113                              struct rte_flow_error *error)
3114 {
3115         struct mlx5_priv *priv = dev->data->dev_private;
3116         struct mlx5_dev_config *config = &priv->config;
3117         int ret;
3118
3119         /* Fall back if no extended metadata register support. */
3120         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3121                 return mlx5_flow_validate_action_flag(action_flags, attr,
3122                                                       error);
3123         /* Extensive metadata mode requires registers. */
3124         if (!mlx5_flow_ext_mreg_supported(dev))
3125                 return rte_flow_error_set(error, ENOTSUP,
3126                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3127                                           "no metadata registers "
3128                                           "to support flag action");
3129         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
3130                 return rte_flow_error_set(error, ENOTSUP,
3131                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3132                                           "extended metadata register"
3133                                           " isn't available");
3134         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3135         if (ret < 0)
3136                 return ret;
3137         MLX5_ASSERT(ret > 0);
3138         if (action_flags & MLX5_FLOW_ACTION_MARK)
3139                 return rte_flow_error_set(error, EINVAL,
3140                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3141                                           "can't mark and flag in same flow");
3142         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3143                 return rte_flow_error_set(error, EINVAL,
3144                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3145                                           "can't have 2 flag"
3146                                           " actions in same flow");
3147         return 0;
3148 }
3149
3150 /**
3151  * Validate MARK action.
3152  *
3153  * @param[in] dev
3154  *   Pointer to the rte_eth_dev structure.
3155  * @param[in] action
3156  *   Pointer to action.
3157  * @param[in] action_flags
3158  *   Holds the actions detected until now.
3159  * @param[in] attr
3160  *   Pointer to flow attributes
3161  * @param[out] error
3162  *   Pointer to error structure.
3163  *
3164  * @return
3165  *   0 on success, a negative errno value otherwise and rte_errno is set.
3166  */
3167 static int
3168 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
3169                              const struct rte_flow_action *action,
3170                              uint64_t action_flags,
3171                              const struct rte_flow_attr *attr,
3172                              struct rte_flow_error *error)
3173 {
3174         struct mlx5_priv *priv = dev->data->dev_private;
3175         struct mlx5_dev_config *config = &priv->config;
3176         const struct rte_flow_action_mark *mark = action->conf;
3177         int ret;
3178
3179         if (is_tunnel_offload_active(dev))
3180                 return rte_flow_error_set(error, ENOTSUP,
3181                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3182                                           "no mark action "
3183                                           "if tunnel offload active");
3184         /* Fall back if no extended metadata register support. */
3185         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
3186                 return mlx5_flow_validate_action_mark(action, action_flags,
3187                                                       attr, error);
3188         /* Extensive metadata mode requires registers. */
3189         if (!mlx5_flow_ext_mreg_supported(dev))
3190                 return rte_flow_error_set(error, ENOTSUP,
3191                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3192                                           "no metadata registers "
3193                                           "to support mark action");
3194         if (!priv->sh->dv_mark_mask)
3195                 return rte_flow_error_set(error, ENOTSUP,
3196                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3197                                           "extended metadata register"
3198                                           " isn't available");
3199         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
3200         if (ret < 0)
3201                 return ret;
3202         MLX5_ASSERT(ret > 0);
3203         if (!mark)
3204                 return rte_flow_error_set(error, EINVAL,
3205                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3206                                           "configuration cannot be null");
3207         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
3208                 return rte_flow_error_set(error, EINVAL,
3209                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3210                                           &mark->id,
3211                                           "mark id exceeds the limit");
3212         if (action_flags & MLX5_FLOW_ACTION_FLAG)
3213                 return rte_flow_error_set(error, EINVAL,
3214                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3215                                           "can't flag and mark in same flow");
3216         if (action_flags & MLX5_FLOW_ACTION_MARK)
3217                 return rte_flow_error_set(error, EINVAL,
3218                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3219                                           "can't have 2 mark actions in same"
3220                                           " flow");
3221         return 0;
3222 }
3223
3224 /**
3225  * Validate SET_META action.
3226  *
3227  * @param[in] dev
3228  *   Pointer to the rte_eth_dev structure.
3229  * @param[in] action
3230  *   Pointer to the action structure.
3231  * @param[in] action_flags
3232  *   Holds the actions detected until now.
3233  * @param[in] attr
3234  *   Pointer to flow attributes
3235  * @param[out] error
3236  *   Pointer to error structure.
3237  *
3238  * @return
3239  *   0 on success, a negative errno value otherwise and rte_errno is set.
3240  */
3241 static int
3242 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
3243                                  const struct rte_flow_action *action,
3244                                  uint64_t action_flags __rte_unused,
3245                                  const struct rte_flow_attr *attr,
3246                                  struct rte_flow_error *error)
3247 {
3248         const struct rte_flow_action_set_meta *conf;
3249         uint32_t nic_mask = UINT32_MAX;
3250         int reg;
3251
3252         if (!mlx5_flow_ext_mreg_supported(dev))
3253                 return rte_flow_error_set(error, ENOTSUP,
3254                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3255                                           "extended metadata register"
3256                                           " isn't supported");
3257         reg = flow_dv_get_metadata_reg(dev, attr, error);
3258         if (reg < 0)
3259                 return reg;
3260         if (reg == REG_NON)
3261                 return rte_flow_error_set(error, ENOTSUP,
3262                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3263                                           "unavalable extended metadata register");
3264         if (reg != REG_A && reg != REG_B) {
3265                 struct mlx5_priv *priv = dev->data->dev_private;
3266
3267                 nic_mask = priv->sh->dv_meta_mask;
3268         }
3269         if (!(action->conf))
3270                 return rte_flow_error_set(error, EINVAL,
3271                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3272                                           "configuration cannot be null");
3273         conf = (const struct rte_flow_action_set_meta *)action->conf;
3274         if (!conf->mask)
3275                 return rte_flow_error_set(error, EINVAL,
3276                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3277                                           "zero mask doesn't have any effect");
3278         if (conf->mask & ~nic_mask)
3279                 return rte_flow_error_set(error, EINVAL,
3280                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3281                                           "meta data must be within reg C0");
3282         return 0;
3283 }
3284
3285 /**
3286  * Validate SET_TAG action.
3287  *
3288  * @param[in] dev
3289  *   Pointer to the rte_eth_dev structure.
3290  * @param[in] action
3291  *   Pointer to the action structure.
3292  * @param[in] action_flags
3293  *   Holds the actions detected until now.
3294  * @param[in] attr
3295  *   Pointer to flow attributes
3296  * @param[out] error
3297  *   Pointer to error structure.
3298  *
3299  * @return
3300  *   0 on success, a negative errno value otherwise and rte_errno is set.
3301  */
3302 static int
3303 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
3304                                 const struct rte_flow_action *action,
3305                                 uint64_t action_flags,
3306                                 const struct rte_flow_attr *attr,
3307                                 struct rte_flow_error *error)
3308 {
3309         const struct rte_flow_action_set_tag *conf;
3310         const uint64_t terminal_action_flags =
3311                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
3312                 MLX5_FLOW_ACTION_RSS;
3313         int ret;
3314
3315         if (!mlx5_flow_ext_mreg_supported(dev))
3316                 return rte_flow_error_set(error, ENOTSUP,
3317                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3318                                           "extensive metadata register"
3319                                           " isn't supported");
3320         if (!(action->conf))
3321                 return rte_flow_error_set(error, EINVAL,
3322                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3323                                           "configuration cannot be null");
3324         conf = (const struct rte_flow_action_set_tag *)action->conf;
3325         if (!conf->mask)
3326                 return rte_flow_error_set(error, EINVAL,
3327                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3328                                           "zero mask doesn't have any effect");
3329         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
3330         if (ret < 0)
3331                 return ret;
3332         if (!attr->transfer && attr->ingress &&
3333             (action_flags & terminal_action_flags))
3334                 return rte_flow_error_set(error, EINVAL,
3335                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3336                                           "set_tag has no effect"
3337                                           " with terminal actions");
3338         return 0;
3339 }
3340
3341 /**
3342  * Check if action counter is shared by either old or new mechanism.
3343  *
3344  * @param[in] action
3345  *   Pointer to the action structure.
3346  *
3347  * @return
3348  *   True when counter is shared, false otherwise.
3349  */
3350 static inline bool
3351 is_shared_action_count(const struct rte_flow_action *action)
3352 {
3353         const struct rte_flow_action_count *count =
3354                         (const struct rte_flow_action_count *)action->conf;
3355
3356         if ((int)action->type == MLX5_RTE_FLOW_ACTION_TYPE_COUNT)
3357                 return true;
3358         return !!(count && count->shared);
3359 }
3360
3361 /**
3362  * Validate count action.
3363  *
3364  * @param[in] dev
3365  *   Pointer to rte_eth_dev structure.
3366  * @param[in] shared
3367  *   Indicator if action is shared.
3368  * @param[in] action_flags
3369  *   Holds the actions detected until now.
3370  * @param[out] error
3371  *   Pointer to error structure.
3372  *
3373  * @return
3374  *   0 on success, a negative errno value otherwise and rte_errno is set.
3375  */
3376 static int
3377 flow_dv_validate_action_count(struct rte_eth_dev *dev, bool shared,
3378                               uint64_t action_flags,
3379                               struct rte_flow_error *error)
3380 {
3381         struct mlx5_priv *priv = dev->data->dev_private;
3382
3383         if (!priv->config.devx)
3384                 goto notsup_err;
3385         if (action_flags & MLX5_FLOW_ACTION_COUNT)
3386                 return rte_flow_error_set(error, EINVAL,
3387                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3388                                           "duplicate count actions set");
3389         if (shared && (action_flags & MLX5_FLOW_ACTION_AGE) &&
3390             !priv->sh->flow_hit_aso_en)
3391                 return rte_flow_error_set(error, EINVAL,
3392                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3393                                           "old age and shared count combination is not supported");
3394 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
3395         return 0;
3396 #endif
3397 notsup_err:
3398         return rte_flow_error_set
3399                       (error, ENOTSUP,
3400                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3401                        NULL,
3402                        "count action not supported");
3403 }
3404
3405 /**
3406  * Validate the L2 encap action.
3407  *
3408  * @param[in] dev
3409  *   Pointer to the rte_eth_dev structure.
3410  * @param[in] action_flags
3411  *   Holds the actions detected until now.
3412  * @param[in] action
3413  *   Pointer to the action structure.
3414  * @param[in] attr
3415  *   Pointer to flow attributes.
3416  * @param[out] error
3417  *   Pointer to error structure.
3418  *
3419  * @return
3420  *   0 on success, a negative errno value otherwise and rte_errno is set.
3421  */
3422 static int
3423 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
3424                                  uint64_t action_flags,
3425                                  const struct rte_flow_action *action,
3426                                  const struct rte_flow_attr *attr,
3427                                  struct rte_flow_error *error)
3428 {
3429         const struct mlx5_priv *priv = dev->data->dev_private;
3430
3431         if (!(action->conf))
3432                 return rte_flow_error_set(error, EINVAL,
3433                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3434                                           "configuration cannot be null");
3435         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3436                 return rte_flow_error_set(error, EINVAL,
3437                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3438                                           "can only have a single encap action "
3439                                           "in a flow");
3440         if (!attr->transfer && priv->representor)
3441                 return rte_flow_error_set(error, ENOTSUP,
3442                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3443                                           "encap action for VF representor "
3444                                           "not supported on NIC table");
3445         return 0;
3446 }
3447
3448 /**
3449  * Validate a decap action.
3450  *
3451  * @param[in] dev
3452  *   Pointer to the rte_eth_dev structure.
3453  * @param[in] action_flags
3454  *   Holds the actions detected until now.
3455  * @param[in] action
3456  *   Pointer to the action structure.
3457  * @param[in] item_flags
3458  *   Holds the items detected.
3459  * @param[in] attr
3460  *   Pointer to flow attributes
3461  * @param[out] error
3462  *   Pointer to error structure.
3463  *
3464  * @return
3465  *   0 on success, a negative errno value otherwise and rte_errno is set.
3466  */
3467 static int
3468 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
3469                               uint64_t action_flags,
3470                               const struct rte_flow_action *action,
3471                               const uint64_t item_flags,
3472                               const struct rte_flow_attr *attr,
3473                               struct rte_flow_error *error)
3474 {
3475         const struct mlx5_priv *priv = dev->data->dev_private;
3476
3477         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
3478             !priv->config.decap_en)
3479                 return rte_flow_error_set(error, ENOTSUP,
3480                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3481                                           "decap is not enabled");
3482         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
3483                 return rte_flow_error_set(error, ENOTSUP,
3484                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3485                                           action_flags &
3486                                           MLX5_FLOW_ACTION_DECAP ? "can only "
3487                                           "have a single decap action" : "decap "
3488                                           "after encap is not supported");
3489         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
3490                 return rte_flow_error_set(error, EINVAL,
3491                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3492                                           "can't have decap action after"
3493                                           " modify action");
3494         if (attr->egress)
3495                 return rte_flow_error_set(error, ENOTSUP,
3496                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
3497                                           NULL,
3498                                           "decap action not supported for "
3499                                           "egress");
3500         if (!attr->transfer && priv->representor)
3501                 return rte_flow_error_set(error, ENOTSUP,
3502                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3503                                           "decap action for VF representor "
3504                                           "not supported on NIC table");
3505         if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_DECAP &&
3506             !(item_flags & MLX5_FLOW_LAYER_VXLAN))
3507                 return rte_flow_error_set(error, ENOTSUP,
3508                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3509                                 "VXLAN item should be present for VXLAN decap");
3510         return 0;
3511 }
3512
3513 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
3514
3515 /**
3516  * Validate the raw encap and decap actions.
3517  *
3518  * @param[in] dev
3519  *   Pointer to the rte_eth_dev structure.
3520  * @param[in] decap
3521  *   Pointer to the decap action.
3522  * @param[in] encap
3523  *   Pointer to the encap action.
3524  * @param[in] attr
3525  *   Pointer to flow attributes
3526  * @param[in/out] action_flags
3527  *   Holds the actions detected until now.
3528  * @param[out] actions_n
3529  *   pointer to the number of actions counter.
3530  * @param[in] action
3531  *   Pointer to the action structure.
3532  * @param[in] item_flags
3533  *   Holds the items detected.
3534  * @param[out] error
3535  *   Pointer to error structure.
3536  *
3537  * @return
3538  *   0 on success, a negative errno value otherwise and rte_errno is set.
3539  */
3540 static int
3541 flow_dv_validate_action_raw_encap_decap
3542         (struct rte_eth_dev *dev,
3543          const struct rte_flow_action_raw_decap *decap,
3544          const struct rte_flow_action_raw_encap *encap,
3545          const struct rte_flow_attr *attr, uint64_t *action_flags,
3546          int *actions_n, const struct rte_flow_action *action,
3547          uint64_t item_flags, struct rte_flow_error *error)
3548 {
3549         const struct mlx5_priv *priv = dev->data->dev_private;
3550         int ret;
3551
3552         if (encap && (!encap->size || !encap->data))
3553                 return rte_flow_error_set(error, EINVAL,
3554                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3555                                           "raw encap data cannot be empty");
3556         if (decap && encap) {
3557                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
3558                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
3559                         /* L3 encap. */
3560                         decap = NULL;
3561                 else if (encap->size <=
3562                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3563                            decap->size >
3564                            MLX5_ENCAPSULATION_DECISION_SIZE)
3565                         /* L3 decap. */
3566                         encap = NULL;
3567                 else if (encap->size >
3568                            MLX5_ENCAPSULATION_DECISION_SIZE &&
3569                            decap->size >
3570                            MLX5_ENCAPSULATION_DECISION_SIZE)
3571                         /* 2 L2 actions: encap and decap. */
3572                         ;
3573                 else
3574                         return rte_flow_error_set(error,
3575                                 ENOTSUP,
3576                                 RTE_FLOW_ERROR_TYPE_ACTION,
3577                                 NULL, "unsupported too small "
3578                                 "raw decap and too small raw "
3579                                 "encap combination");
3580         }
3581         if (decap) {
3582                 ret = flow_dv_validate_action_decap(dev, *action_flags, action,
3583                                                     item_flags, attr, error);
3584                 if (ret < 0)
3585                         return ret;
3586                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
3587                 ++(*actions_n);
3588         }
3589         if (encap) {
3590                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
3591                         return rte_flow_error_set(error, ENOTSUP,
3592                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3593                                                   NULL,
3594                                                   "small raw encap size");
3595                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
3596                         return rte_flow_error_set(error, EINVAL,
3597                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3598                                                   NULL,
3599                                                   "more than one encap action");
3600                 if (!attr->transfer && priv->representor)
3601                         return rte_flow_error_set
3602                                         (error, ENOTSUP,
3603                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3604                                          "encap action for VF representor "
3605                                          "not supported on NIC table");
3606                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
3607                 ++(*actions_n);
3608         }
3609         return 0;
3610 }
3611
3612 /*
3613  * Validate the ASO CT action.
3614  *
3615  * @param[in] dev
3616  *   Pointer to the rte_eth_dev structure.
3617  * @param[in] action_flags
3618  *   Holds the actions detected until now.
3619  * @param[in] item_flags
3620  *   The items found in this flow rule.
3621  * @param[in] attr
3622  *   Pointer to flow attributes.
3623  * @param[out] error
3624  *   Pointer to error structure.
3625  *
3626  * @return
3627  *   0 on success, a negative errno value otherwise and rte_errno is set.
3628  */
3629 static int
3630 flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev,
3631                                uint64_t action_flags,
3632                                uint64_t item_flags,
3633                                const struct rte_flow_attr *attr,
3634                                struct rte_flow_error *error)
3635 {
3636         RTE_SET_USED(dev);
3637
3638         if (attr->group == 0 && !attr->transfer)
3639                 return rte_flow_error_set(error, ENOTSUP,
3640                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3641                                           NULL,
3642                                           "Only support non-root table");
3643         if (action_flags & MLX5_FLOW_FATE_ACTIONS)
3644                 return rte_flow_error_set(error, ENOTSUP,
3645                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3646                                           "CT cannot follow a fate action");
3647         if ((action_flags & MLX5_FLOW_ACTION_METER) ||
3648             (action_flags & MLX5_FLOW_ACTION_AGE))
3649                 return rte_flow_error_set(error, EINVAL,
3650                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3651                                           "Only one ASO action is supported");
3652         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3653                 return rte_flow_error_set(error, EINVAL,
3654                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3655                                           "Encap cannot exist before CT");
3656         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_TCP))
3657                 return rte_flow_error_set(error, EINVAL,
3658                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3659                                           "Not a outer TCP packet");
3660         return 0;
3661 }
3662
3663 int
3664 flow_dv_encap_decap_match_cb(void *tool_ctx __rte_unused,
3665                              struct mlx5_list_entry *entry, void *cb_ctx)
3666 {
3667         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3668         struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
3669         struct mlx5_flow_dv_encap_decap_resource *resource;
3670
3671         resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource,
3672                                 entry);
3673         if (resource->reformat_type == ctx_resource->reformat_type &&
3674             resource->ft_type == ctx_resource->ft_type &&
3675             resource->flags == ctx_resource->flags &&
3676             resource->size == ctx_resource->size &&
3677             !memcmp((const void *)resource->buf,
3678                     (const void *)ctx_resource->buf,
3679                     resource->size))
3680                 return 0;
3681         return -1;
3682 }
3683
3684 struct mlx5_list_entry *
3685 flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
3686 {
3687         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3688         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3689         struct mlx5dv_dr_domain *domain;
3690         struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
3691         struct mlx5_flow_dv_encap_decap_resource *resource;
3692         uint32_t idx;
3693         int ret;
3694
3695         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3696                 domain = sh->fdb_domain;
3697         else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3698                 domain = sh->rx_domain;
3699         else
3700                 domain = sh->tx_domain;
3701         /* Register new encap/decap resource. */
3702         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &idx);
3703         if (!resource) {
3704                 rte_flow_error_set(ctx->error, ENOMEM,
3705                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3706                                    "cannot allocate resource memory");
3707                 return NULL;
3708         }
3709         *resource = *ctx_resource;
3710         resource->idx = idx;
3711         ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->ctx, domain,
3712                                                               resource,
3713                                                              &resource->action);
3714         if (ret) {
3715                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
3716                 rte_flow_error_set(ctx->error, ENOMEM,
3717                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3718                                    NULL, "cannot create action");
3719                 return NULL;
3720         }
3721
3722         return &resource->entry;
3723 }
3724
3725 struct mlx5_list_entry *
3726 flow_dv_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
3727                              void *cb_ctx)
3728 {
3729         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3730         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3731         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
3732         uint32_t idx;
3733
3734         cache_resource = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
3735                                            &idx);
3736         if (!cache_resource) {
3737                 rte_flow_error_set(ctx->error, ENOMEM,
3738                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3739                                    "cannot allocate resource memory");
3740                 return NULL;
3741         }
3742         memcpy(cache_resource, oentry, sizeof(*cache_resource));
3743         cache_resource->idx = idx;
3744         return &cache_resource->entry;
3745 }
3746
3747 void
3748 flow_dv_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3749 {
3750         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3751         struct mlx5_flow_dv_encap_decap_resource *res =
3752                                        container_of(entry, typeof(*res), entry);
3753
3754         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
3755 }
3756
3757 /**
3758  * Find existing encap/decap resource or create and register a new one.
3759  *
3760  * @param[in, out] dev
3761  *   Pointer to rte_eth_dev structure.
3762  * @param[in, out] resource
3763  *   Pointer to encap/decap resource.
3764  * @parm[in, out] dev_flow
3765  *   Pointer to the dev_flow.
3766  * @param[out] error
3767  *   pointer to error structure.
3768  *
3769  * @return
3770  *   0 on success otherwise -errno and errno is set.
3771  */
3772 static int
3773 flow_dv_encap_decap_resource_register
3774                         (struct rte_eth_dev *dev,
3775                          struct mlx5_flow_dv_encap_decap_resource *resource,
3776                          struct mlx5_flow *dev_flow,
3777                          struct rte_flow_error *error)
3778 {
3779         struct mlx5_priv *priv = dev->data->dev_private;
3780         struct mlx5_dev_ctx_shared *sh = priv->sh;
3781         struct mlx5_list_entry *entry;
3782         union {
3783                 struct {
3784                         uint32_t ft_type:8;
3785                         uint32_t refmt_type:8;
3786                         /*
3787                          * Header reformat actions can be shared between
3788                          * non-root tables. One bit to indicate non-root
3789                          * table or not.
3790                          */
3791                         uint32_t is_root:1;
3792                         uint32_t reserve:15;
3793                 };
3794                 uint32_t v32;
3795         } encap_decap_key = {
3796                 {
3797                         .ft_type = resource->ft_type,
3798                         .refmt_type = resource->reformat_type,
3799                         .is_root = !!dev_flow->dv.group,
3800                         .reserve = 0,
3801                 }
3802         };
3803         struct mlx5_flow_cb_ctx ctx = {
3804                 .error = error,
3805                 .data = resource,
3806         };
3807         struct mlx5_hlist *encaps_decaps;
3808         uint64_t key64;
3809
3810         encaps_decaps = flow_dv_hlist_prepare(sh, &sh->encaps_decaps,
3811                                 "encaps_decaps",
3812                                 MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
3813                                 true, true, sh,
3814                                 flow_dv_encap_decap_create_cb,
3815                                 flow_dv_encap_decap_match_cb,
3816                                 flow_dv_encap_decap_remove_cb,
3817                                 flow_dv_encap_decap_clone_cb,
3818                                 flow_dv_encap_decap_clone_free_cb);
3819         if (unlikely(!encaps_decaps))
3820                 return -rte_errno;
3821         resource->flags = dev_flow->dv.group ? 0 : 1;
3822         key64 =  __rte_raw_cksum(&encap_decap_key.v32,
3823                                  sizeof(encap_decap_key.v32), 0);
3824         if (resource->reformat_type !=
3825             MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 &&
3826             resource->size)
3827                 key64 = __rte_raw_cksum(resource->buf, resource->size, key64);
3828         entry = mlx5_hlist_register(encaps_decaps, key64, &ctx);
3829         if (!entry)
3830                 return -rte_errno;
3831         resource = container_of(entry, typeof(*resource), entry);
3832         dev_flow->dv.encap_decap = resource;
3833         dev_flow->handle->dvh.rix_encap_decap = resource->idx;
3834         return 0;
3835 }
3836
3837 /**
3838  * Find existing table jump resource or create and register a new one.
3839  *
3840  * @param[in, out] dev
3841  *   Pointer to rte_eth_dev structure.
3842  * @param[in, out] tbl
3843  *   Pointer to flow table resource.
3844  * @parm[in, out] dev_flow
3845  *   Pointer to the dev_flow.
3846  * @param[out] error
3847  *   pointer to error structure.
3848  *
3849  * @return
3850  *   0 on success otherwise -errno and errno is set.
3851  */
3852 static int
3853 flow_dv_jump_tbl_resource_register
3854                         (struct rte_eth_dev *dev __rte_unused,
3855                          struct mlx5_flow_tbl_resource *tbl,
3856                          struct mlx5_flow *dev_flow,
3857                          struct rte_flow_error *error __rte_unused)
3858 {
3859         struct mlx5_flow_tbl_data_entry *tbl_data =
3860                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
3861
3862         MLX5_ASSERT(tbl);
3863         MLX5_ASSERT(tbl_data->jump.action);
3864         dev_flow->handle->rix_jump = tbl_data->idx;
3865         dev_flow->dv.jump = &tbl_data->jump;
3866         return 0;
3867 }
3868
3869 int
3870 flow_dv_port_id_match_cb(void *tool_ctx __rte_unused,
3871                          struct mlx5_list_entry *entry, void *cb_ctx)
3872 {
3873         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3874         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3875         struct mlx5_flow_dv_port_id_action_resource *res =
3876                                        container_of(entry, typeof(*res), entry);
3877
3878         return ref->port_id != res->port_id;
3879 }
3880
3881 struct mlx5_list_entry *
3882 flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx)
3883 {
3884         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3885         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3886         struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
3887         struct mlx5_flow_dv_port_id_action_resource *resource;
3888         uint32_t idx;
3889         int ret;
3890
3891         /* Register new port id action resource. */
3892         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
3893         if (!resource) {
3894                 rte_flow_error_set(ctx->error, ENOMEM,
3895                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3896                                    "cannot allocate port_id action memory");
3897                 return NULL;
3898         }
3899         *resource = *ref;
3900         ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain,
3901                                                         ref->port_id,
3902                                                         &resource->action);
3903         if (ret) {
3904                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx);
3905                 rte_flow_error_set(ctx->error, ENOMEM,
3906                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3907                                    "cannot create action");
3908                 return NULL;
3909         }
3910         resource->idx = idx;
3911         return &resource->entry;
3912 }
3913
3914 struct mlx5_list_entry *
3915 flow_dv_port_id_clone_cb(void *tool_ctx,
3916                          struct mlx5_list_entry *entry __rte_unused,
3917                          void *cb_ctx)
3918 {
3919         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3920         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3921         struct mlx5_flow_dv_port_id_action_resource *resource;
3922         uint32_t idx;
3923
3924         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
3925         if (!resource) {
3926                 rte_flow_error_set(ctx->error, ENOMEM,
3927                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3928                                    "cannot allocate port_id action memory");
3929                 return NULL;
3930         }
3931         memcpy(resource, entry, sizeof(*resource));
3932         resource->idx = idx;
3933         return &resource->entry;
3934 }
3935
3936 void
3937 flow_dv_port_id_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
3938 {
3939         struct mlx5_dev_ctx_shared *sh = tool_ctx;
3940         struct mlx5_flow_dv_port_id_action_resource *resource =
3941                                   container_of(entry, typeof(*resource), entry);
3942
3943         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
3944 }
3945
3946 /**
3947  * Find existing table port ID resource or create and register a new one.
3948  *
3949  * @param[in, out] dev
3950  *   Pointer to rte_eth_dev structure.
3951  * @param[in, out] ref
3952  *   Pointer to port ID action resource reference.
3953  * @parm[in, out] dev_flow
3954  *   Pointer to the dev_flow.
3955  * @param[out] error
3956  *   pointer to error structure.
3957  *
3958  * @return
3959  *   0 on success otherwise -errno and errno is set.
3960  */
3961 static int
3962 flow_dv_port_id_action_resource_register
3963                         (struct rte_eth_dev *dev,
3964                          struct mlx5_flow_dv_port_id_action_resource *ref,
3965                          struct mlx5_flow *dev_flow,
3966                          struct rte_flow_error *error)
3967 {
3968         struct mlx5_priv *priv = dev->data->dev_private;
3969         struct mlx5_list_entry *entry;
3970         struct mlx5_flow_dv_port_id_action_resource *resource;
3971         struct mlx5_flow_cb_ctx ctx = {
3972                 .error = error,
3973                 .data = ref,
3974         };
3975
3976         entry = mlx5_list_register(priv->sh->port_id_action_list, &ctx);
3977         if (!entry)
3978                 return -rte_errno;
3979         resource = container_of(entry, typeof(*resource), entry);
3980         dev_flow->dv.port_id_action = resource;
3981         dev_flow->handle->rix_port_id_action = resource->idx;
3982         return 0;
3983 }
3984
3985 int
3986 flow_dv_push_vlan_match_cb(void *tool_ctx __rte_unused,
3987                            struct mlx5_list_entry *entry, void *cb_ctx)
3988 {
3989         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
3990         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
3991         struct mlx5_flow_dv_push_vlan_action_resource *res =
3992                                        container_of(entry, typeof(*res), entry);
3993
3994         return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
3995 }
3996
3997 struct mlx5_list_entry *
3998 flow_dv_push_vlan_create_cb(void *tool_ctx, void *cb_ctx)
3999 {
4000         struct mlx5_dev_ctx_shared *sh = tool_ctx;
4001         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4002         struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
4003         struct mlx5_flow_dv_push_vlan_action_resource *resource;
4004         struct mlx5dv_dr_domain *domain;
4005         uint32_t idx;
4006         int ret;
4007
4008         /* Register new port id action resource. */
4009         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
4010         if (!resource) {
4011                 rte_flow_error_set(ctx->error, ENOMEM,
4012                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4013                                    "cannot allocate push_vlan action memory");
4014                 return NULL;
4015         }
4016         *resource = *ref;
4017         if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
4018                 domain = sh->fdb_domain;
4019         else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
4020                 domain = sh->rx_domain;
4021         else
4022                 domain = sh->tx_domain;
4023         ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag,
4024                                                         &resource->action);
4025         if (ret) {
4026                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
4027                 rte_flow_error_set(ctx->error, ENOMEM,
4028                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4029                                    "cannot create push vlan action");
4030                 return NULL;
4031         }
4032         resource->idx = idx;
4033         return &resource->entry;
4034 }
4035
4036 struct mlx5_list_entry *
4037 flow_dv_push_vlan_clone_cb(void *tool_ctx,
4038                            struct mlx5_list_entry *entry __rte_unused,
4039                            void *cb_ctx)
4040 {
4041         struct mlx5_dev_ctx_shared *sh = tool_ctx;
4042         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
4043         struct mlx5_flow_dv_push_vlan_action_resource *resource;
4044         uint32_t idx;
4045
4046         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
4047         if (!resource) {
4048                 rte_flow_error_set(ctx->error, ENOMEM,
4049                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4050                                    "cannot allocate push_vlan action memory");
4051                 return NULL;
4052         }
4053         memcpy(resource, entry, sizeof(*resource));
4054         resource->idx = idx;
4055         return &resource->entry;
4056 }
4057
4058 void
4059 flow_dv_push_vlan_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
4060 {
4061         struct mlx5_dev_ctx_shared *sh = tool_ctx;
4062         struct mlx5_flow_dv_push_vlan_action_resource *resource =
4063                                   container_of(entry, typeof(*resource), entry);
4064
4065         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
4066 }
4067
4068 /**
4069  * Find existing push vlan resource or create and register a new one.
4070  *
4071  * @param [in, out] dev
4072  *   Pointer to rte_eth_dev structure.
4073  * @param[in, out] ref
4074  *   Pointer to port ID action resource reference.
4075  * @parm[in, out] dev_flow
4076  *   Pointer to the dev_flow.
4077  * @param[out] error
4078  *   pointer to error structure.
4079  *
4080  * @return
4081  *   0 on success otherwise -errno and errno is set.
4082  */
4083 static int
4084 flow_dv_push_vlan_action_resource_register
4085                        (struct rte_eth_dev *dev,
4086                         struct mlx5_flow_dv_push_vlan_action_resource *ref,
4087                         struct mlx5_flow *dev_flow,
4088                         struct rte_flow_error *error)
4089 {
4090         struct mlx5_priv *priv = dev->data->dev_private;
4091         struct mlx5_flow_dv_push_vlan_action_resource *resource;
4092         struct mlx5_list_entry *entry;
4093         struct mlx5_flow_cb_ctx ctx = {
4094                 .error = error,
4095                 .data = ref,
4096         };
4097
4098         entry = mlx5_list_register(priv->sh->push_vlan_action_list, &ctx);
4099         if (!entry)
4100                 return -rte_errno;
4101         resource = container_of(entry, typeof(*resource), entry);
4102
4103         dev_flow->handle->dvh.rix_push_vlan = resource->idx;
4104         dev_flow->dv.push_vlan_res = resource;
4105         return 0;
4106 }
4107
4108 /**
4109  * Get the size of specific rte_flow_item_type hdr size
4110  *
4111  * @param[in] item_type
4112  *   Tested rte_flow_item_type.
4113  *
4114  * @return
4115  *   sizeof struct item_type, 0 if void or irrelevant.
4116  */
4117 static size_t
4118 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
4119 {
4120         size_t retval;
4121
4122         switch (item_type) {
4123         case RTE_FLOW_ITEM_TYPE_ETH:
4124                 retval = sizeof(struct rte_ether_hdr);
4125                 break;
4126         case RTE_FLOW_ITEM_TYPE_VLAN:
4127                 retval = sizeof(struct rte_vlan_hdr);
4128                 break;
4129         case RTE_FLOW_ITEM_TYPE_IPV4:
4130                 retval = sizeof(struct rte_ipv4_hdr);
4131                 break;
4132         case RTE_FLOW_ITEM_TYPE_IPV6:
4133                 retval = sizeof(struct rte_ipv6_hdr);
4134                 break;
4135         case RTE_FLOW_ITEM_TYPE_UDP:
4136                 retval = sizeof(struct rte_udp_hdr);
4137                 break;
4138         case RTE_FLOW_ITEM_TYPE_TCP:
4139                 retval = sizeof(struct rte_tcp_hdr);
4140                 break;
4141         case RTE_FLOW_ITEM_TYPE_VXLAN:
4142         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4143                 retval = sizeof(struct rte_vxlan_hdr);
4144                 break;
4145         case RTE_FLOW_ITEM_TYPE_GRE:
4146         case RTE_FLOW_ITEM_TYPE_NVGRE:
4147                 retval = sizeof(struct rte_gre_hdr);
4148                 break;
4149         case RTE_FLOW_ITEM_TYPE_MPLS:
4150                 retval = sizeof(struct rte_mpls_hdr);
4151                 break;
4152         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
4153         default:
4154                 retval = 0;
4155                 break;
4156         }
4157         return retval;
4158 }
4159
4160 #define MLX5_ENCAP_IPV4_VERSION         0x40
4161 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
4162 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
4163 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
4164 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
4165 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
4166 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
4167
4168 /**
4169  * Convert the encap action data from list of rte_flow_item to raw buffer
4170  *
4171  * @param[in] items
4172  *   Pointer to rte_flow_item objects list.
4173  * @param[out] buf
4174  *   Pointer to the output buffer.
4175  * @param[out] size
4176  *   Pointer to the output buffer size.
4177  * @param[out] error
4178  *   Pointer to the error structure.
4179  *
4180  * @return
4181  *   0 on success, a negative errno value otherwise and rte_errno is set.
4182  */
4183 static int
4184 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
4185                            size_t *size, struct rte_flow_error *error)
4186 {
4187         struct rte_ether_hdr *eth = NULL;
4188         struct rte_vlan_hdr *vlan = NULL;
4189         struct rte_ipv4_hdr *ipv4 = NULL;
4190         struct rte_ipv6_hdr *ipv6 = NULL;
4191         struct rte_udp_hdr *udp = NULL;
4192         struct rte_vxlan_hdr *vxlan = NULL;
4193         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
4194         struct rte_gre_hdr *gre = NULL;
4195         size_t len;
4196         size_t temp_size = 0;
4197
4198         if (!items)
4199                 return rte_flow_error_set(error, EINVAL,
4200                                           RTE_FLOW_ERROR_TYPE_ACTION,
4201                                           NULL, "invalid empty data");
4202         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4203                 len = flow_dv_get_item_hdr_len(items->type);
4204                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
4205                         return rte_flow_error_set(error, EINVAL,
4206                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4207                                                   (void *)items->type,
4208                                                   "items total size is too big"
4209                                                   " for encap action");
4210                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
4211                 switch (items->type) {
4212                 case RTE_FLOW_ITEM_TYPE_ETH:
4213                         eth = (struct rte_ether_hdr *)&buf[temp_size];
4214                         break;
4215                 case RTE_FLOW_ITEM_TYPE_VLAN:
4216                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
4217                         if (!eth)
4218                                 return rte_flow_error_set(error, EINVAL,
4219                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4220                                                 (void *)items->type,
4221                                                 "eth header not found");
4222                         if (!eth->ether_type)
4223                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
4224                         break;
4225                 case RTE_FLOW_ITEM_TYPE_IPV4:
4226                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
4227                         if (!vlan && !eth)
4228                                 return rte_flow_error_set(error, EINVAL,
4229                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4230                                                 (void *)items->type,
4231                                                 "neither eth nor vlan"
4232                                                 " header found");
4233                         if (vlan && !vlan->eth_proto)
4234                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4235                         else if (eth && !eth->ether_type)
4236                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
4237                         if (!ipv4->version_ihl)
4238                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
4239                                                     MLX5_ENCAP_IPV4_IHL_MIN;
4240                         if (!ipv4->time_to_live)
4241                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
4242                         break;
4243                 case RTE_FLOW_ITEM_TYPE_IPV6:
4244                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
4245                         if (!vlan && !eth)
4246                                 return rte_flow_error_set(error, EINVAL,
4247                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4248                                                 (void *)items->type,
4249                                                 "neither eth nor vlan"
4250                                                 " header found");
4251                         if (vlan && !vlan->eth_proto)
4252                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4253                         else if (eth && !eth->ether_type)
4254                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
4255                         if (!ipv6->vtc_flow)
4256                                 ipv6->vtc_flow =
4257                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
4258                         if (!ipv6->hop_limits)
4259                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
4260                         break;
4261                 case RTE_FLOW_ITEM_TYPE_UDP:
4262                         udp = (struct rte_udp_hdr *)&buf[temp_size];
4263                         if (!ipv4 && !ipv6)
4264                                 return rte_flow_error_set(error, EINVAL,
4265                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4266                                                 (void *)items->type,
4267                                                 "ip header not found");
4268                         if (ipv4 && !ipv4->next_proto_id)
4269                                 ipv4->next_proto_id = IPPROTO_UDP;
4270                         else if (ipv6 && !ipv6->proto)
4271                                 ipv6->proto = IPPROTO_UDP;
4272                         break;
4273                 case RTE_FLOW_ITEM_TYPE_VXLAN:
4274                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
4275                         if (!udp)
4276                                 return rte_flow_error_set(error, EINVAL,
4277                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4278                                                 (void *)items->type,
4279                                                 "udp header not found");
4280                         if (!udp->dst_port)
4281                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
4282                         if (!vxlan->vx_flags)
4283                                 vxlan->vx_flags =
4284                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
4285                         break;
4286                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
4287                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
4288                         if (!udp)
4289                                 return rte_flow_error_set(error, EINVAL,
4290                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4291                                                 (void *)items->type,
4292                                                 "udp header not found");
4293                         if (!vxlan_gpe->proto)
4294                                 return rte_flow_error_set(error, EINVAL,
4295                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4296                                                 (void *)items->type,
4297                                                 "next protocol not found");
4298                         if (!udp->dst_port)
4299                                 udp->dst_port =
4300                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
4301                         if (!vxlan_gpe->vx_flags)
4302                                 vxlan_gpe->vx_flags =
4303                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
4304                         break;
4305                 case RTE_FLOW_ITEM_TYPE_GRE:
4306                 case RTE_FLOW_ITEM_TYPE_NVGRE:
4307                         gre = (struct rte_gre_hdr *)&buf[temp_size];
4308                         if (!gre->proto)
4309                                 return rte_flow_error_set(error, EINVAL,
4310                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4311                                                 (void *)items->type,
4312                                                 "next protocol not found");
4313                         if (!ipv4 && !ipv6)
4314                                 return rte_flow_error_set(error, EINVAL,
4315                                                 RTE_FLOW_ERROR_TYPE_ACTION,
4316                                                 (void *)items->type,
4317                                                 "ip header not found");
4318                         if (ipv4 && !ipv4->next_proto_id)
4319                                 ipv4->next_proto_id = IPPROTO_GRE;
4320                         else if (ipv6 && !ipv6->proto)
4321                                 ipv6->proto = IPPROTO_GRE;
4322                         break;
4323                 case RTE_FLOW_ITEM_TYPE_VOID:
4324                         break;
4325                 default:
4326                         return rte_flow_error_set(error, EINVAL,
4327                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4328                                                   (void *)items->type,
4329                                                   "unsupported item type");
4330                         break;
4331                 }
4332                 temp_size += len;
4333         }
4334         *size = temp_size;
4335         return 0;
4336 }
4337
4338 static int
4339 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
4340 {
4341         struct rte_ether_hdr *eth = NULL;
4342         struct rte_vlan_hdr *vlan = NULL;
4343         struct rte_ipv6_hdr *ipv6 = NULL;
4344         struct rte_udp_hdr *udp = NULL;
4345         char *next_hdr;
4346         uint16_t proto;
4347
4348         eth = (struct rte_ether_hdr *)data;
4349         next_hdr = (char *)(eth + 1);
4350         proto = RTE_BE16(eth->ether_type);
4351
4352         /* VLAN skipping */
4353         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
4354                 vlan = (struct rte_vlan_hdr *)next_hdr;
4355                 proto = RTE_BE16(vlan->eth_proto);
4356                 next_hdr += sizeof(struct rte_vlan_hdr);
4357         }
4358
4359         /* HW calculates IPv4 csum. no need to proceed */
4360         if (proto == RTE_ETHER_TYPE_IPV4)
4361                 return 0;
4362
4363         /* non IPv4/IPv6 header. not supported */
4364         if (proto != RTE_ETHER_TYPE_IPV6) {
4365                 return rte_flow_error_set(error, ENOTSUP,
4366                                           RTE_FLOW_ERROR_TYPE_ACTION,
4367                                           NULL, "Cannot offload non IPv4/IPv6");
4368         }
4369
4370         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
4371
4372         /* ignore non UDP */
4373         if (ipv6->proto != IPPROTO_UDP)
4374                 return 0;
4375
4376         udp = (struct rte_udp_hdr *)(ipv6 + 1);
4377         udp->dgram_cksum = 0;
4378
4379         return 0;
4380 }
4381
4382 /**
4383  * Convert L2 encap action to DV specification.
4384  *
4385  * @param[in] dev
4386  *   Pointer to rte_eth_dev structure.
4387  * @param[in] action
4388  *   Pointer to action structure.
4389  * @param[in, out] dev_flow
4390  *   Pointer to the mlx5_flow.
4391  * @param[in] transfer
4392  *   Mark if the flow is E-Switch flow.
4393  * @param[out] error
4394  *   Pointer to the error structure.
4395  *
4396  * @return
4397  *   0 on success, a negative errno value otherwise and rte_errno is set.
4398  */
4399 static int
4400 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
4401                                const struct rte_flow_action *action,
4402                                struct mlx5_flow *dev_flow,
4403                                uint8_t transfer,
4404                                struct rte_flow_error *error)
4405 {
4406         const struct rte_flow_item *encap_data;
4407         const struct rte_flow_action_raw_encap *raw_encap_data;
4408         struct mlx5_flow_dv_encap_decap_resource res = {
4409                 .reformat_type =
4410                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
4411                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4412                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
4413         };
4414
4415         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
4416                 raw_encap_data =
4417                         (const struct rte_flow_action_raw_encap *)action->conf;
4418                 res.size = raw_encap_data->size;
4419                 memcpy(res.buf, raw_encap_data->data, res.size);
4420         } else {
4421                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
4422                         encap_data =
4423                                 ((const struct rte_flow_action_vxlan_encap *)
4424                                                 action->conf)->definition;
4425                 else
4426                         encap_data =
4427                                 ((const struct rte_flow_action_nvgre_encap *)
4428                                                 action->conf)->definition;
4429                 if (flow_dv_convert_encap_data(encap_data, res.buf,
4430                                                &res.size, error))
4431                         return -rte_errno;
4432         }
4433         if (flow_dv_zero_encap_udp_csum(res.buf, error))
4434                 return -rte_errno;
4435         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4436                 return rte_flow_error_set(error, EINVAL,
4437                                           RTE_FLOW_ERROR_TYPE_ACTION,
4438                                           NULL, "can't create L2 encap action");
4439         return 0;
4440 }
4441
4442 /**
4443  * Convert L2 decap action to DV specification.
4444  *
4445  * @param[in] dev
4446  *   Pointer to rte_eth_dev structure.
4447  * @param[in, out] dev_flow
4448  *   Pointer to the mlx5_flow.
4449  * @param[in] transfer
4450  *   Mark if the flow is E-Switch flow.
4451  * @param[out] error
4452  *   Pointer to the error structure.
4453  *
4454  * @return
4455  *   0 on success, a negative errno value otherwise and rte_errno is set.
4456  */
4457 static int
4458 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
4459                                struct mlx5_flow *dev_flow,
4460                                uint8_t transfer,
4461                                struct rte_flow_error *error)
4462 {
4463         struct mlx5_flow_dv_encap_decap_resource res = {
4464                 .size = 0,
4465                 .reformat_type =
4466                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
4467                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
4468                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
4469         };
4470
4471         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4472                 return rte_flow_error_set(error, EINVAL,
4473                                           RTE_FLOW_ERROR_TYPE_ACTION,
4474                                           NULL, "can't create L2 decap action");
4475         return 0;
4476 }
4477
4478 /**
4479  * Convert raw decap/encap (L3 tunnel) action to DV specification.
4480  *
4481  * @param[in] dev
4482  *   Pointer to rte_eth_dev structure.
4483  * @param[in] action
4484  *   Pointer to action structure.
4485  * @param[in, out] dev_flow
4486  *   Pointer to the mlx5_flow.
4487  * @param[in] attr
4488  *   Pointer to the flow attributes.
4489  * @param[out] error
4490  *   Pointer to the error structure.
4491  *
4492  * @return
4493  *   0 on success, a negative errno value otherwise and rte_errno is set.
4494  */
4495 static int
4496 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
4497                                 const struct rte_flow_action *action,
4498                                 struct mlx5_flow *dev_flow,
4499                                 const struct rte_flow_attr *attr,
4500                                 struct rte_flow_error *error)
4501 {
4502         const struct rte_flow_action_raw_encap *encap_data;
4503         struct mlx5_flow_dv_encap_decap_resource res;
4504
4505         memset(&res, 0, sizeof(res));
4506         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
4507         res.size = encap_data->size;
4508         memcpy(res.buf, encap_data->data, res.size);
4509         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
4510                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
4511                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
4512         if (attr->transfer)
4513                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4514         else
4515                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4516                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4517         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
4518                 return rte_flow_error_set(error, EINVAL,
4519                                           RTE_FLOW_ERROR_TYPE_ACTION,
4520                                           NULL, "can't create encap action");
4521         return 0;
4522 }
4523
4524 /**
4525  * Create action push VLAN.
4526  *
4527  * @param[in] dev
4528  *   Pointer to rte_eth_dev structure.
4529  * @param[in] attr
4530  *   Pointer to the flow attributes.
4531  * @param[in] vlan
4532  *   Pointer to the vlan to push to the Ethernet header.
4533  * @param[in, out] dev_flow
4534  *   Pointer to the mlx5_flow.
4535  * @param[out] error
4536  *   Pointer to the error structure.
4537  *
4538  * @return
4539  *   0 on success, a negative errno value otherwise and rte_errno is set.
4540  */
4541 static int
4542 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
4543                                 const struct rte_flow_attr *attr,
4544                                 const struct rte_vlan_hdr *vlan,
4545                                 struct mlx5_flow *dev_flow,
4546                                 struct rte_flow_error *error)
4547 {
4548         struct mlx5_flow_dv_push_vlan_action_resource res;
4549
4550         memset(&res, 0, sizeof(res));
4551         res.vlan_tag =
4552                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
4553                                  vlan->vlan_tci);
4554         if (attr->transfer)
4555                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
4556         else
4557                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
4558                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
4559         return flow_dv_push_vlan_action_resource_register
4560                                             (dev, &res, dev_flow, error);
4561 }
4562
4563 /**
4564  * Validate the modify-header actions.
4565  *
4566  * @param[in] action_flags
4567  *   Holds the actions detected until now.
4568  * @param[in] action
4569  *   Pointer to the modify action.
4570  * @param[out] error
4571  *   Pointer to error structure.
4572  *
4573  * @return
4574  *   0 on success, a negative errno value otherwise and rte_errno is set.
4575  */
4576 static int
4577 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
4578                                    const struct rte_flow_action *action,
4579                                    struct rte_flow_error *error)
4580 {
4581         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
4582                 return rte_flow_error_set(error, EINVAL,
4583                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4584                                           NULL, "action configuration not set");
4585         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
4586                 return rte_flow_error_set(error, EINVAL,
4587                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4588                                           "can't have encap action before"
4589                                           " modify action");
4590         return 0;
4591 }
4592
4593 /**
4594  * Validate the modify-header MAC address actions.
4595  *
4596  * @param[in] action_flags
4597  *   Holds the actions detected until now.
4598  * @param[in] action
4599  *   Pointer to the modify action.
4600  * @param[in] item_flags
4601  *   Holds the items detected.
4602  * @param[out] error
4603  *   Pointer to error structure.
4604  *
4605  * @return
4606  *   0 on success, a negative errno value otherwise and rte_errno is set.
4607  */
4608 static int
4609 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
4610                                    const struct rte_flow_action *action,
4611                                    const uint64_t item_flags,
4612                                    struct rte_flow_error *error)
4613 {
4614         int ret = 0;
4615
4616         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4617         if (!ret) {
4618                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
4619                         return rte_flow_error_set(error, EINVAL,
4620                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4621                                                   NULL,
4622                                                   "no L2 item in pattern");
4623         }
4624         return ret;
4625 }
4626
4627 /**
4628  * Validate the modify-header IPv4 address actions.
4629  *
4630  * @param[in] action_flags
4631  *   Holds the actions detected until now.
4632  * @param[in] action
4633  *   Pointer to the modify action.
4634  * @param[in] item_flags
4635  *   Holds the items detected.
4636  * @param[out] error
4637  *   Pointer to error structure.
4638  *
4639  * @return
4640  *   0 on success, a negative errno value otherwise and rte_errno is set.
4641  */
4642 static int
4643 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
4644                                     const struct rte_flow_action *action,
4645                                     const uint64_t item_flags,
4646                                     struct rte_flow_error *error)
4647 {
4648         int ret = 0;
4649         uint64_t layer;
4650
4651         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4652         if (!ret) {
4653                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4654                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
4655                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
4656                 if (!(item_flags & layer))
4657                         return rte_flow_error_set(error, EINVAL,
4658                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4659                                                   NULL,
4660                                                   "no ipv4 item in pattern");
4661         }
4662         return ret;
4663 }
4664
4665 /**
4666  * Validate the modify-header IPv6 address actions.
4667  *
4668  * @param[in] action_flags
4669  *   Holds the actions detected until now.
4670  * @param[in] action
4671  *   Pointer to the modify action.
4672  * @param[in] item_flags
4673  *   Holds the items detected.
4674  * @param[out] error
4675  *   Pointer to error structure.
4676  *
4677  * @return
4678  *   0 on success, a negative errno value otherwise and rte_errno is set.
4679  */
4680 static int
4681 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
4682                                     const struct rte_flow_action *action,
4683                                     const uint64_t item_flags,
4684                                     struct rte_flow_error *error)
4685 {
4686         int ret = 0;
4687         uint64_t layer;
4688
4689         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4690         if (!ret) {
4691                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4692                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
4693                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
4694                 if (!(item_flags & layer))
4695                         return rte_flow_error_set(error, EINVAL,
4696                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4697                                                   NULL,
4698                                                   "no ipv6 item in pattern");
4699         }
4700         return ret;
4701 }
4702
4703 /**
4704  * Validate the modify-header TP actions.
4705  *
4706  * @param[in] action_flags
4707  *   Holds the actions detected until now.
4708  * @param[in] action
4709  *   Pointer to the modify action.
4710  * @param[in] item_flags
4711  *   Holds the items detected.
4712  * @param[out] error
4713  *   Pointer to error structure.
4714  *
4715  * @return
4716  *   0 on success, a negative errno value otherwise and rte_errno is set.
4717  */
4718 static int
4719 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
4720                                   const struct rte_flow_action *action,
4721                                   const uint64_t item_flags,
4722                                   struct rte_flow_error *error)
4723 {
4724         int ret = 0;
4725         uint64_t layer;
4726
4727         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4728         if (!ret) {
4729                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4730                                  MLX5_FLOW_LAYER_INNER_L4 :
4731                                  MLX5_FLOW_LAYER_OUTER_L4;
4732                 if (!(item_flags & layer))
4733                         return rte_flow_error_set(error, EINVAL,
4734                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4735                                                   NULL, "no transport layer "
4736                                                   "in pattern");
4737         }
4738         return ret;
4739 }
4740
4741 /**
4742  * Validate the modify-header actions of increment/decrement
4743  * TCP Sequence-number.
4744  *
4745  * @param[in] action_flags
4746  *   Holds the actions detected until now.
4747  * @param[in] action
4748  *   Pointer to the modify action.
4749  * @param[in] item_flags
4750  *   Holds the items detected.
4751  * @param[out] error
4752  *   Pointer to error structure.
4753  *
4754  * @return
4755  *   0 on success, a negative errno value otherwise and rte_errno is set.
4756  */
4757 static int
4758 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
4759                                        const struct rte_flow_action *action,
4760                                        const uint64_t item_flags,
4761                                        struct rte_flow_error *error)
4762 {
4763         int ret = 0;
4764         uint64_t layer;
4765
4766         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4767         if (!ret) {
4768                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4769                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4770                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4771                 if (!(item_flags & layer))
4772                         return rte_flow_error_set(error, EINVAL,
4773                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4774                                                   NULL, "no TCP item in"
4775                                                   " pattern");
4776                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
4777                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
4778                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
4779                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
4780                         return rte_flow_error_set(error, EINVAL,
4781                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4782                                                   NULL,
4783                                                   "cannot decrease and increase"
4784                                                   " TCP sequence number"
4785                                                   " at the same time");
4786         }
4787         return ret;
4788 }
4789
4790 /**
4791  * Validate the modify-header actions of increment/decrement
4792  * TCP Acknowledgment number.
4793  *
4794  * @param[in] action_flags
4795  *   Holds the actions detected until now.
4796  * @param[in] action
4797  *   Pointer to the modify action.
4798  * @param[in] item_flags
4799  *   Holds the items detected.
4800  * @param[out] error
4801  *   Pointer to error structure.
4802  *
4803  * @return
4804  *   0 on success, a negative errno value otherwise and rte_errno is set.
4805  */
4806 static int
4807 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
4808                                        const struct rte_flow_action *action,
4809                                        const uint64_t item_flags,
4810                                        struct rte_flow_error *error)
4811 {
4812         int ret = 0;
4813         uint64_t layer;
4814
4815         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4816         if (!ret) {
4817                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4818                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
4819                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
4820                 if (!(item_flags & layer))
4821                         return rte_flow_error_set(error, EINVAL,
4822                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4823                                                   NULL, "no TCP item in"
4824                                                   " pattern");
4825                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
4826                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
4827                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
4828                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
4829                         return rte_flow_error_set(error, EINVAL,
4830                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4831                                                   NULL,
4832                                                   "cannot decrease and increase"
4833                                                   " TCP acknowledgment number"
4834                                                   " at the same time");
4835         }
4836         return ret;
4837 }
4838
4839 /**
4840  * Validate the modify-header TTL actions.
4841  *
4842  * @param[in] action_flags
4843  *   Holds the actions detected until now.
4844  * @param[in] action
4845  *   Pointer to the modify action.
4846  * @param[in] item_flags
4847  *   Holds the items detected.
4848  * @param[out] error
4849  *   Pointer to error structure.
4850  *
4851  * @return
4852  *   0 on success, a negative errno value otherwise and rte_errno is set.
4853  */
4854 static int
4855 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
4856                                    const struct rte_flow_action *action,
4857                                    const uint64_t item_flags,
4858                                    struct rte_flow_error *error)
4859 {
4860         int ret = 0;
4861         uint64_t layer;
4862
4863         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4864         if (!ret) {
4865                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
4866                                  MLX5_FLOW_LAYER_INNER_L3 :
4867                                  MLX5_FLOW_LAYER_OUTER_L3;
4868                 if (!(item_flags & layer))
4869                         return rte_flow_error_set(error, EINVAL,
4870                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4871                                                   NULL,
4872                                                   "no IP protocol in pattern");
4873         }
4874         return ret;
4875 }
4876
4877 /**
4878  * Validate the generic modify field actions.
4879  * @param[in] dev
4880  *   Pointer to the rte_eth_dev structure.
4881  * @param[in] action_flags
4882  *   Holds the actions detected until now.
4883  * @param[in] action
4884  *   Pointer to the modify action.
4885  * @param[in] attr
4886  *   Pointer to the flow attributes.
4887  * @param[out] error
4888  *   Pointer to error structure.
4889  *
4890  * @return
4891  *   Number of header fields to modify (0 or more) on success,
4892  *   a negative errno value otherwise and rte_errno is set.
4893  */
4894 static int
4895 flow_dv_validate_action_modify_field(struct rte_eth_dev *dev,
4896                                    const uint64_t action_flags,
4897                                    const struct rte_flow_action *action,
4898                                    const struct rte_flow_attr *attr,
4899                                    struct rte_flow_error *error)
4900 {
4901         int ret = 0;
4902         struct mlx5_priv *priv = dev->data->dev_private;
4903         struct mlx5_dev_config *config = &priv->config;
4904         const struct rte_flow_action_modify_field *action_modify_field =
4905                 action->conf;
4906         uint32_t dst_width = mlx5_flow_item_field_width(config,
4907                                 action_modify_field->dst.field);
4908         uint32_t src_width = mlx5_flow_item_field_width(config,
4909                                 action_modify_field->src.field);
4910
4911         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4912         if (ret)
4913                 return ret;
4914
4915         if (action_modify_field->width == 0)
4916                 return rte_flow_error_set(error, EINVAL,
4917                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4918                                 "no bits are requested to be modified");
4919         else if (action_modify_field->width > dst_width ||
4920                  action_modify_field->width > src_width)
4921                 return rte_flow_error_set(error, EINVAL,
4922                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4923                                 "cannot modify more bits than"
4924                                 " the width of a field");
4925         if (action_modify_field->dst.field != RTE_FLOW_FIELD_VALUE &&
4926             action_modify_field->dst.field != RTE_FLOW_FIELD_POINTER) {
4927                 if ((action_modify_field->dst.offset +
4928                      action_modify_field->width > dst_width) ||
4929                     (action_modify_field->dst.offset % 32))
4930                         return rte_flow_error_set(error, EINVAL,
4931                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4932                                         "destination offset is too big"
4933                                         " or not aligned to 4 bytes");
4934                 if (action_modify_field->dst.level &&
4935                     action_modify_field->dst.field != RTE_FLOW_FIELD_TAG)
4936                         return rte_flow_error_set(error, ENOTSUP,
4937                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4938                                         "inner header fields modification"
4939                                         " is not supported");
4940         }
4941         if (action_modify_field->src.field != RTE_FLOW_FIELD_VALUE &&
4942             action_modify_field->src.field != RTE_FLOW_FIELD_POINTER) {
4943                 if (!attr->transfer && !attr->group)
4944                         return rte_flow_error_set(error, ENOTSUP,
4945                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4946                                         "modify field action is not"
4947                                         " supported for group 0");
4948                 if ((action_modify_field->src.offset +
4949                      action_modify_field->width > src_width) ||
4950                     (action_modify_field->src.offset % 32))
4951                         return rte_flow_error_set(error, EINVAL,
4952                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4953                                         "source offset is too big"
4954                                         " or not aligned to 4 bytes");
4955                 if (action_modify_field->src.level &&
4956                     action_modify_field->src.field != RTE_FLOW_FIELD_TAG)
4957                         return rte_flow_error_set(error, ENOTSUP,
4958                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
4959                                         "inner header fields modification"
4960                                         " is not supported");
4961         }
4962         if ((action_modify_field->dst.field ==
4963              action_modify_field->src.field) &&
4964             (action_modify_field->dst.level ==
4965              action_modify_field->src.level))
4966                 return rte_flow_error_set(error, EINVAL,
4967                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4968                                 "source and destination fields"
4969                                 " cannot be the same");
4970         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VALUE ||
4971             action_modify_field->dst.field == RTE_FLOW_FIELD_POINTER)
4972                 return rte_flow_error_set(error, EINVAL,
4973                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4974                                 "immediate value or a pointer to it"
4975                                 " cannot be used as a destination");
4976         if (action_modify_field->dst.field == RTE_FLOW_FIELD_START ||
4977             action_modify_field->src.field == RTE_FLOW_FIELD_START)
4978                 return rte_flow_error_set(error, ENOTSUP,
4979                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4980                                 "modifications of an arbitrary"
4981                                 " place in a packet is not supported");
4982         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VLAN_TYPE ||
4983             action_modify_field->src.field == RTE_FLOW_FIELD_VLAN_TYPE)
4984                 return rte_flow_error_set(error, ENOTSUP,
4985                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4986                                 "modifications of the 802.1Q Tag"
4987                                 " Identifier is not supported");
4988         if (action_modify_field->dst.field == RTE_FLOW_FIELD_VXLAN_VNI ||
4989             action_modify_field->src.field == RTE_FLOW_FIELD_VXLAN_VNI)
4990                 return rte_flow_error_set(error, ENOTSUP,
4991                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4992                                 "modifications of the VXLAN Network"
4993                                 " Identifier is not supported");
4994         if (action_modify_field->dst.field == RTE_FLOW_FIELD_GENEVE_VNI ||
4995             action_modify_field->src.field == RTE_FLOW_FIELD_GENEVE_VNI)
4996                 return rte_flow_error_set(error, ENOTSUP,
4997                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
4998                                 "modifications of the GENEVE Network"
4999                                 " Identifier is not supported");
5000         if (action_modify_field->dst.field == RTE_FLOW_FIELD_MARK ||
5001             action_modify_field->src.field == RTE_FLOW_FIELD_MARK ||
5002             action_modify_field->dst.field == RTE_FLOW_FIELD_META ||
5003             action_modify_field->src.field == RTE_FLOW_FIELD_META) {
5004                 if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
5005                     !mlx5_flow_ext_mreg_supported(dev))
5006                         return rte_flow_error_set(error, ENOTSUP,
5007                                         RTE_FLOW_ERROR_TYPE_ACTION, action,
5008                                         "cannot modify mark or metadata without"
5009                                         " extended metadata register support");
5010         }
5011         if (action_modify_field->operation != RTE_FLOW_MODIFY_SET)
5012                 return rte_flow_error_set(error, ENOTSUP,
5013                                 RTE_FLOW_ERROR_TYPE_ACTION, action,
5014                                 "add and sub operations"
5015                                 " are not supported");
5016         return (action_modify_field->width / 32) +
5017                !!(action_modify_field->width % 32);
5018 }
5019
5020 /**
5021  * Validate jump action.
5022  *
5023  * @param[in] action
5024  *   Pointer to the jump action.
5025  * @param[in] action_flags
5026  *   Holds the actions detected until now.
5027  * @param[in] attributes
5028  *   Pointer to flow attributes
5029  * @param[in] external
5030  *   Action belongs to flow rule created by request external to PMD.
5031  * @param[out] error
5032  *   Pointer to error structure.
5033  *
5034  * @return
5035  *   0 on success, a negative errno value otherwise and rte_errno is set.
5036  */
5037 static int
5038 flow_dv_validate_action_jump(struct rte_eth_dev *dev,
5039                              const struct mlx5_flow_tunnel *tunnel,
5040                              const struct rte_flow_action *action,
5041                              uint64_t action_flags,
5042                              const struct rte_flow_attr *attributes,
5043                              bool external, struct rte_flow_error *error)
5044 {
5045         uint32_t target_group, table;
5046         int ret = 0;
5047         struct flow_grp_info grp_info = {
5048                 .external = !!external,
5049                 .transfer = !!attributes->transfer,
5050                 .fdb_def_rule = 1,
5051                 .std_tbl_fix = 0
5052         };
5053         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
5054                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5055                 return rte_flow_error_set(error, EINVAL,
5056                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5057                                           "can't have 2 fate actions in"
5058                                           " same flow");
5059         if (!action->conf)
5060                 return rte_flow_error_set(error, EINVAL,
5061                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
5062                                           NULL, "action configuration not set");
5063         target_group =
5064                 ((const struct rte_flow_action_jump *)action->conf)->group;
5065         ret = mlx5_flow_group_to_table(dev, tunnel, target_group, &table,
5066                                        &grp_info, error);
5067         if (ret)
5068                 return ret;
5069         if (attributes->group == target_group &&
5070             !(action_flags & (MLX5_FLOW_ACTION_TUNNEL_SET |
5071                               MLX5_FLOW_ACTION_TUNNEL_MATCH)))
5072                 return rte_flow_error_set(error, EINVAL,
5073                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5074                                           "target group must be other than"
5075                                           " the current flow group");
5076         return 0;
5077 }
5078
5079 /*
5080  * Validate the port_id action.
5081  *
5082  * @param[in] dev
5083  *   Pointer to rte_eth_dev structure.
5084  * @param[in] action_flags
5085  *   Bit-fields that holds the actions detected until now.
5086  * @param[in] action
5087  *   Port_id RTE action structure.
5088  * @param[in] attr
5089  *   Attributes of flow that includes this action.
5090  * @param[out] error
5091  *   Pointer to error structure.
5092  *
5093  * @return
5094  *   0 on success, a negative errno value otherwise and rte_errno is set.
5095  */
5096 static int
5097 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
5098                                 uint64_t action_flags,
5099                                 const struct rte_flow_action *action,
5100                                 const struct rte_flow_attr *attr,
5101                                 struct rte_flow_error *error)
5102 {
5103         const struct rte_flow_action_port_id *port_id;
5104         struct mlx5_priv *act_priv;
5105         struct mlx5_priv *dev_priv;
5106         uint16_t port;
5107
5108         if (!attr->transfer)
5109                 return rte_flow_error_set(error, ENOTSUP,
5110                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5111                                           NULL,
5112                                           "port id action is valid in transfer"
5113                                           " mode only");
5114         if (!action || !action->conf)
5115                 return rte_flow_error_set(error, ENOTSUP,
5116                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
5117                                           NULL,
5118                                           "port id action parameters must be"
5119                                           " specified");
5120         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
5121                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5122                 return rte_flow_error_set(error, EINVAL,
5123                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5124                                           "can have only one fate actions in"
5125                                           " a flow");
5126         dev_priv = mlx5_dev_to_eswitch_info(dev);
5127         if (!dev_priv)
5128                 return rte_flow_error_set(error, rte_errno,
5129                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5130                                           NULL,
5131                                           "failed to obtain E-Switch info");
5132         port_id = action->conf;
5133         port = port_id->original ? dev->data->port_id : port_id->id;
5134         act_priv = mlx5_port_to_eswitch_info(port, false);
5135         if (!act_priv)
5136                 return rte_flow_error_set
5137                                 (error, rte_errno,
5138                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
5139                                  "failed to obtain E-Switch port id for port");
5140         if (act_priv->domain_id != dev_priv->domain_id)
5141                 return rte_flow_error_set
5142                                 (error, EINVAL,
5143                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5144                                  "port does not belong to"
5145                                  " E-Switch being configured");
5146         return 0;
5147 }
5148
5149 /**
5150  * Get the maximum number of modify header actions.
5151  *
5152  * @param dev
5153  *   Pointer to rte_eth_dev structure.
5154  * @param root
5155  *   Whether action is on root table.
5156  *
5157  * @return
5158  *   Max number of modify header actions device can support.
5159  */
5160 static inline unsigned int
5161 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
5162                               bool root)
5163 {
5164         /*
5165          * There's no way to directly query the max capacity from FW.
5166          * The maximal value on root table should be assumed to be supported.
5167          */
5168         if (!root)
5169                 return MLX5_MAX_MODIFY_NUM;
5170         else
5171                 return MLX5_ROOT_TBL_MODIFY_NUM;
5172 }
5173
5174 /**
5175  * Validate the meter action.
5176  *
5177  * @param[in] dev
5178  *   Pointer to rte_eth_dev structure.
5179  * @param[in] action_flags
5180  *   Bit-fields that holds the actions detected until now.
5181  * @param[in] action
5182  *   Pointer to the meter action.
5183  * @param[in] attr
5184  *   Attributes of flow that includes this action.
5185  * @param[in] port_id_item
5186  *   Pointer to item indicating port id.
5187  * @param[out] error
5188  *   Pointer to error structure.
5189  *
5190  * @return
5191  *   0 on success, a negative errno value otherwise and rte_ernno is set.
5192  */
5193 static int
5194 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
5195                                 uint64_t action_flags,
5196                                 const struct rte_flow_action *action,
5197                                 const struct rte_flow_attr *attr,
5198                                 const struct rte_flow_item *port_id_item,
5199                                 bool *def_policy,
5200                                 struct rte_flow_error *error)
5201 {
5202         struct mlx5_priv *priv = dev->data->dev_private;
5203         const struct rte_flow_action_meter *am = action->conf;
5204         struct mlx5_flow_meter_info *fm;
5205         struct mlx5_flow_meter_policy *mtr_policy;
5206         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
5207
5208         if (!am)
5209                 return rte_flow_error_set(error, EINVAL,
5210                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5211                                           "meter action conf is NULL");
5212
5213         if (action_flags & MLX5_FLOW_ACTION_METER)
5214                 return rte_flow_error_set(error, ENOTSUP,
5215                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5216                                           "meter chaining not support");
5217         if (action_flags & MLX5_FLOW_ACTION_JUMP)
5218                 return rte_flow_error_set(error, ENOTSUP,
5219                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5220                                           "meter with jump not support");
5221         if (!priv->mtr_en)
5222                 return rte_flow_error_set(error, ENOTSUP,
5223                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5224                                           NULL,
5225                                           "meter action not supported");
5226         fm = mlx5_flow_meter_find(priv, am->mtr_id, NULL);
5227         if (!fm)
5228                 return rte_flow_error_set(error, EINVAL,
5229                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5230                                           "Meter not found");
5231         /* aso meter can always be shared by different domains */
5232         if (fm->ref_cnt && !priv->sh->meter_aso_en &&
5233             !(fm->transfer == attr->transfer ||
5234               (!fm->ingress && !attr->ingress && attr->egress) ||
5235               (!fm->egress && !attr->egress && attr->ingress)))
5236                 return rte_flow_error_set(error, EINVAL,
5237                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5238                         "Flow attributes domain are either invalid "
5239                         "or have a domain conflict with current "
5240                         "meter attributes");
5241         if (fm->def_policy) {
5242                 if (!((attr->transfer &&
5243                         mtrmng->def_policy[MLX5_MTR_DOMAIN_TRANSFER]) ||
5244                         (attr->egress &&
5245                         mtrmng->def_policy[MLX5_MTR_DOMAIN_EGRESS]) ||
5246                         (attr->ingress &&
5247                         mtrmng->def_policy[MLX5_MTR_DOMAIN_INGRESS])))
5248                         return rte_flow_error_set(error, EINVAL,
5249                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5250                                           "Flow attributes domain "
5251                                           "have a conflict with current "
5252                                           "meter domain attributes");
5253                 *def_policy = true;
5254         } else {
5255                 mtr_policy = mlx5_flow_meter_policy_find(dev,
5256                                                 fm->policy_id, NULL);
5257                 if (!mtr_policy)
5258                         return rte_flow_error_set(error, EINVAL,
5259                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5260                                           "Invalid policy id for meter ");
5261                 if (!((attr->transfer && mtr_policy->transfer) ||
5262                         (attr->egress && mtr_policy->egress) ||
5263                         (attr->ingress && mtr_policy->ingress)))
5264                         return rte_flow_error_set(error, EINVAL,
5265                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5266                                           "Flow attributes domain "
5267                                           "have a conflict with current "
5268                                           "meter domain attributes");
5269                 if (attr->transfer && mtr_policy->dev) {
5270                         /**
5271                          * When policy has fate action of port_id,
5272                          * the flow should have the same src port as policy.
5273                          */
5274                         struct mlx5_priv *policy_port_priv =
5275                                         mtr_policy->dev->data->dev_private;
5276                         int32_t flow_src_port = priv->representor_id;
5277
5278                         if (port_id_item) {
5279                                 const struct rte_flow_item_port_id *spec =
5280                                                         port_id_item->spec;
5281                                 struct mlx5_priv *port_priv =
5282                                         mlx5_port_to_eswitch_info(spec->id,
5283                                                                   false);
5284                                 if (!port_priv)
5285                                         return rte_flow_error_set(error,
5286                                                 rte_errno,
5287                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5288                                                 spec,
5289                                                 "Failed to get port info.");
5290                                 flow_src_port = port_priv->representor_id;
5291                         }
5292                         if (flow_src_port != policy_port_priv->representor_id)
5293                                 return rte_flow_error_set(error,
5294                                                 rte_errno,
5295                                                 RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
5296                                                 NULL,
5297                                                 "Flow and meter policy "
5298                                                 "have different src port.");
5299                 }
5300                 *def_policy = false;
5301         }
5302         return 0;
5303 }
5304
5305 /**
5306  * Validate the age action.
5307  *
5308  * @param[in] action_flags
5309  *   Holds the actions detected until now.
5310  * @param[in] action
5311  *   Pointer to the age action.
5312  * @param[in] dev
5313  *   Pointer to the Ethernet device structure.
5314  * @param[out] error
5315  *   Pointer to error structure.
5316  *
5317  * @return
5318  *   0 on success, a negative errno value otherwise and rte_errno is set.
5319  */
5320 static int
5321 flow_dv_validate_action_age(uint64_t action_flags,
5322                             const struct rte_flow_action *action,
5323                             struct rte_eth_dev *dev,
5324                             struct rte_flow_error *error)
5325 {
5326         struct mlx5_priv *priv = dev->data->dev_private;
5327         const struct rte_flow_action_age *age = action->conf;
5328
5329         if (!priv->config.devx || (priv->sh->cmng.counter_fallback &&
5330             !priv->sh->aso_age_mng))
5331                 return rte_flow_error_set(error, ENOTSUP,
5332                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5333                                           NULL,
5334                                           "age action not supported");
5335         if (!(action->conf))
5336                 return rte_flow_error_set(error, EINVAL,
5337                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5338                                           "configuration cannot be null");
5339         if (!(age->timeout))
5340                 return rte_flow_error_set(error, EINVAL,
5341                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5342                                           "invalid timeout value 0");
5343         if (action_flags & MLX5_FLOW_ACTION_AGE)
5344                 return rte_flow_error_set(error, EINVAL,
5345                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5346                                           "duplicate age actions set");
5347         return 0;
5348 }
5349
5350 /**
5351  * Validate the modify-header IPv4 DSCP actions.
5352  *
5353  * @param[in] action_flags
5354  *   Holds the actions detected until now.
5355  * @param[in] action
5356  *   Pointer to the modify action.
5357  * @param[in] item_flags
5358  *   Holds the items detected.
5359  * @param[out] error
5360  *   Pointer to error structure.
5361  *
5362  * @return
5363  *   0 on success, a negative errno value otherwise and rte_errno is set.
5364  */
5365 static int
5366 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
5367                                          const struct rte_flow_action *action,
5368                                          const uint64_t item_flags,
5369                                          struct rte_flow_error *error)
5370 {
5371         int ret = 0;
5372
5373         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5374         if (!ret) {
5375                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
5376                         return rte_flow_error_set(error, EINVAL,
5377                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5378                                                   NULL,
5379                                                   "no ipv4 item in pattern");
5380         }
5381         return ret;
5382 }
5383
5384 /**
5385  * Validate the modify-header IPv6 DSCP actions.
5386  *
5387  * @param[in] action_flags
5388  *   Holds the actions detected until now.
5389  * @param[in] action
5390  *   Pointer to the modify action.
5391  * @param[in] item_flags
5392  *   Holds the items detected.
5393  * @param[out] error
5394  *   Pointer to error structure.
5395  *
5396  * @return
5397  *   0 on success, a negative errno value otherwise and rte_errno is set.
5398  */
5399 static int
5400 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
5401                                          const struct rte_flow_action *action,
5402                                          const uint64_t item_flags,
5403                                          struct rte_flow_error *error)
5404 {
5405         int ret = 0;
5406
5407         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
5408         if (!ret) {
5409                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
5410                         return rte_flow_error_set(error, EINVAL,
5411                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5412                                                   NULL,
5413                                                   "no ipv6 item in pattern");
5414         }
5415         return ret;
5416 }
5417
5418 int
5419 flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
5420                         struct mlx5_list_entry *entry, void *cb_ctx)
5421 {
5422         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5423         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5424         struct mlx5_flow_dv_modify_hdr_resource *resource =
5425                                   container_of(entry, typeof(*resource), entry);
5426         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5427
5428         key_len += ref->actions_num * sizeof(ref->actions[0]);
5429         return ref->actions_num != resource->actions_num ||
5430                memcmp(&ref->ft_type, &resource->ft_type, key_len);
5431 }
5432
5433 static struct mlx5_indexed_pool *
5434 flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
5435 {
5436         struct mlx5_indexed_pool *ipool = __atomic_load_n
5437                                      (&sh->mdh_ipools[index], __ATOMIC_SEQ_CST);
5438
5439         if (!ipool) {
5440                 struct mlx5_indexed_pool *expected = NULL;
5441                 struct mlx5_indexed_pool_config cfg =
5442                     (struct mlx5_indexed_pool_config) {
5443                        .size = sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
5444                                                                    (index + 1) *
5445                                            sizeof(struct mlx5_modification_cmd),
5446                        .trunk_size = 64,
5447                        .grow_trunk = 3,
5448                        .grow_shift = 2,
5449                        .need_lock = 1,
5450                        .release_mem_en = !!sh->reclaim_mode,
5451                        .per_core_cache = sh->reclaim_mode ? 0 : (1 << 16),
5452                        .malloc = mlx5_malloc,
5453                        .free = mlx5_free,
5454                        .type = "mlx5_modify_action_resource",
5455                 };
5456
5457                 cfg.size = RTE_ALIGN(cfg.size, sizeof(ipool));
5458                 ipool = mlx5_ipool_create(&cfg);
5459                 if (!ipool)
5460                         return NULL;
5461                 if (!__atomic_compare_exchange_n(&sh->mdh_ipools[index],
5462                                                  &expected, ipool, false,
5463                                                  __ATOMIC_SEQ_CST,
5464                                                  __ATOMIC_SEQ_CST)) {
5465                         mlx5_ipool_destroy(ipool);
5466                         ipool = __atomic_load_n(&sh->mdh_ipools[index],
5467                                                 __ATOMIC_SEQ_CST);
5468                 }
5469         }
5470         return ipool;
5471 }
5472
5473 struct mlx5_list_entry *
5474 flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
5475 {
5476         struct mlx5_dev_ctx_shared *sh = tool_ctx;
5477         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5478         struct mlx5dv_dr_domain *ns;
5479         struct mlx5_flow_dv_modify_hdr_resource *entry;
5480         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5481         struct mlx5_indexed_pool *ipool = flow_dv_modify_ipool_get(sh,
5482                                                           ref->actions_num - 1);
5483         int ret;
5484         uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
5485         uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
5486         uint32_t idx;
5487
5488         if (unlikely(!ipool)) {
5489                 rte_flow_error_set(ctx->error, ENOMEM,
5490                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5491                                    NULL, "cannot allocate modify ipool");
5492                 return NULL;
5493         }
5494         entry = mlx5_ipool_zmalloc(ipool, &idx);
5495         if (!entry) {
5496                 rte_flow_error_set(ctx->error, ENOMEM,
5497                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5498                                    "cannot allocate resource memory");
5499                 return NULL;
5500         }
5501         rte_memcpy(&entry->ft_type,
5502                    RTE_PTR_ADD(ref, offsetof(typeof(*ref), ft_type)),
5503                    key_len + data_len);
5504         if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
5505                 ns = sh->fdb_domain;
5506         else if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
5507                 ns = sh->tx_domain;
5508         else
5509                 ns = sh->rx_domain;
5510         ret = mlx5_flow_os_create_flow_action_modify_header
5511                                         (sh->ctx, ns, entry,
5512                                          data_len, &entry->action);
5513         if (ret) {
5514                 mlx5_ipool_free(sh->mdh_ipools[ref->actions_num - 1], idx);
5515                 rte_flow_error_set(ctx->error, ENOMEM,
5516                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5517                                    NULL, "cannot create modification action");
5518                 return NULL;
5519         }
5520         entry->idx = idx;
5521         return &entry->entry;
5522 }
5523
5524 struct mlx5_list_entry *
5525 flow_dv_modify_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
5526                         void *cb_ctx)
5527 {
5528         struct mlx5_dev_ctx_shared *sh = tool_ctx;
5529         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
5530         struct mlx5_flow_dv_modify_hdr_resource *entry;
5531         struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
5532         uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
5533         uint32_t idx;
5534
5535         entry = mlx5_ipool_malloc(sh->mdh_ipools[ref->actions_num - 1],
5536                                   &idx);
5537         if (!entry) {
5538                 rte_flow_error_set(ctx->error, ENOMEM,
5539                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5540                                    "cannot allocate resource memory");
5541                 return NULL;
5542         }
5543         memcpy(entry, oentry, sizeof(*entry) + data_len);
5544         entry->idx = idx;
5545         return &entry->entry;
5546 }
5547
5548 void
5549 flow_dv_modify_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
5550 {
5551         struct mlx5_dev_ctx_shared *sh = tool_ctx;
5552         struct mlx5_flow_dv_modify_hdr_resource *res =
5553                 container_of(entry, typeof(*res), entry);
5554
5555         mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
5556 }
5557
5558 /**
5559  * Validate the sample action.
5560  *
5561  * @param[in, out] action_flags
5562  *   Holds the actions detected until now.
5563  * @param[in] action
5564  *   Pointer to the sample action.
5565  * @param[in] dev
5566  *   Pointer to the Ethernet device structure.
5567  * @param[in] attr
5568  *   Attributes of flow that includes this action.
5569  * @param[in] item_flags
5570  *   Holds the items detected.
5571  * @param[in] rss
5572  *   Pointer to the RSS action.
5573  * @param[out] sample_rss
5574  *   Pointer to the RSS action in sample action list.
5575  * @param[out] count
5576  *   Pointer to the COUNT action in sample action list.
5577  * @param[out] fdb_mirror_limit
5578  *   Pointer to the FDB mirror limitation flag.
5579  * @param[out] error
5580  *   Pointer to error structure.
5581  *
5582  * @return
5583  *   0 on success, a negative errno value otherwise and rte_errno is set.
5584  */
5585 static int
5586 flow_dv_validate_action_sample(uint64_t *action_flags,
5587                                const struct rte_flow_action *action,
5588                                struct rte_eth_dev *dev,
5589                                const struct rte_flow_attr *attr,
5590                                uint64_t item_flags,
5591                                const struct rte_flow_action_rss *rss,
5592                                const struct rte_flow_action_rss **sample_rss,
5593                                const struct rte_flow_action_count **count,
5594                                int *fdb_mirror_limit,
5595                                struct rte_flow_error *error)
5596 {
5597         struct mlx5_priv *priv = dev->data->dev_private;
5598         struct mlx5_dev_config *dev_conf = &priv->config;
5599         const struct rte_flow_action_sample *sample = action->conf;
5600         const struct rte_flow_action *act;
5601         uint64_t sub_action_flags = 0;
5602         uint16_t queue_index = 0xFFFF;
5603         int actions_n = 0;
5604         int ret;
5605
5606         if (!sample)
5607                 return rte_flow_error_set(error, EINVAL,
5608                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5609                                           "configuration cannot be NULL");
5610         if (sample->ratio == 0)
5611                 return rte_flow_error_set(error, EINVAL,
5612                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5613                                           "ratio value starts from 1");
5614         if (!priv->config.devx || (sample->ratio > 0 && !priv->sampler_en))
5615                 return rte_flow_error_set(error, ENOTSUP,
5616                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5617                                           NULL,
5618                                           "sample action not supported");
5619         if (*action_flags & MLX5_FLOW_ACTION_SAMPLE)
5620                 return rte_flow_error_set(error, EINVAL,
5621                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5622                                           "Multiple sample actions not "
5623                                           "supported");
5624         if (*action_flags & MLX5_FLOW_ACTION_METER)
5625                 return rte_flow_error_set(error, EINVAL,
5626                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5627                                           "wrong action order, meter should "
5628                                           "be after sample action");
5629         if (*action_flags & MLX5_FLOW_ACTION_JUMP)
5630                 return rte_flow_error_set(error, EINVAL,
5631                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
5632                                           "wrong action order, jump should "
5633                                           "be after sample action");
5634         act = sample->actions;
5635         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
5636                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5637                         return rte_flow_error_set(error, ENOTSUP,
5638                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5639                                                   act, "too many actions");
5640                 switch (act->type) {
5641                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5642                         ret = mlx5_flow_validate_action_queue(act,
5643                                                               sub_action_flags,
5644                                                               dev,
5645                                                               attr, error);
5646                         if (ret < 0)
5647                                 return ret;
5648                         queue_index = ((const struct rte_flow_action_queue *)
5649                                                         (act->conf))->index;
5650                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
5651                         ++actions_n;
5652                         break;
5653                 case RTE_FLOW_ACTION_TYPE_RSS:
5654                         *sample_rss = act->conf;
5655                         ret = mlx5_flow_validate_action_rss(act,
5656                                                             sub_action_flags,
5657                                                             dev, attr,
5658                                                             item_flags,
5659                                                             error);
5660                         if (ret < 0)
5661                                 return ret;
5662                         if (rss && *sample_rss &&
5663                             ((*sample_rss)->level != rss->level ||
5664                             (*sample_rss)->types != rss->types))
5665                                 return rte_flow_error_set(error, ENOTSUP,
5666                                         RTE_FLOW_ERROR_TYPE_ACTION,
5667                                         NULL,
5668                                         "Can't use the different RSS types "
5669                                         "or level in the same flow");
5670                         if (*sample_rss != NULL && (*sample_rss)->queue_num)
5671                                 queue_index = (*sample_rss)->queue[0];
5672                         sub_action_flags |= MLX5_FLOW_ACTION_RSS;
5673                         ++actions_n;
5674                         break;
5675                 case RTE_FLOW_ACTION_TYPE_MARK:
5676                         ret = flow_dv_validate_action_mark(dev, act,
5677                                                            sub_action_flags,
5678                                                            attr, error);
5679                         if (ret < 0)
5680                                 return ret;
5681                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
5682                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
5683                                                 MLX5_FLOW_ACTION_MARK_EXT;
5684                         else
5685                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
5686                         ++actions_n;
5687                         break;
5688                 case RTE_FLOW_ACTION_TYPE_COUNT:
5689                         ret = flow_dv_validate_action_count
5690                                 (dev, is_shared_action_count(act),
5691                                  *action_flags | sub_action_flags,
5692                                  error);
5693                         if (ret < 0)
5694                                 return ret;
5695                         *count = act->conf;
5696                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
5697                         *action_flags |= MLX5_FLOW_ACTION_COUNT;
5698                         ++actions_n;
5699                         break;
5700                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5701                         ret = flow_dv_validate_action_port_id(dev,
5702                                                               sub_action_flags,
5703                                                               act,
5704                                                               attr,
5705                                                               error);
5706                         if (ret)
5707                                 return ret;
5708                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5709                         ++actions_n;
5710                         break;
5711                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5712                         ret = flow_dv_validate_action_raw_encap_decap
5713                                 (dev, NULL, act->conf, attr, &sub_action_flags,
5714                                  &actions_n, action, item_flags, error);
5715                         if (ret < 0)
5716                                 return ret;
5717                         ++actions_n;
5718                         break;
5719                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5720                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5721                         ret = flow_dv_validate_action_l2_encap(dev,
5722                                                                sub_action_flags,
5723                                                                act, attr,
5724                                                                error);
5725                         if (ret < 0)
5726                                 return ret;
5727                         sub_action_flags |= MLX5_FLOW_ACTION_ENCAP;
5728                         ++actions_n;
5729                         break;
5730                 default:
5731                         return rte_flow_error_set(error, ENOTSUP,
5732                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5733                                                   NULL,
5734                                                   "Doesn't support optional "
5735                                                   "action");
5736                 }
5737         }
5738         if (attr->ingress && !attr->transfer) {
5739                 if (!(sub_action_flags & (MLX5_FLOW_ACTION_QUEUE |
5740                                           MLX5_FLOW_ACTION_RSS)))
5741                         return rte_flow_error_set(error, EINVAL,
5742                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5743                                                   NULL,
5744                                                   "Ingress must has a dest "
5745                                                   "QUEUE for Sample");
5746         } else if (attr->egress && !attr->transfer) {
5747                 return rte_flow_error_set(error, ENOTSUP,
5748                                           RTE_FLOW_ERROR_TYPE_ACTION,
5749                                           NULL,
5750                                           "Sample Only support Ingress "
5751                                           "or E-Switch");
5752         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
5753                 MLX5_ASSERT(attr->transfer);
5754                 if (sample->ratio > 1)
5755                         return rte_flow_error_set(error, ENOTSUP,
5756                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5757                                                   NULL,
5758                                                   "E-Switch doesn't support "
5759                                                   "any optional action "
5760                                                   "for sampling");
5761                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
5762                         return rte_flow_error_set(error, ENOTSUP,
5763                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5764                                                   NULL,
5765                                                   "unsupported action QUEUE");
5766                 if (sub_action_flags & MLX5_FLOW_ACTION_RSS)
5767                         return rte_flow_error_set(error, ENOTSUP,
5768                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5769                                                   NULL,
5770                                                   "unsupported action QUEUE");
5771                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
5772                         return rte_flow_error_set(error, EINVAL,
5773                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5774                                                   NULL,
5775                                                   "E-Switch must has a dest "
5776                                                   "port for mirroring");
5777                 if (!priv->config.hca_attr.reg_c_preserve &&
5778                      priv->representor_id != UINT16_MAX)
5779                         *fdb_mirror_limit = 1;
5780         }
5781         /* Continue validation for Xcap actions.*/
5782         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
5783             (queue_index == 0xFFFF ||
5784              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
5785                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
5786                      MLX5_FLOW_XCAP_ACTIONS)
5787                         return rte_flow_error_set(error, ENOTSUP,
5788                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5789                                                   NULL, "encap and decap "
5790                                                   "combination aren't "
5791                                                   "supported");
5792                 if (!attr->transfer && attr->ingress && (sub_action_flags &
5793                                                         MLX5_FLOW_ACTION_ENCAP))
5794                         return rte_flow_error_set(error, ENOTSUP,
5795                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5796                                                   NULL, "encap is not supported"
5797                                                   " for ingress traffic");
5798         }
5799         return 0;
5800 }
5801
5802 /**
5803  * Find existing modify-header resource or create and register a new one.
5804  *
5805  * @param dev[in, out]
5806  *   Pointer to rte_eth_dev structure.
5807  * @param[in, out] resource
5808  *   Pointer to modify-header resource.
5809  * @parm[in, out] dev_flow
5810  *   Pointer to the dev_flow.
5811  * @param[out] error
5812  *   pointer to error structure.
5813  *
5814  * @return
5815  *   0 on success otherwise -errno and errno is set.
5816  */
5817 static int
5818 flow_dv_modify_hdr_resource_register
5819                         (struct rte_eth_dev *dev,
5820                          struct mlx5_flow_dv_modify_hdr_resource *resource,
5821                          struct mlx5_flow *dev_flow,
5822                          struct rte_flow_error *error)
5823 {
5824         struct mlx5_priv *priv = dev->data->dev_private;
5825         struct mlx5_dev_ctx_shared *sh = priv->sh;
5826         uint32_t key_len = sizeof(*resource) -
5827                            offsetof(typeof(*resource), ft_type) +
5828                            resource->actions_num * sizeof(resource->actions[0]);
5829         struct mlx5_list_entry *entry;
5830         struct mlx5_flow_cb_ctx ctx = {
5831                 .error = error,
5832                 .data = resource,
5833         };
5834         struct mlx5_hlist *modify_cmds;
5835         uint64_t key64;
5836
5837         modify_cmds = flow_dv_hlist_prepare(sh, &sh->modify_cmds,
5838                                 "hdr_modify",
5839                                 MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
5840                                 true, false, sh,
5841                                 flow_dv_modify_create_cb,
5842                                 flow_dv_modify_match_cb,
5843                                 flow_dv_modify_remove_cb,
5844                                 flow_dv_modify_clone_cb,
5845                                 flow_dv_modify_clone_free_cb);
5846         if (unlikely(!modify_cmds))
5847                 return -rte_errno;
5848         resource->root = !dev_flow->dv.group;
5849         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
5850                                                                 resource->root))
5851                 return rte_flow_error_set(error, EOVERFLOW,
5852                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5853                                           "too many modify header items");
5854         key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0);
5855         entry = mlx5_hlist_register(modify_cmds, key64, &ctx);
5856         if (!entry)
5857                 return -rte_errno;
5858         resource = container_of(entry, typeof(*resource), entry);
5859         dev_flow->handle->dvh.modify_hdr = resource;
5860         return 0;
5861 }
5862
5863 /**
5864  * Get DV flow counter by index.
5865  *
5866  * @param[in] dev
5867  *   Pointer to the Ethernet device structure.
5868  * @param[in] idx
5869  *   mlx5 flow counter index in the container.
5870  * @param[out] ppool
5871  *   mlx5 flow counter pool in the container.
5872  *
5873  * @return
5874  *   Pointer to the counter, NULL otherwise.
5875  */
5876 static struct mlx5_flow_counter *
5877 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
5878                            uint32_t idx,
5879                            struct mlx5_flow_counter_pool **ppool)
5880 {
5881         struct mlx5_priv *priv = dev->data->dev_private;
5882         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5883         struct mlx5_flow_counter_pool *pool;
5884
5885         /* Decrease to original index and clear shared bit. */
5886         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
5887         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cmng->n);
5888         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
5889         MLX5_ASSERT(pool);
5890         if (ppool)
5891                 *ppool = pool;
5892         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
5893 }
5894
5895 /**
5896  * Check the devx counter belongs to the pool.
5897  *
5898  * @param[in] pool
5899  *   Pointer to the counter pool.
5900  * @param[in] id
5901  *   The counter devx ID.
5902  *
5903  * @return
5904  *   True if counter belongs to the pool, false otherwise.
5905  */
5906 static bool
5907 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
5908 {
5909         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
5910                    MLX5_COUNTERS_PER_POOL;
5911
5912         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
5913                 return true;
5914         return false;
5915 }
5916
5917 /**
5918  * Get a pool by devx counter ID.
5919  *
5920  * @param[in] cmng
5921  *   Pointer to the counter management.
5922  * @param[in] id
5923  *   The counter devx ID.
5924  *
5925  * @return
5926  *   The counter pool pointer if exists, NULL otherwise,
5927  */
5928 static struct mlx5_flow_counter_pool *
5929 flow_dv_find_pool_by_id(struct mlx5_flow_counter_mng *cmng, int id)
5930 {
5931         uint32_t i;
5932         struct mlx5_flow_counter_pool *pool = NULL;
5933
5934         rte_spinlock_lock(&cmng->pool_update_sl);
5935         /* Check last used pool. */
5936         if (cmng->last_pool_idx != POOL_IDX_INVALID &&
5937             flow_dv_is_counter_in_pool(cmng->pools[cmng->last_pool_idx], id)) {
5938                 pool = cmng->pools[cmng->last_pool_idx];
5939                 goto out;
5940         }
5941         /* ID out of range means no suitable pool in the container. */
5942         if (id > cmng->max_id || id < cmng->min_id)
5943                 goto out;
5944         /*
5945          * Find the pool from the end of the container, since mostly counter
5946          * ID is sequence increasing, and the last pool should be the needed
5947          * one.
5948          */
5949         i = cmng->n_valid;
5950         while (i--) {
5951                 struct mlx5_flow_counter_pool *pool_tmp = cmng->pools[i];
5952
5953                 if (flow_dv_is_counter_in_pool(pool_tmp, id)) {
5954                         pool = pool_tmp;
5955                         break;
5956                 }
5957         }
5958 out:
5959         rte_spinlock_unlock(&cmng->pool_update_sl);
5960         return pool;
5961 }
5962
5963 /**
5964  * Resize a counter container.
5965  *
5966  * @param[in] dev
5967  *   Pointer to the Ethernet device structure.
5968  *
5969  * @return
5970  *   0 on success, otherwise negative errno value and rte_errno is set.
5971  */
5972 static int
5973 flow_dv_container_resize(struct rte_eth_dev *dev)
5974 {
5975         struct mlx5_priv *priv = dev->data->dev_private;
5976         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
5977         void *old_pools = cmng->pools;
5978         uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
5979         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
5980         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
5981
5982         if (!pools) {
5983                 rte_errno = ENOMEM;
5984                 return -ENOMEM;
5985         }
5986         if (old_pools)
5987                 memcpy(pools, old_pools, cmng->n *
5988                                        sizeof(struct mlx5_flow_counter_pool *));
5989         cmng->n = resize;
5990         cmng->pools = pools;
5991         if (old_pools)
5992                 mlx5_free(old_pools);
5993         return 0;
5994 }
5995
5996 /**
5997  * Query a devx flow counter.
5998  *
5999  * @param[in] dev
6000  *   Pointer to the Ethernet device structure.
6001  * @param[in] counter
6002  *   Index to the flow counter.
6003  * @param[out] pkts
6004  *   The statistics value of packets.
6005  * @param[out] bytes
6006  *   The statistics value of bytes.
6007  *
6008  * @return
6009  *   0 on success, otherwise a negative errno value and rte_errno is set.
6010  */
6011 static inline int
6012 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
6013                      uint64_t *bytes)
6014 {
6015         struct mlx5_priv *priv = dev->data->dev_private;
6016         struct mlx5_flow_counter_pool *pool = NULL;
6017         struct mlx5_flow_counter *cnt;
6018         int offset;
6019
6020         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
6021         MLX5_ASSERT(pool);
6022         if (priv->sh->cmng.counter_fallback)
6023                 return mlx5_devx_cmd_flow_counter_query(cnt->dcs_when_active, 0,
6024                                         0, pkts, bytes, 0, NULL, NULL, 0);
6025         rte_spinlock_lock(&pool->sl);
6026         if (!pool->raw) {
6027                 *pkts = 0;
6028                 *bytes = 0;
6029         } else {
6030                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
6031                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
6032                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
6033         }
6034         rte_spinlock_unlock(&pool->sl);
6035         return 0;
6036 }
6037
6038 /**
6039  * Create and initialize a new counter pool.
6040  *
6041  * @param[in] dev
6042  *   Pointer to the Ethernet device structure.
6043  * @param[out] dcs
6044  *   The devX counter handle.
6045  * @param[in] age
6046  *   Whether the pool is for counter that was allocated for aging.
6047  * @param[in/out] cont_cur
6048  *   Pointer to the container pointer, it will be update in pool resize.
6049  *
6050  * @return
6051  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
6052  */
6053 static struct mlx5_flow_counter_pool *
6054 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
6055                     uint32_t age)
6056 {
6057         struct mlx5_priv *priv = dev->data->dev_private;
6058         struct mlx5_flow_counter_pool *pool;
6059         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6060         bool fallback = priv->sh->cmng.counter_fallback;
6061         uint32_t size = sizeof(*pool);
6062
6063         size += MLX5_COUNTERS_PER_POOL * MLX5_CNT_SIZE;
6064         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * MLX5_AGE_SIZE);
6065         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
6066         if (!pool) {
6067                 rte_errno = ENOMEM;
6068                 return NULL;
6069         }
6070         pool->raw = NULL;
6071         pool->is_aged = !!age;
6072         pool->query_gen = 0;
6073         pool->min_dcs = dcs;
6074         rte_spinlock_init(&pool->sl);
6075         rte_spinlock_init(&pool->csl);
6076         TAILQ_INIT(&pool->counters[0]);
6077         TAILQ_INIT(&pool->counters[1]);
6078         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
6079         rte_spinlock_lock(&cmng->pool_update_sl);
6080         pool->index = cmng->n_valid;
6081         if (pool->index == cmng->n && flow_dv_container_resize(dev)) {
6082                 mlx5_free(pool);
6083                 rte_spinlock_unlock(&cmng->pool_update_sl);
6084                 return NULL;
6085         }
6086         cmng->pools[pool->index] = pool;
6087         cmng->n_valid++;
6088         if (unlikely(fallback)) {
6089                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
6090
6091                 if (base < cmng->min_id)
6092                         cmng->min_id = base;
6093                 if (base > cmng->max_id)
6094                         cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
6095                 cmng->last_pool_idx = pool->index;
6096         }
6097         rte_spinlock_unlock(&cmng->pool_update_sl);
6098         return pool;
6099 }
6100
6101 /**
6102  * Prepare a new counter and/or a new counter pool.
6103  *
6104  * @param[in] dev
6105  *   Pointer to the Ethernet device structure.
6106  * @param[out] cnt_free
6107  *   Where to put the pointer of a new counter.
6108  * @param[in] age
6109  *   Whether the pool is for counter that was allocated for aging.
6110  *
6111  * @return
6112  *   The counter pool pointer and @p cnt_free is set on success,
6113  *   NULL otherwise and rte_errno is set.
6114  */
6115 static struct mlx5_flow_counter_pool *
6116 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
6117                              struct mlx5_flow_counter **cnt_free,
6118                              uint32_t age)
6119 {
6120         struct mlx5_priv *priv = dev->data->dev_private;
6121         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6122         struct mlx5_flow_counter_pool *pool;
6123         struct mlx5_counters tmp_tq;
6124         struct mlx5_devx_obj *dcs = NULL;
6125         struct mlx5_flow_counter *cnt;
6126         enum mlx5_counter_type cnt_type =
6127                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
6128         bool fallback = priv->sh->cmng.counter_fallback;
6129         uint32_t i;
6130
6131         if (fallback) {
6132                 /* bulk_bitmap must be 0 for single counter allocation. */
6133                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
6134                 if (!dcs)
6135                         return NULL;
6136                 pool = flow_dv_find_pool_by_id(cmng, dcs->id);
6137                 if (!pool) {
6138                         pool = flow_dv_pool_create(dev, dcs, age);
6139                         if (!pool) {
6140                                 mlx5_devx_cmd_destroy(dcs);
6141                                 return NULL;
6142                         }
6143                 }
6144                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
6145                 cnt = MLX5_POOL_GET_CNT(pool, i);
6146                 cnt->pool = pool;
6147                 cnt->dcs_when_free = dcs;
6148                 *cnt_free = cnt;
6149                 return pool;
6150         }
6151         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
6152         if (!dcs) {
6153                 rte_errno = ENODATA;
6154                 return NULL;
6155         }
6156         pool = flow_dv_pool_create(dev, dcs, age);
6157         if (!pool) {
6158                 mlx5_devx_cmd_destroy(dcs);
6159                 return NULL;
6160         }
6161         TAILQ_INIT(&tmp_tq);
6162         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
6163                 cnt = MLX5_POOL_GET_CNT(pool, i);
6164                 cnt->pool = pool;
6165                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
6166         }
6167         rte_spinlock_lock(&cmng->csl[cnt_type]);
6168         TAILQ_CONCAT(&cmng->counters[cnt_type], &tmp_tq, next);
6169         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6170         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
6171         (*cnt_free)->pool = pool;
6172         return pool;
6173 }
6174
6175 /**
6176  * Allocate a flow counter.
6177  *
6178  * @param[in] dev
6179  *   Pointer to the Ethernet device structure.
6180  * @param[in] age
6181  *   Whether the counter was allocated for aging.
6182  *
6183  * @return
6184  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6185  */
6186 static uint32_t
6187 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t age)
6188 {
6189         struct mlx5_priv *priv = dev->data->dev_private;
6190         struct mlx5_flow_counter_pool *pool = NULL;
6191         struct mlx5_flow_counter *cnt_free = NULL;
6192         bool fallback = priv->sh->cmng.counter_fallback;
6193         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
6194         enum mlx5_counter_type cnt_type =
6195                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
6196         uint32_t cnt_idx;
6197
6198         if (!priv->config.devx) {
6199                 rte_errno = ENOTSUP;
6200                 return 0;
6201         }
6202         /* Get free counters from container. */
6203         rte_spinlock_lock(&cmng->csl[cnt_type]);
6204         cnt_free = TAILQ_FIRST(&cmng->counters[cnt_type]);
6205         if (cnt_free)
6206                 TAILQ_REMOVE(&cmng->counters[cnt_type], cnt_free, next);
6207         rte_spinlock_unlock(&cmng->csl[cnt_type]);
6208         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, age))
6209                 goto err;
6210         pool = cnt_free->pool;
6211         if (fallback)
6212                 cnt_free->dcs_when_active = cnt_free->dcs_when_free;
6213         /* Create a DV counter action only in the first time usage. */
6214         if (!cnt_free->action) {
6215                 uint16_t offset;
6216                 struct mlx5_devx_obj *dcs;
6217                 int ret;
6218
6219                 if (!fallback) {
6220                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
6221                         dcs = pool->min_dcs;
6222                 } else {
6223                         offset = 0;
6224                         dcs = cnt_free->dcs_when_free;
6225                 }
6226                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
6227                                                             &cnt_free->action);
6228                 if (ret) {
6229                         rte_errno = errno;
6230                         goto err;
6231                 }
6232         }
6233         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
6234                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
6235         /* Update the counter reset values. */
6236         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
6237                                  &cnt_free->bytes))
6238                 goto err;
6239         if (!fallback && !priv->sh->cmng.query_thread_on)
6240                 /* Start the asynchronous batch query by the host thread. */
6241                 mlx5_set_query_alarm(priv->sh);
6242         /*
6243          * When the count action isn't shared (by ID), shared_info field is
6244          * used for indirect action API's refcnt.
6245          * When the counter action is not shared neither by ID nor by indirect
6246          * action API, shared info must be 1.
6247          */
6248         cnt_free->shared_info.refcnt = 1;
6249         return cnt_idx;
6250 err:
6251         if (cnt_free) {
6252                 cnt_free->pool = pool;
6253                 if (fallback)
6254                         cnt_free->dcs_when_free = cnt_free->dcs_when_active;
6255                 rte_spinlock_lock(&cmng->csl[cnt_type]);
6256                 TAILQ_INSERT_TAIL(&cmng->counters[cnt_type], cnt_free, next);
6257                 rte_spinlock_unlock(&cmng->csl[cnt_type]);
6258         }
6259         return 0;
6260 }
6261
6262 /**
6263  * Allocate a shared flow counter.
6264  *
6265  * @param[in] ctx
6266  *   Pointer to the shared counter configuration.
6267  * @param[in] data
6268  *   Pointer to save the allocated counter index.
6269  *
6270  * @return
6271  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6272  */
6273
6274 static int32_t
6275 flow_dv_counter_alloc_shared_cb(void *ctx, union mlx5_l3t_data *data)
6276 {
6277         struct mlx5_shared_counter_conf *conf = ctx;
6278         struct rte_eth_dev *dev = conf->dev;
6279         struct mlx5_flow_counter *cnt;
6280
6281         data->dword = flow_dv_counter_alloc(dev, 0);
6282         data->dword |= MLX5_CNT_SHARED_OFFSET;
6283         cnt = flow_dv_counter_get_by_idx(dev, data->dword, NULL);
6284         cnt->shared_info.id = conf->id;
6285         return 0;
6286 }
6287
6288 /**
6289  * Get a shared flow counter.
6290  *
6291  * @param[in] dev
6292  *   Pointer to the Ethernet device structure.
6293  * @param[in] id
6294  *   Counter identifier.
6295  *
6296  * @return
6297  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
6298  */
6299 static uint32_t
6300 flow_dv_counter_get_shared(struct rte_eth_dev *dev, uint32_t id)
6301 {
6302         struct mlx5_priv *priv = dev->data->dev_private;
6303         struct mlx5_shared_counter_conf conf = {
6304                 .dev = dev,
6305                 .id = id,
6306         };
6307         union mlx5_l3t_data data = {
6308                 .dword = 0,
6309         };
6310
6311         mlx5_l3t_prepare_entry(priv->sh->cnt_id_tbl, id, &data,
6312                                flow_dv_counter_alloc_shared_cb, &conf);
6313         return data.dword;
6314 }
6315
6316 /**
6317  * Get age param from counter index.
6318  *
6319  * @param[in] dev
6320  *   Pointer to the Ethernet device structure.
6321  * @param[in] counter
6322  *   Index to the counter handler.
6323  *
6324  * @return
6325  *   The aging parameter specified for the counter index.
6326  */
6327 static struct mlx5_age_param*
6328 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
6329                                 uint32_t counter)
6330 {
6331         struct mlx5_flow_counter *cnt;
6332         struct mlx5_flow_counter_pool *pool = NULL;
6333
6334         flow_dv_counter_get_by_idx(dev, counter, &pool);
6335         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
6336         cnt = MLX5_POOL_GET_CNT(pool, counter);
6337         return MLX5_CNT_TO_AGE(cnt);
6338 }
6339
6340 /**
6341  * Remove a flow counter from aged counter list.
6342  *
6343  * @param[in] dev
6344  *   Pointer to the Ethernet device structure.
6345  * @param[in] counter
6346  *   Index to the counter handler.
6347  * @param[in] cnt
6348  *   Pointer to the counter handler.
6349  */
6350 static void
6351 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
6352                                 uint32_t counter, struct mlx5_flow_counter *cnt)
6353 {
6354         struct mlx5_age_info *age_info;
6355         struct mlx5_age_param *age_param;
6356         struct mlx5_priv *priv = dev->data->dev_private;
6357         uint16_t expected = AGE_CANDIDATE;
6358
6359         age_info = GET_PORT_AGE_INFO(priv);
6360         age_param = flow_dv_counter_idx_get_age(dev, counter);
6361         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
6362                                          AGE_FREE, false, __ATOMIC_RELAXED,
6363                                          __ATOMIC_RELAXED)) {
6364                 /**
6365                  * We need the lock even it is age timeout,
6366                  * since counter may still in process.
6367                  */
6368                 rte_spinlock_lock(&age_info->aged_sl);
6369                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
6370                 rte_spinlock_unlock(&age_info->aged_sl);
6371                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
6372         }
6373 }
6374
6375 /**
6376  * Release a flow counter.
6377  *
6378  * @param[in] dev
6379  *   Pointer to the Ethernet device structure.
6380  * @param[in] counter
6381  *   Index to the counter handler.
6382  */
6383 static void
6384 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter)
6385 {
6386         struct mlx5_priv *priv = dev->data->dev_private;
6387         struct mlx5_flow_counter_pool *pool = NULL;
6388         struct mlx5_flow_counter *cnt;
6389         enum mlx5_counter_type cnt_type;
6390
6391         if (!counter)
6392                 return;
6393         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
6394         MLX5_ASSERT(pool);
6395         if (pool->is_aged) {
6396                 flow_dv_counter_remove_from_age(dev, counter, cnt);
6397         } else {
6398                 /*
6399                  * If the counter action is shared by ID, the l3t_clear_entry
6400                  * function reduces its references counter. If after the
6401                  * reduction the action is still referenced, the function
6402                  * returns here and does not release it.
6403                  */
6404                 if (IS_LEGACY_SHARED_CNT(counter) &&
6405                     mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
6406                                          cnt->shared_info.id))
6407                         return;
6408                 /*
6409                  * If the counter action is shared by indirect action API,
6410                  * the atomic function reduces its references counter.
6411                  * If after the reduction the action is still referenced, the
6412                  * function returns here and does not release it.
6413                  * When the counter action is not shared neither by ID nor by
6414                  * indirect action API, shared info is 1 before the reduction,
6415                  * so this condition is failed and function doesn't return here.
6416                  */
6417                 if (!IS_LEGACY_SHARED_CNT(counter) &&
6418                     __atomic_sub_fetch(&cnt->shared_info.refcnt, 1,
6419                                        __ATOMIC_RELAXED))
6420                         return;
6421         }
6422         cnt->pool = pool;
6423         /*
6424          * Put the counter back to list to be updated in none fallback mode.
6425          * Currently, we are using two list alternately, while one is in query,
6426          * add the freed counter to the other list based on the pool query_gen
6427          * value. After query finishes, add counter the list to the global
6428          * container counter list. The list changes while query starts. In
6429          * this case, lock will not be needed as query callback and release
6430          * function both operate with the different list.
6431          */
6432         if (!priv->sh->cmng.counter_fallback) {
6433                 rte_spinlock_lock(&pool->csl);
6434                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
6435                 rte_spinlock_unlock(&pool->csl);
6436         } else {
6437                 cnt->dcs_when_free = cnt->dcs_when_active;
6438                 cnt_type = pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
6439                                            MLX5_COUNTER_TYPE_ORIGIN;
6440                 rte_spinlock_lock(&priv->sh->cmng.csl[cnt_type]);
6441                 TAILQ_INSERT_TAIL(&priv->sh->cmng.counters[cnt_type],
6442                                   cnt, next);
6443                 rte_spinlock_unlock(&priv->sh->cmng.csl[cnt_type]);
6444         }
6445 }
6446
6447 /**
6448  * Resize a meter id container.
6449  *
6450  * @param[in] dev
6451  *   Pointer to the Ethernet device structure.
6452  *
6453  * @return
6454  *   0 on success, otherwise negative errno value and rte_errno is set.
6455  */
6456 static int
6457 flow_dv_mtr_container_resize(struct rte_eth_dev *dev)
6458 {
6459         struct mlx5_priv *priv = dev->data->dev_private;
6460         struct mlx5_aso_mtr_pools_mng *pools_mng =
6461                                 &priv->sh->mtrmng->pools_mng;
6462         void *old_pools = pools_mng->pools;
6463         uint32_t resize = pools_mng->n + MLX5_MTRS_CONTAINER_RESIZE;
6464         uint32_t mem_size = sizeof(struct mlx5_aso_mtr_pool *) * resize;
6465         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
6466
6467         if (!pools) {
6468                 rte_errno = ENOMEM;
6469                 return -ENOMEM;
6470         }
6471         if (!pools_mng->n)
6472                 if (mlx5_aso_queue_init(priv->sh, ASO_OPC_MOD_POLICER)) {
6473                         mlx5_free(pools);
6474                         return -ENOMEM;
6475                 }
6476         if (old_pools)
6477                 memcpy(pools, old_pools, pools_mng->n *
6478                                        sizeof(struct mlx5_aso_mtr_pool *));
6479         pools_mng->n = resize;
6480         pools_mng->pools = pools;
6481         if (old_pools)
6482                 mlx5_free(old_pools);
6483         return 0;
6484 }
6485
6486 /**
6487  * Prepare a new meter and/or a new meter pool.
6488  *
6489  * @param[in] dev
6490  *   Pointer to the Ethernet device structure.
6491  * @param[out] mtr_free
6492  *   Where to put the pointer of a new meter.g.
6493  *
6494  * @return
6495  *   The meter pool pointer and @mtr_free is set on success,
6496  *   NULL otherwise and rte_errno is set.
6497  */
6498 static struct mlx5_aso_mtr_pool *
6499 flow_dv_mtr_pool_create(struct rte_eth_dev *dev,
6500                              struct mlx5_aso_mtr **mtr_free)
6501 {
6502         struct mlx5_priv *priv = dev->data->dev_private;
6503         struct mlx5_aso_mtr_pools_mng *pools_mng =
6504                                 &priv->sh->mtrmng->pools_mng;
6505         struct mlx5_aso_mtr_pool *pool = NULL;
6506         struct mlx5_devx_obj *dcs = NULL;
6507         uint32_t i;
6508         uint32_t log_obj_size;
6509
6510         log_obj_size = rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1);
6511         dcs = mlx5_devx_cmd_create_flow_meter_aso_obj(priv->sh->ctx,
6512                         priv->sh->pdn, log_obj_size);
6513         if (!dcs) {
6514                 rte_errno = ENODATA;
6515                 return NULL;
6516         }
6517         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
6518         if (!pool) {
6519                 rte_errno = ENOMEM;
6520                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6521                 return NULL;
6522         }
6523         pool->devx_obj = dcs;
6524         pool->index = pools_mng->n_valid;
6525         if (pool->index == pools_mng->n && flow_dv_mtr_container_resize(dev)) {
6526                 mlx5_free(pool);
6527                 claim_zero(mlx5_devx_cmd_destroy(dcs));
6528                 return NULL;
6529         }
6530         pools_mng->pools[pool->index] = pool;
6531         pools_mng->n_valid++;
6532         for (i = 1; i < MLX5_ASO_MTRS_PER_POOL; ++i) {
6533                 pool->mtrs[i].offset = i;
6534                 LIST_INSERT_HEAD(&pools_mng->meters,
6535                                                 &pool->mtrs[i], next);
6536         }
6537         pool->mtrs[0].offset = 0;
6538         *mtr_free = &pool->mtrs[0];
6539         return pool;
6540 }
6541
6542 /**
6543  * Release a flow meter into pool.
6544  *
6545  * @param[in] dev
6546  *   Pointer to the Ethernet device structure.
6547  * @param[in] mtr_idx
6548  *   Index to aso flow meter.
6549  */
6550 static void
6551 flow_dv_aso_mtr_release_to_pool(struct rte_eth_dev *dev, uint32_t mtr_idx)
6552 {
6553         struct mlx5_priv *priv = dev->data->dev_private;
6554         struct mlx5_aso_mtr_pools_mng *pools_mng =
6555                                 &priv->sh->mtrmng->pools_mng;
6556         struct mlx5_aso_mtr *aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
6557
6558         MLX5_ASSERT(aso_mtr);
6559         rte_spinlock_lock(&pools_mng->mtrsl);
6560         memset(&aso_mtr->fm, 0, sizeof(struct mlx5_flow_meter_info));
6561         aso_mtr->state = ASO_METER_FREE;
6562         LIST_INSERT_HEAD(&pools_mng->meters, aso_mtr, next);
6563         rte_spinlock_unlock(&pools_mng->mtrsl);
6564 }
6565
6566 /**
6567  * Allocate a aso flow meter.
6568  *
6569  * @param[in] dev
6570  *   Pointer to the Ethernet device structure.
6571  *
6572  * @return
6573  *   Index to aso flow meter on success, 0 otherwise and rte_errno is set.
6574  */
6575 static uint32_t
6576 flow_dv_mtr_alloc(struct rte_eth_dev *dev)
6577 {
6578         struct mlx5_priv *priv = dev->data->dev_private;
6579         struct mlx5_aso_mtr *mtr_free = NULL;
6580         struct mlx5_aso_mtr_pools_mng *pools_mng =
6581                                 &priv->sh->mtrmng->pools_mng;
6582         struct mlx5_aso_mtr_pool *pool;
6583         uint32_t mtr_idx = 0;
6584
6585         if (!priv->config.devx) {
6586                 rte_errno = ENOTSUP;
6587                 return 0;
6588         }
6589         /* Allocate the flow meter memory. */
6590         /* Get free meters from management. */
6591         rte_spinlock_lock(&pools_mng->mtrsl);
6592         mtr_free = LIST_FIRST(&pools_mng->meters);
6593         if (mtr_free)
6594                 LIST_REMOVE(mtr_free, next);
6595         if (!mtr_free && !flow_dv_mtr_pool_create(dev, &mtr_free)) {
6596                 rte_spinlock_unlock(&pools_mng->mtrsl);
6597                 return 0;
6598         }
6599         mtr_free->state = ASO_METER_WAIT;
6600         rte_spinlock_unlock(&pools_mng->mtrsl);
6601         pool = container_of(mtr_free,
6602                         struct mlx5_aso_mtr_pool,
6603                         mtrs[mtr_free->offset]);
6604         mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset);
6605         if (!mtr_free->fm.meter_action) {
6606 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
6607                 struct rte_flow_error error;
6608                 uint8_t reg_id;
6609
6610                 reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &error);
6611                 mtr_free->fm.meter_action =
6612                         mlx5_glue->dv_create_flow_action_aso
6613                                                 (priv->sh->rx_domain,
6614                                                  pool->devx_obj->obj,
6615                                                  mtr_free->offset,
6616                                                  (1 << MLX5_FLOW_COLOR_GREEN),
6617                                                  reg_id - REG_C_0);
6618 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
6619                 if (!mtr_free->fm.meter_action) {
6620                         flow_dv_aso_mtr_release_to_pool(dev, mtr_idx);
6621                         return 0;
6622                 }
6623         }
6624         return mtr_idx;
6625 }
6626
6627 /**
6628  * Verify the @p attributes will be correctly understood by the NIC and store
6629  * them in the @p flow if everything is correct.
6630  *
6631  * @param[in] dev
6632  *   Pointer to dev struct.
6633  * @param[in] attributes
6634  *   Pointer to flow attributes
6635  * @param[in] external
6636  *   This flow rule is created by request external to PMD.
6637  * @param[out] error
6638  *   Pointer to error structure.
6639  *
6640  * @return
6641  *   - 0 on success and non root table.
6642  *   - 1 on success and root table.
6643  *   - a negative errno value otherwise and rte_errno is set.
6644  */
6645 static int
6646 flow_dv_validate_attributes(struct rte_eth_dev *dev,
6647                             const struct mlx5_flow_tunnel *tunnel,
6648                             const struct rte_flow_attr *attributes,
6649                             const struct flow_grp_info *grp_info,
6650                             struct rte_flow_error *error)
6651 {
6652         struct mlx5_priv *priv = dev->data->dev_private;
6653         uint32_t lowest_priority = mlx5_get_lowest_priority(dev, attributes);
6654         int ret = 0;
6655
6656 #ifndef HAVE_MLX5DV_DR
6657         RTE_SET_USED(tunnel);
6658         RTE_SET_USED(grp_info);
6659         if (attributes->group)
6660                 return rte_flow_error_set(error, ENOTSUP,
6661                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
6662                                           NULL,
6663                                           "groups are not supported");
6664 #else
6665         uint32_t table = 0;
6666
6667         ret = mlx5_flow_group_to_table(dev, tunnel, attributes->group, &table,
6668                                        grp_info, error);
6669         if (ret)
6670                 return ret;
6671         if (!table)
6672                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
6673 #endif
6674         if (attributes->priority != MLX5_FLOW_LOWEST_PRIO_INDICATOR &&
6675             attributes->priority > lowest_priority)
6676                 return rte_flow_error_set(error, ENOTSUP,
6677                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
6678                                           NULL,
6679                                           "priority out of range");
6680         if (attributes->transfer) {
6681                 if (!priv->config.dv_esw_en)
6682                         return rte_flow_error_set
6683                                 (error, ENOTSUP,
6684                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6685                                  "E-Switch dr is not supported");
6686                 if (!(priv->representor || priv->master))
6687                         return rte_flow_error_set
6688                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6689                                  NULL, "E-Switch configuration can only be"
6690                                  " done by a master or a representor device");
6691                 if (attributes->egress)
6692                         return rte_flow_error_set
6693                                 (error, ENOTSUP,
6694                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
6695                                  "egress is not supported");
6696         }
6697         if (!(attributes->egress ^ attributes->ingress))
6698                 return rte_flow_error_set(error, ENOTSUP,
6699                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
6700                                           "must specify exactly one of "
6701                                           "ingress or egress");
6702         return ret;
6703 }
6704
6705 static uint16_t
6706 mlx5_flow_locate_proto_l3(const struct rte_flow_item **head,
6707                           const struct rte_flow_item *end)
6708 {
6709         const struct rte_flow_item *item = *head;
6710         uint16_t l3_protocol;
6711
6712         for (; item != end; item++) {
6713                 switch (item->type) {
6714                 default:
6715                         break;
6716                 case RTE_FLOW_ITEM_TYPE_IPV4:
6717                         l3_protocol = RTE_ETHER_TYPE_IPV4;
6718                         goto l3_ok;
6719                 case RTE_FLOW_ITEM_TYPE_IPV6:
6720                         l3_protocol = RTE_ETHER_TYPE_IPV6;
6721                         goto l3_ok;
6722                 case RTE_FLOW_ITEM_TYPE_ETH:
6723                         if (item->mask && item->spec) {
6724                                 MLX5_ETHER_TYPE_FROM_HEADER(rte_flow_item_eth,
6725                                                             type, item,
6726                                                             l3_protocol);
6727                                 if (l3_protocol == RTE_ETHER_TYPE_IPV4 ||
6728                                     l3_protocol == RTE_ETHER_TYPE_IPV6)
6729                                         goto l3_ok;
6730                         }
6731                         break;
6732                 case RTE_FLOW_ITEM_TYPE_VLAN:
6733                         if (item->mask && item->spec) {
6734                                 MLX5_ETHER_TYPE_FROM_HEADER(rte_flow_item_vlan,
6735                                                             inner_type, item,
6736                                                             l3_protocol);
6737                                 if (l3_protocol == RTE_ETHER_TYPE_IPV4 ||
6738                                     l3_protocol == RTE_ETHER_TYPE_IPV6)
6739                                         goto l3_ok;
6740                         }
6741                         break;
6742                 }
6743         }
6744         return 0;
6745 l3_ok:
6746         *head = item;
6747         return l3_protocol;
6748 }
6749
6750 static uint8_t
6751 mlx5_flow_locate_proto_l4(const struct rte_flow_item **head,
6752                           const struct rte_flow_item *end)
6753 {
6754         const struct rte_flow_item *item = *head;
6755         uint8_t l4_protocol;
6756
6757         for (; item != end; item++) {
6758                 switch (item->type) {
6759                 default:
6760                         break;
6761                 case RTE_FLOW_ITEM_TYPE_TCP:
6762                         l4_protocol = IPPROTO_TCP;
6763                         goto l4_ok;
6764                 case RTE_FLOW_ITEM_TYPE_UDP:
6765                         l4_protocol = IPPROTO_UDP;
6766                         goto l4_ok;
6767                 case RTE_FLOW_ITEM_TYPE_IPV4:
6768                         if (item->mask && item->spec) {
6769                                 const struct rte_flow_item_ipv4 *mask, *spec;
6770
6771                                 mask = (typeof(mask))item->mask;
6772                                 spec = (typeof(spec))item->spec;
6773                                 l4_protocol = mask->hdr.next_proto_id &
6774                                               spec->hdr.next_proto_id;
6775                                 if (l4_protocol == IPPROTO_TCP ||
6776                                     l4_protocol == IPPROTO_UDP)
6777                                         goto l4_ok;
6778                         }
6779                         break;
6780                 case RTE_FLOW_ITEM_TYPE_IPV6:
6781                         if (item->mask && item->spec) {
6782                                 const struct rte_flow_item_ipv6 *mask, *spec;
6783                                 mask = (typeof(mask))item->mask;
6784                                 spec = (typeof(spec))item->spec;
6785                                 l4_protocol = mask->hdr.proto & spec->hdr.proto;
6786                                 if (l4_protocol == IPPROTO_TCP ||
6787                                     l4_protocol == IPPROTO_UDP)
6788                                         goto l4_ok;
6789                         }
6790                         break;
6791                 }
6792         }
6793         return 0;
6794 l4_ok:
6795         *head = item;
6796         return l4_protocol;
6797 }
6798
6799 static int
6800 flow_dv_validate_item_integrity(struct rte_eth_dev *dev,
6801                                 const struct rte_flow_item *rule_items,
6802                                 const struct rte_flow_item *integrity_item,
6803                                 struct rte_flow_error *error)
6804 {
6805         struct mlx5_priv *priv = dev->data->dev_private;
6806         const struct rte_flow_item *tunnel_item, *end_item, *item = rule_items;
6807         const struct rte_flow_item_integrity *mask = (typeof(mask))
6808                                                      integrity_item->mask;
6809         const struct rte_flow_item_integrity *spec = (typeof(spec))
6810                                                      integrity_item->spec;
6811         uint32_t protocol;
6812
6813         if (!priv->config.hca_attr.pkt_integrity_match)
6814                 return rte_flow_error_set(error, ENOTSUP,
6815                                           RTE_FLOW_ERROR_TYPE_ITEM,
6816                                           integrity_item,
6817                                           "packet integrity integrity_item not supported");
6818         if (!mask)
6819                 mask = &rte_flow_item_integrity_mask;
6820         if (!mlx5_validate_integrity_item(mask))
6821                 return rte_flow_error_set(error, ENOTSUP,
6822                                           RTE_FLOW_ERROR_TYPE_ITEM,
6823                                           integrity_item,
6824                                           "unsupported integrity filter");
6825         tunnel_item = mlx5_flow_find_tunnel_item(rule_items);
6826         if (spec->level > 1) {
6827                 if (!tunnel_item)
6828                         return rte_flow_error_set(error, ENOTSUP,
6829                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6830                                                   integrity_item,
6831                                                   "missing tunnel item");
6832                 item = tunnel_item;
6833                 end_item = mlx5_find_end_item(tunnel_item);
6834         } else {
6835                 end_item = tunnel_item ? tunnel_item :
6836                            mlx5_find_end_item(integrity_item);
6837         }
6838         if (mask->l3_ok || mask->ipv4_csum_ok) {
6839                 protocol = mlx5_flow_locate_proto_l3(&item, end_item);
6840                 if (!protocol)
6841                         return rte_flow_error_set(error, EINVAL,
6842                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6843                                                   integrity_item,
6844                                                   "missing L3 protocol");
6845         }
6846         if (mask->l4_ok || mask->l4_csum_ok) {
6847                 protocol = mlx5_flow_locate_proto_l4(&item, end_item);
6848                 if (!protocol)
6849                         return rte_flow_error_set(error, EINVAL,
6850                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6851                                                   integrity_item,
6852                                                   "missing L4 protocol");
6853         }
6854         return 0;
6855 }
6856
6857 /**
6858  * Internal validation function. For validating both actions and items.
6859  *
6860  * @param[in] dev
6861  *   Pointer to the rte_eth_dev structure.
6862  * @param[in] attr
6863  *   Pointer to the flow attributes.
6864  * @param[in] items
6865  *   Pointer to the list of items.
6866  * @param[in] actions
6867  *   Pointer to the list of actions.
6868  * @param[in] external
6869  *   This flow rule is created by request external to PMD.
6870  * @param[in] hairpin
6871  *   Number of hairpin TX actions, 0 means classic flow.
6872  * @param[out] error
6873  *   Pointer to the error structure.
6874  *
6875  * @return
6876  *   0 on success, a negative errno value otherwise and rte_errno is set.
6877  */
6878 static int
6879 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
6880                  const struct rte_flow_item items[],
6881                  const struct rte_flow_action actions[],
6882                  bool external, int hairpin, struct rte_flow_error *error)
6883 {
6884         int ret;
6885         uint64_t action_flags = 0;
6886         uint64_t item_flags = 0;
6887         uint64_t last_item = 0;
6888         uint8_t next_protocol = 0xff;
6889         uint16_t ether_type = 0;
6890         int actions_n = 0;
6891         uint8_t item_ipv6_proto = 0;
6892         int fdb_mirror_limit = 0;
6893         int modify_after_mirror = 0;
6894         const struct rte_flow_item *geneve_item = NULL;
6895         const struct rte_flow_item *gre_item = NULL;
6896         const struct rte_flow_item *gtp_item = NULL;
6897         const struct rte_flow_action_raw_decap *decap;
6898         const struct rte_flow_action_raw_encap *encap;
6899         const struct rte_flow_action_rss *rss = NULL;
6900         const struct rte_flow_action_rss *sample_rss = NULL;
6901         const struct rte_flow_action_count *sample_count = NULL;
6902         const struct rte_flow_item_tcp nic_tcp_mask = {
6903                 .hdr = {
6904                         .tcp_flags = 0xFF,
6905                         .src_port = RTE_BE16(UINT16_MAX),
6906                         .dst_port = RTE_BE16(UINT16_MAX),
6907                 }
6908         };
6909         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
6910                 .hdr = {
6911                         .src_addr =
6912                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6913                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6914                         .dst_addr =
6915                         "\xff\xff\xff\xff\xff\xff\xff\xff"
6916                         "\xff\xff\xff\xff\xff\xff\xff\xff",
6917                         .vtc_flow = RTE_BE32(0xffffffff),
6918                         .proto = 0xff,
6919                         .hop_limits = 0xff,
6920                 },
6921                 .has_frag_ext = 1,
6922         };
6923         const struct rte_flow_item_ecpri nic_ecpri_mask = {
6924                 .hdr = {
6925                         .common = {
6926                                 .u32 =
6927                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
6928                                         .type = 0xFF,
6929                                         }).u32),
6930                         },
6931                         .dummy[0] = 0xffffffff,
6932                 },
6933         };
6934         struct mlx5_priv *priv = dev->data->dev_private;
6935         struct mlx5_dev_config *dev_conf = &priv->config;
6936         uint16_t queue_index = 0xFFFF;
6937         const struct rte_flow_item_vlan *vlan_m = NULL;
6938         uint32_t rw_act_num = 0;
6939         uint64_t is_root;
6940         const struct mlx5_flow_tunnel *tunnel;
6941         enum mlx5_tof_rule_type tof_rule_type;
6942         struct flow_grp_info grp_info = {
6943                 .external = !!external,
6944                 .transfer = !!attr->transfer,
6945                 .fdb_def_rule = !!priv->fdb_def_rule,
6946                 .std_tbl_fix = true,
6947         };
6948         const struct rte_eth_hairpin_conf *conf;
6949         const struct rte_flow_item *rule_items = items;
6950         const struct rte_flow_item *port_id_item = NULL;
6951         bool def_policy = false;
6952
6953         if (items == NULL)
6954                 return -1;
6955         tunnel = is_tunnel_offload_active(dev) ?
6956                  mlx5_get_tof(items, actions, &tof_rule_type) : NULL;
6957         if (tunnel) {
6958                 if (priv->representor)
6959                         return rte_flow_error_set
6960                                 (error, ENOTSUP,
6961                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
6962                                  NULL, "decap not supported for VF representor");
6963                 if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_SET_RULE)
6964                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
6965                 else if (tof_rule_type == MLX5_TUNNEL_OFFLOAD_MATCH_RULE)
6966                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_MATCH |
6967                                         MLX5_FLOW_ACTION_DECAP;
6968                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
6969                                         (dev, attr, tunnel, tof_rule_type);
6970         }
6971         ret = flow_dv_validate_attributes(dev, tunnel, attr, &grp_info, error);
6972         if (ret < 0)
6973                 return ret;
6974         is_root = (uint64_t)ret;
6975         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
6976                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
6977                 int type = items->type;
6978
6979                 if (!mlx5_flow_os_item_supported(type))
6980                         return rte_flow_error_set(error, ENOTSUP,
6981                                                   RTE_FLOW_ERROR_TYPE_ITEM,
6982                                                   NULL, "item not supported");
6983                 switch (type) {
6984                 case RTE_FLOW_ITEM_TYPE_VOID:
6985                         break;
6986                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
6987                         ret = flow_dv_validate_item_port_id
6988                                         (dev, items, attr, item_flags, error);
6989                         if (ret < 0)
6990                                 return ret;
6991                         last_item = MLX5_FLOW_ITEM_PORT_ID;
6992                         port_id_item = items;
6993                         break;
6994                 case RTE_FLOW_ITEM_TYPE_ETH:
6995                         ret = mlx5_flow_validate_item_eth(items, item_flags,
6996                                                           true, error);
6997                         if (ret < 0)
6998                                 return ret;
6999                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
7000                                              MLX5_FLOW_LAYER_OUTER_L2;
7001                         if (items->mask != NULL && items->spec != NULL) {
7002                                 ether_type =
7003                                         ((const struct rte_flow_item_eth *)
7004                                          items->spec)->type;
7005                                 ether_type &=
7006                                         ((const struct rte_flow_item_eth *)
7007                                          items->mask)->type;
7008                                 ether_type = rte_be_to_cpu_16(ether_type);
7009                         } else {
7010                                 ether_type = 0;
7011                         }
7012                         break;
7013                 case RTE_FLOW_ITEM_TYPE_VLAN:
7014                         ret = flow_dv_validate_item_vlan(items, item_flags,
7015                                                          dev, error);
7016                         if (ret < 0)
7017                                 return ret;
7018                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
7019                                              MLX5_FLOW_LAYER_OUTER_VLAN;
7020                         if (items->mask != NULL && items->spec != NULL) {
7021                                 ether_type =
7022                                         ((const struct rte_flow_item_vlan *)
7023                                          items->spec)->inner_type;
7024                                 ether_type &=
7025                                         ((const struct rte_flow_item_vlan *)
7026                                          items->mask)->inner_type;
7027                                 ether_type = rte_be_to_cpu_16(ether_type);
7028                         } else {
7029                                 ether_type = 0;
7030                         }
7031                         /* Store outer VLAN mask for of_push_vlan action. */
7032                         if (!tunnel)
7033                                 vlan_m = items->mask;
7034                         break;
7035                 case RTE_FLOW_ITEM_TYPE_IPV4:
7036                         mlx5_flow_tunnel_ip_check(items, next_protocol,
7037                                                   &item_flags, &tunnel);
7038                         ret = flow_dv_validate_item_ipv4(dev, items, item_flags,
7039                                                          last_item, ether_type,
7040                                                          error);
7041                         if (ret < 0)
7042                                 return ret;
7043                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
7044                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
7045                         if (items->mask != NULL &&
7046                             ((const struct rte_flow_item_ipv4 *)
7047                              items->mask)->hdr.next_proto_id) {
7048                                 next_protocol =
7049                                         ((const struct rte_flow_item_ipv4 *)
7050                                          (items->spec))->hdr.next_proto_id;
7051                                 next_protocol &=
7052                                         ((const struct rte_flow_item_ipv4 *)
7053                                          (items->mask))->hdr.next_proto_id;
7054                         } else {
7055                                 /* Reset for inner layer. */
7056                                 next_protocol = 0xff;
7057                         }
7058                         break;
7059                 case RTE_FLOW_ITEM_TYPE_IPV6:
7060                         mlx5_flow_tunnel_ip_check(items, next_protocol,
7061                                                   &item_flags, &tunnel);
7062                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
7063                                                            last_item,
7064                                                            ether_type,
7065                                                            &nic_ipv6_mask,
7066                                                            error);
7067                         if (ret < 0)
7068                                 return ret;
7069                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
7070                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
7071                         if (items->mask != NULL &&
7072                             ((const struct rte_flow_item_ipv6 *)
7073                              items->mask)->hdr.proto) {
7074                                 item_ipv6_proto =
7075                                         ((const struct rte_flow_item_ipv6 *)
7076                                          items->spec)->hdr.proto;
7077                                 next_protocol =
7078                                         ((const struct rte_flow_item_ipv6 *)
7079                                          items->spec)->hdr.proto;
7080                                 next_protocol &=
7081                                         ((const struct rte_flow_item_ipv6 *)
7082                                          items->mask)->hdr.proto;
7083                         } else {
7084                                 /* Reset for inner layer. */
7085                                 next_protocol = 0xff;
7086                         }
7087                         break;
7088                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
7089                         ret = flow_dv_validate_item_ipv6_frag_ext(items,
7090                                                                   item_flags,
7091                                                                   error);
7092                         if (ret < 0)
7093                                 return ret;
7094                         last_item = tunnel ?
7095                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
7096                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
7097                         if (items->mask != NULL &&
7098                             ((const struct rte_flow_item_ipv6_frag_ext *)
7099                              items->mask)->hdr.next_header) {
7100                                 next_protocol =
7101                                 ((const struct rte_flow_item_ipv6_frag_ext *)
7102                                  items->spec)->hdr.next_header;
7103                                 next_protocol &=
7104                                 ((const struct rte_flow_item_ipv6_frag_ext *)
7105                                  items->mask)->hdr.next_header;
7106                         } else {
7107                                 /* Reset for inner layer. */
7108                                 next_protocol = 0xff;
7109                         }
7110                         break;
7111                 case RTE_FLOW_ITEM_TYPE_TCP:
7112                         ret = mlx5_flow_validate_item_tcp
7113                                                 (items, item_flags,
7114                                                  next_protocol,
7115                                                  &nic_tcp_mask,
7116                                                  error);
7117                         if (ret < 0)
7118                                 return ret;
7119                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
7120                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
7121                         break;
7122                 case RTE_FLOW_ITEM_TYPE_UDP:
7123                         ret = mlx5_flow_validate_item_udp(items, item_flags,
7124                                                           next_protocol,
7125                                                           error);
7126                         if (ret < 0)
7127                                 return ret;
7128                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
7129                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
7130                         break;
7131                 case RTE_FLOW_ITEM_TYPE_GRE:
7132                         ret = mlx5_flow_validate_item_gre(items, item_flags,
7133                                                           next_protocol, error);
7134                         if (ret < 0)
7135                                 return ret;
7136                         gre_item = items;
7137                         last_item = MLX5_FLOW_LAYER_GRE;
7138                         break;
7139                 case RTE_FLOW_ITEM_TYPE_NVGRE:
7140                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
7141                                                             next_protocol,
7142                                                             error);
7143                         if (ret < 0)
7144                                 return ret;
7145                         last_item = MLX5_FLOW_LAYER_NVGRE;
7146                         break;
7147                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
7148                         ret = mlx5_flow_validate_item_gre_key
7149                                 (items, item_flags, gre_item, error);
7150                         if (ret < 0)
7151                                 return ret;
7152                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
7153                         break;
7154                 case RTE_FLOW_ITEM_TYPE_VXLAN:
7155                         ret = mlx5_flow_validate_item_vxlan(dev, items,
7156                                                             item_flags, attr,
7157                                                             error);
7158                         if (ret < 0)
7159                                 return ret;
7160                         last_item = MLX5_FLOW_LAYER_VXLAN;
7161                         break;
7162                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
7163                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
7164                                                                 item_flags, dev,
7165                                                                 error);
7166                         if (ret < 0)
7167                                 return ret;
7168                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
7169                         break;
7170                 case RTE_FLOW_ITEM_TYPE_GENEVE:
7171                         ret = mlx5_flow_validate_item_geneve(items,
7172                                                              item_flags, dev,
7173                                                              error);
7174                         if (ret < 0)
7175                                 return ret;
7176                         geneve_item = items;
7177                         last_item = MLX5_FLOW_LAYER_GENEVE;
7178                         break;
7179                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
7180                         ret = mlx5_flow_validate_item_geneve_opt(items,
7181                                                                  last_item,
7182                                                                  geneve_item,
7183                                                                  dev,
7184                                                                  error);
7185                         if (ret < 0)
7186                                 return ret;
7187                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
7188                         break;
7189                 case RTE_FLOW_ITEM_TYPE_MPLS:
7190                         ret = mlx5_flow_validate_item_mpls(dev, items,
7191                                                            item_flags,
7192                                                            last_item, error);
7193                         if (ret < 0)
7194                                 return ret;
7195                         last_item = MLX5_FLOW_LAYER_MPLS;
7196                         break;
7197
7198                 case RTE_FLOW_ITEM_TYPE_MARK:
7199                         ret = flow_dv_validate_item_mark(dev, items, attr,
7200                                                          error);
7201                         if (ret < 0)
7202                                 return ret;
7203                         last_item = MLX5_FLOW_ITEM_MARK;
7204                         break;
7205                 case RTE_FLOW_ITEM_TYPE_META:
7206                         ret = flow_dv_validate_item_meta(dev, items, attr,
7207                                                          error);
7208                         if (ret < 0)
7209                                 return ret;
7210                         last_item = MLX5_FLOW_ITEM_METADATA;
7211                         break;
7212                 case RTE_FLOW_ITEM_TYPE_ICMP:
7213                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
7214                                                            next_protocol,
7215                                                            error);
7216                         if (ret < 0)
7217                                 return ret;
7218                         last_item = MLX5_FLOW_LAYER_ICMP;
7219                         break;
7220                 case RTE_FLOW_ITEM_TYPE_ICMP6:
7221                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
7222                                                             next_protocol,
7223                                                             error);
7224                         if (ret < 0)
7225                                 return ret;
7226                         item_ipv6_proto = IPPROTO_ICMPV6;
7227                         last_item = MLX5_FLOW_LAYER_ICMP6;
7228                         break;
7229                 case RTE_FLOW_ITEM_TYPE_TAG:
7230                         ret = flow_dv_validate_item_tag(dev, items,
7231                                                         attr, error);
7232                         if (ret < 0)
7233                                 return ret;
7234                         last_item = MLX5_FLOW_ITEM_TAG;
7235                         break;
7236                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
7237                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
7238                         break;
7239                 case RTE_FLOW_ITEM_TYPE_GTP:
7240                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
7241                                                         error);
7242                         if (ret < 0)
7243                                 return ret;
7244                         gtp_item = items;
7245                         last_item = MLX5_FLOW_LAYER_GTP;
7246                         break;
7247                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
7248                         ret = flow_dv_validate_item_gtp_psc(items, last_item,
7249                                                             gtp_item, attr,
7250                                                             error);
7251                         if (ret < 0)
7252                                 return ret;
7253                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
7254                         break;
7255                 case RTE_FLOW_ITEM_TYPE_ECPRI:
7256                         /* Capacity will be checked in the translate stage. */
7257                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
7258                                                             last_item,
7259                                                             ether_type,
7260                                                             &nic_ecpri_mask,
7261                                                             error);
7262                         if (ret < 0)
7263                                 return ret;
7264                         last_item = MLX5_FLOW_LAYER_ECPRI;
7265                         break;
7266                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
7267                         if (item_flags & MLX5_FLOW_ITEM_INTEGRITY)
7268                                 return rte_flow_error_set
7269                                         (error, ENOTSUP,
7270                                          RTE_FLOW_ERROR_TYPE_ITEM,
7271                                          NULL, "multiple integrity items not supported");
7272                         ret = flow_dv_validate_item_integrity(dev, rule_items,
7273                                                               items, error);
7274                         if (ret < 0)
7275                                 return ret;
7276                         last_item = MLX5_FLOW_ITEM_INTEGRITY;
7277                         break;
7278                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
7279                         ret = flow_dv_validate_item_aso_ct(dev, items,
7280                                                            &item_flags, error);
7281                         if (ret < 0)
7282                                 return ret;
7283                         break;
7284                 case MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL:
7285                         /* tunnel offload item was processed before
7286                          * list it here as a supported type
7287                          */
7288                         break;
7289                 default:
7290                         return rte_flow_error_set(error, ENOTSUP,
7291                                                   RTE_FLOW_ERROR_TYPE_ITEM,
7292                                                   NULL, "item not supported");
7293                 }
7294                 item_flags |= last_item;
7295         }
7296         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
7297                 int type = actions->type;
7298                 bool shared_count = false;
7299
7300                 if (!mlx5_flow_os_action_supported(type))
7301                         return rte_flow_error_set(error, ENOTSUP,
7302                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7303                                                   actions,
7304                                                   "action not supported");
7305                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
7306                         return rte_flow_error_set(error, ENOTSUP,
7307                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7308                                                   actions, "too many actions");
7309                 if (action_flags &
7310                         MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
7311                         return rte_flow_error_set(error, ENOTSUP,
7312                                 RTE_FLOW_ERROR_TYPE_ACTION,
7313                                 NULL, "meter action with policy "
7314                                 "must be the last action");
7315                 switch (type) {
7316                 case RTE_FLOW_ACTION_TYPE_VOID:
7317                         break;
7318                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
7319                         ret = flow_dv_validate_action_port_id(dev,
7320                                                               action_flags,
7321                                                               actions,
7322                                                               attr,
7323                                                               error);
7324                         if (ret)
7325                                 return ret;
7326                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
7327                         ++actions_n;
7328                         break;
7329                 case RTE_FLOW_ACTION_TYPE_FLAG:
7330                         ret = flow_dv_validate_action_flag(dev, action_flags,
7331                                                            attr, error);
7332                         if (ret < 0)
7333                                 return ret;
7334                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7335                                 /* Count all modify-header actions as one. */
7336                                 if (!(action_flags &
7337                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7338                                         ++actions_n;
7339                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
7340                                                 MLX5_FLOW_ACTION_MARK_EXT;
7341                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7342                                         modify_after_mirror = 1;
7343
7344                         } else {
7345                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
7346                                 ++actions_n;
7347                         }
7348                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7349                         break;
7350                 case RTE_FLOW_ACTION_TYPE_MARK:
7351                         ret = flow_dv_validate_action_mark(dev, actions,
7352                                                            action_flags,
7353                                                            attr, error);
7354                         if (ret < 0)
7355                                 return ret;
7356                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7357                                 /* Count all modify-header actions as one. */
7358                                 if (!(action_flags &
7359                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
7360                                         ++actions_n;
7361                                 action_flags |= MLX5_FLOW_ACTION_MARK |
7362                                                 MLX5_FLOW_ACTION_MARK_EXT;
7363                                 if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7364                                         modify_after_mirror = 1;
7365                         } else {
7366                                 action_flags |= MLX5_FLOW_ACTION_MARK;
7367                                 ++actions_n;
7368                         }
7369                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
7370                         break;
7371                 case RTE_FLOW_ACTION_TYPE_SET_META:
7372                         ret = flow_dv_validate_action_set_meta(dev, actions,
7373                                                                action_flags,
7374                                                                attr, error);
7375                         if (ret < 0)
7376                                 return ret;
7377                         /* Count all modify-header actions as one action. */
7378                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7379                                 ++actions_n;
7380                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7381                                 modify_after_mirror = 1;
7382                         action_flags |= MLX5_FLOW_ACTION_SET_META;
7383                         rw_act_num += MLX5_ACT_NUM_SET_META;
7384                         break;
7385                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
7386                         ret = flow_dv_validate_action_set_tag(dev, actions,
7387                                                               action_flags,
7388                                                               attr, error);
7389                         if (ret < 0)
7390                                 return ret;
7391                         /* Count all modify-header actions as one action. */
7392                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7393                                 ++actions_n;
7394                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7395                                 modify_after_mirror = 1;
7396                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
7397                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7398                         break;
7399                 case RTE_FLOW_ACTION_TYPE_DROP:
7400                         ret = mlx5_flow_validate_action_drop(action_flags,
7401                                                              attr, error);
7402                         if (ret < 0)
7403                                 return ret;
7404                         action_flags |= MLX5_FLOW_ACTION_DROP;
7405                         ++actions_n;
7406                         break;
7407                 case RTE_FLOW_ACTION_TYPE_QUEUE:
7408                         ret = mlx5_flow_validate_action_queue(actions,
7409                                                               action_flags, dev,
7410                                                               attr, error);
7411                         if (ret < 0)
7412                                 return ret;
7413                         queue_index = ((const struct rte_flow_action_queue *)
7414                                                         (actions->conf))->index;
7415                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
7416                         ++actions_n;
7417                         break;
7418                 case RTE_FLOW_ACTION_TYPE_RSS:
7419                         rss = actions->conf;
7420                         ret = mlx5_flow_validate_action_rss(actions,
7421                                                             action_flags, dev,
7422                                                             attr, item_flags,
7423                                                             error);
7424                         if (ret < 0)
7425                                 return ret;
7426                         if (rss && sample_rss &&
7427                             (sample_rss->level != rss->level ||
7428                             sample_rss->types != rss->types))
7429                                 return rte_flow_error_set(error, ENOTSUP,
7430                                         RTE_FLOW_ERROR_TYPE_ACTION,
7431                                         NULL,
7432                                         "Can't use the different RSS types "
7433                                         "or level in the same flow");
7434                         if (rss != NULL && rss->queue_num)
7435                                 queue_index = rss->queue[0];
7436                         action_flags |= MLX5_FLOW_ACTION_RSS;
7437                         ++actions_n;
7438                         break;
7439                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
7440                         ret =
7441                         mlx5_flow_validate_action_default_miss(action_flags,
7442                                         attr, error);
7443                         if (ret < 0)
7444                                 return ret;
7445                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
7446                         ++actions_n;
7447                         break;
7448                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
7449                 case RTE_FLOW_ACTION_TYPE_COUNT:
7450                         shared_count = is_shared_action_count(actions);
7451                         ret = flow_dv_validate_action_count(dev, shared_count,
7452                                                             action_flags,
7453                                                             error);
7454                         if (ret < 0)
7455                                 return ret;
7456                         action_flags |= MLX5_FLOW_ACTION_COUNT;
7457                         ++actions_n;
7458                         break;
7459                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
7460                         if (flow_dv_validate_action_pop_vlan(dev,
7461                                                              action_flags,
7462                                                              actions,
7463                                                              item_flags, attr,
7464                                                              error))
7465                                 return -rte_errno;
7466                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7467                                 modify_after_mirror = 1;
7468                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
7469                         ++actions_n;
7470                         break;
7471                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
7472                         ret = flow_dv_validate_action_push_vlan(dev,
7473                                                                 action_flags,
7474                                                                 vlan_m,
7475                                                                 actions, attr,
7476                                                                 error);
7477                         if (ret < 0)
7478                                 return ret;
7479                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7480                                 modify_after_mirror = 1;
7481                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
7482                         ++actions_n;
7483                         break;
7484                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
7485                         ret = flow_dv_validate_action_set_vlan_pcp
7486                                                 (action_flags, actions, error);
7487                         if (ret < 0)
7488                                 return ret;
7489                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7490                                 modify_after_mirror = 1;
7491                         /* Count PCP with push_vlan command. */
7492                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
7493                         break;
7494                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
7495                         ret = flow_dv_validate_action_set_vlan_vid
7496                                                 (item_flags, action_flags,
7497                                                  actions, error);
7498                         if (ret < 0)
7499                                 return ret;
7500                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7501                                 modify_after_mirror = 1;
7502                         /* Count VID with push_vlan command. */
7503                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
7504                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
7505                         break;
7506                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
7507                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
7508                         ret = flow_dv_validate_action_l2_encap(dev,
7509                                                                action_flags,
7510                                                                actions, attr,
7511                                                                error);
7512                         if (ret < 0)
7513                                 return ret;
7514                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
7515                         ++actions_n;
7516                         break;
7517                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
7518                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
7519                         ret = flow_dv_validate_action_decap(dev, action_flags,
7520                                                             actions, item_flags,
7521                                                             attr, error);
7522                         if (ret < 0)
7523                                 return ret;
7524                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7525                                 modify_after_mirror = 1;
7526                         action_flags |= MLX5_FLOW_ACTION_DECAP;
7527                         ++actions_n;
7528                         break;
7529                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
7530                         ret = flow_dv_validate_action_raw_encap_decap
7531                                 (dev, NULL, actions->conf, attr, &action_flags,
7532                                  &actions_n, actions, item_flags, error);
7533                         if (ret < 0)
7534                                 return ret;
7535                         break;
7536                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
7537                         decap = actions->conf;
7538                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
7539                                 ;
7540                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
7541                                 encap = NULL;
7542                                 actions--;
7543                         } else {
7544                                 encap = actions->conf;
7545                         }
7546                         ret = flow_dv_validate_action_raw_encap_decap
7547                                            (dev,
7548                                             decap ? decap : &empty_decap, encap,
7549                                             attr, &action_flags, &actions_n,
7550                                             actions, item_flags, error);
7551                         if (ret < 0)
7552                                 return ret;
7553                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7554                             (action_flags & MLX5_FLOW_ACTION_DECAP))
7555                                 modify_after_mirror = 1;
7556                         break;
7557                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
7558                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
7559                         ret = flow_dv_validate_action_modify_mac(action_flags,
7560                                                                  actions,
7561                                                                  item_flags,
7562                                                                  error);
7563                         if (ret < 0)
7564                                 return ret;
7565                         /* Count all modify-header actions as one action. */
7566                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7567                                 ++actions_n;
7568                         action_flags |= actions->type ==
7569                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
7570                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
7571                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
7572                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7573                                 modify_after_mirror = 1;
7574                         /*
7575                          * Even if the source and destination MAC addresses have
7576                          * overlap in the header with 4B alignment, the convert
7577                          * function will handle them separately and 4 SW actions
7578                          * will be created. And 2 actions will be added each
7579                          * time no matter how many bytes of address will be set.
7580                          */
7581                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
7582                         break;
7583                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
7584                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
7585                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
7586                                                                   actions,
7587                                                                   item_flags,
7588                                                                   error);
7589                         if (ret < 0)
7590                                 return ret;
7591                         /* Count all modify-header actions as one action. */
7592                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7593                                 ++actions_n;
7594                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7595                                 modify_after_mirror = 1;
7596                         action_flags |= actions->type ==
7597                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
7598                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
7599                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
7600                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
7601                         break;
7602                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
7603                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
7604                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
7605                                                                   actions,
7606                                                                   item_flags,
7607                                                                   error);
7608                         if (ret < 0)
7609                                 return ret;
7610                         if (item_ipv6_proto == IPPROTO_ICMPV6)
7611                                 return rte_flow_error_set(error, ENOTSUP,
7612                                         RTE_FLOW_ERROR_TYPE_ACTION,
7613                                         actions,
7614                                         "Can't change header "
7615                                         "with ICMPv6 proto");
7616                         /* Count all modify-header actions as one action. */
7617                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7618                                 ++actions_n;
7619                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7620                                 modify_after_mirror = 1;
7621                         action_flags |= actions->type ==
7622                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
7623                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
7624                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
7625                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
7626                         break;
7627                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
7628                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
7629                         ret = flow_dv_validate_action_modify_tp(action_flags,
7630                                                                 actions,
7631                                                                 item_flags,
7632                                                                 error);
7633                         if (ret < 0)
7634                                 return ret;
7635                         /* Count all modify-header actions as one action. */
7636                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7637                                 ++actions_n;
7638                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7639                                 modify_after_mirror = 1;
7640                         action_flags |= actions->type ==
7641                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
7642                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
7643                                                 MLX5_FLOW_ACTION_SET_TP_DST;
7644                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
7645                         break;
7646                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
7647                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
7648                         ret = flow_dv_validate_action_modify_ttl(action_flags,
7649                                                                  actions,
7650                                                                  item_flags,
7651                                                                  error);
7652                         if (ret < 0)
7653                                 return ret;
7654                         /* Count all modify-header actions as one action. */
7655                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7656                                 ++actions_n;
7657                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7658                                 modify_after_mirror = 1;
7659                         action_flags |= actions->type ==
7660                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
7661                                                 MLX5_FLOW_ACTION_SET_TTL :
7662                                                 MLX5_FLOW_ACTION_DEC_TTL;
7663                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
7664                         break;
7665                 case RTE_FLOW_ACTION_TYPE_JUMP:
7666                         ret = flow_dv_validate_action_jump(dev, tunnel, actions,
7667                                                            action_flags,
7668                                                            attr, external,
7669                                                            error);
7670                         if (ret)
7671                                 return ret;
7672                         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) &&
7673                             fdb_mirror_limit)
7674                                 return rte_flow_error_set(error, EINVAL,
7675                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7676                                                   NULL,
7677                                                   "sample and jump action combination is not supported");
7678                         ++actions_n;
7679                         action_flags |= MLX5_FLOW_ACTION_JUMP;
7680                         break;
7681                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
7682                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
7683                         ret = flow_dv_validate_action_modify_tcp_seq
7684                                                                 (action_flags,
7685                                                                  actions,
7686                                                                  item_flags,
7687                                                                  error);
7688                         if (ret < 0)
7689                                 return ret;
7690                         /* Count all modify-header actions as one action. */
7691                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7692                                 ++actions_n;
7693                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7694                                 modify_after_mirror = 1;
7695                         action_flags |= actions->type ==
7696                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
7697                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
7698                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
7699                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
7700                         break;
7701                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
7702                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
7703                         ret = flow_dv_validate_action_modify_tcp_ack
7704                                                                 (action_flags,
7705                                                                  actions,
7706                                                                  item_flags,
7707                                                                  error);
7708                         if (ret < 0)
7709                                 return ret;
7710                         /* Count all modify-header actions as one action. */
7711                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7712                                 ++actions_n;
7713                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7714                                 modify_after_mirror = 1;
7715                         action_flags |= actions->type ==
7716                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
7717                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
7718                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
7719                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
7720                         break;
7721                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
7722                         break;
7723                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
7724                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
7725                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7726                         break;
7727                 case RTE_FLOW_ACTION_TYPE_METER:
7728                         ret = mlx5_flow_validate_action_meter(dev,
7729                                                               action_flags,
7730                                                               actions, attr,
7731                                                               port_id_item,
7732                                                               &def_policy,
7733                                                               error);
7734                         if (ret < 0)
7735                                 return ret;
7736                         action_flags |= MLX5_FLOW_ACTION_METER;
7737                         if (!def_policy)
7738                                 action_flags |=
7739                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
7740                         ++actions_n;
7741                         /* Meter action will add one more TAG action. */
7742                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
7743                         break;
7744                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
7745                         if (!attr->transfer && !attr->group)
7746                                 return rte_flow_error_set(error, ENOTSUP,
7747                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7748                                                                            NULL,
7749                           "Shared ASO age action is not supported for group 0");
7750                         if (action_flags & MLX5_FLOW_ACTION_AGE)
7751                                 return rte_flow_error_set
7752                                                   (error, EINVAL,
7753                                                    RTE_FLOW_ERROR_TYPE_ACTION,
7754                                                    NULL,
7755                                                    "duplicate age actions set");
7756                         action_flags |= MLX5_FLOW_ACTION_AGE;
7757                         ++actions_n;
7758                         break;
7759                 case RTE_FLOW_ACTION_TYPE_AGE:
7760                         ret = flow_dv_validate_action_age(action_flags,
7761                                                           actions, dev,
7762                                                           error);
7763                         if (ret < 0)
7764                                 return ret;
7765                         /*
7766                          * Validate the regular AGE action (using counter)
7767                          * mutual exclusion with share counter actions.
7768                          */
7769                         if (!priv->sh->flow_hit_aso_en) {
7770                                 if (shared_count)
7771                                         return rte_flow_error_set
7772                                                 (error, EINVAL,
7773                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7774                                                 NULL,
7775                                                 "old age and shared count combination is not supported");
7776                                 if (sample_count)
7777                                         return rte_flow_error_set
7778                                                 (error, EINVAL,
7779                                                 RTE_FLOW_ERROR_TYPE_ACTION,
7780                                                 NULL,
7781                                                 "old age action and count must be in the same sub flow");
7782                         }
7783                         action_flags |= MLX5_FLOW_ACTION_AGE;
7784                         ++actions_n;
7785                         break;
7786                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
7787                         ret = flow_dv_validate_action_modify_ipv4_dscp
7788                                                          (action_flags,
7789                                                           actions,
7790                                                           item_flags,
7791                                                           error);
7792                         if (ret < 0)
7793                                 return ret;
7794                         /* Count all modify-header actions as one action. */
7795                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7796                                 ++actions_n;
7797                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7798                                 modify_after_mirror = 1;
7799                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
7800                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7801                         break;
7802                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
7803                         ret = flow_dv_validate_action_modify_ipv6_dscp
7804                                                                 (action_flags,
7805                                                                  actions,
7806                                                                  item_flags,
7807                                                                  error);
7808                         if (ret < 0)
7809                                 return ret;
7810                         /* Count all modify-header actions as one action. */
7811                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7812                                 ++actions_n;
7813                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7814                                 modify_after_mirror = 1;
7815                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
7816                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
7817                         break;
7818                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
7819                         ret = flow_dv_validate_action_sample(&action_flags,
7820                                                              actions, dev,
7821                                                              attr, item_flags,
7822                                                              rss, &sample_rss,
7823                                                              &sample_count,
7824                                                              &fdb_mirror_limit,
7825                                                              error);
7826                         if (ret < 0)
7827                                 return ret;
7828                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
7829                         ++actions_n;
7830                         break;
7831                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
7832                         ret = flow_dv_validate_action_modify_field(dev,
7833                                                                    action_flags,
7834                                                                    actions,
7835                                                                    attr,
7836                                                                    error);
7837                         if (ret < 0)
7838                                 return ret;
7839                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
7840                                 modify_after_mirror = 1;
7841                         /* Count all modify-header actions as one action. */
7842                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
7843                                 ++actions_n;
7844                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
7845                         rw_act_num += ret;
7846                         break;
7847                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
7848                         ret = flow_dv_validate_action_aso_ct(dev, action_flags,
7849                                                              item_flags, attr,
7850                                                              error);
7851                         if (ret < 0)
7852                                 return ret;
7853                         action_flags |= MLX5_FLOW_ACTION_CT;
7854                         break;
7855                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
7856                         /* tunnel offload action was processed before
7857                          * list it here as a supported type
7858                          */
7859                         break;
7860                 default:
7861                         return rte_flow_error_set(error, ENOTSUP,
7862                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7863                                                   actions,
7864                                                   "action not supported");
7865                 }
7866         }
7867         /*
7868          * Validate actions in flow rules
7869          * - Explicit decap action is prohibited by the tunnel offload API.
7870          * - Drop action in tunnel steer rule is prohibited by the API.
7871          * - Application cannot use MARK action because it's value can mask
7872          *   tunnel default miss nitification.
7873          * - JUMP in tunnel match rule has no support in current PMD
7874          *   implementation.
7875          * - TAG & META are reserved for future uses.
7876          */
7877         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_SET) {
7878                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_DECAP    |
7879                                             MLX5_FLOW_ACTION_MARK     |
7880                                             MLX5_FLOW_ACTION_SET_TAG  |
7881                                             MLX5_FLOW_ACTION_SET_META |
7882                                             MLX5_FLOW_ACTION_DROP;
7883
7884                 if (action_flags & bad_actions_mask)
7885                         return rte_flow_error_set
7886                                         (error, EINVAL,
7887                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7888                                         "Invalid RTE action in tunnel "
7889                                         "set decap rule");
7890                 if (!(action_flags & MLX5_FLOW_ACTION_JUMP))
7891                         return rte_flow_error_set
7892                                         (error, EINVAL,
7893                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7894                                         "tunnel set decap rule must terminate "
7895                                         "with JUMP");
7896                 if (!attr->ingress)
7897                         return rte_flow_error_set
7898                                         (error, EINVAL,
7899                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7900                                         "tunnel flows for ingress traffic only");
7901         }
7902         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_MATCH) {
7903                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_JUMP    |
7904                                             MLX5_FLOW_ACTION_MARK    |
7905                                             MLX5_FLOW_ACTION_SET_TAG |
7906                                             MLX5_FLOW_ACTION_SET_META;
7907
7908                 if (action_flags & bad_actions_mask)
7909                         return rte_flow_error_set
7910                                         (error, EINVAL,
7911                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7912                                         "Invalid RTE action in tunnel "
7913                                         "set match rule");
7914         }
7915         /*
7916          * Validate the drop action mutual exclusion with other actions.
7917          * Drop action is mutually-exclusive with any other action, except for
7918          * Count action.
7919          * Drop action compatibility with tunnel offload was already validated.
7920          */
7921         if (action_flags & (MLX5_FLOW_ACTION_TUNNEL_MATCH |
7922                             MLX5_FLOW_ACTION_TUNNEL_MATCH));
7923         else if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
7924             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
7925                 return rte_flow_error_set(error, EINVAL,
7926                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
7927                                           "Drop action is mutually-exclusive "
7928                                           "with any other action, except for "
7929                                           "Count action");
7930         /* Eswitch has few restrictions on using items and actions */
7931         if (attr->transfer) {
7932                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7933                     action_flags & MLX5_FLOW_ACTION_FLAG)
7934                         return rte_flow_error_set(error, ENOTSUP,
7935                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7936                                                   NULL,
7937                                                   "unsupported action FLAG");
7938                 if (!mlx5_flow_ext_mreg_supported(dev) &&
7939                     action_flags & MLX5_FLOW_ACTION_MARK)
7940                         return rte_flow_error_set(error, ENOTSUP,
7941                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7942                                                   NULL,
7943                                                   "unsupported action MARK");
7944                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
7945                         return rte_flow_error_set(error, ENOTSUP,
7946                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7947                                                   NULL,
7948                                                   "unsupported action QUEUE");
7949                 if (action_flags & MLX5_FLOW_ACTION_RSS)
7950                         return rte_flow_error_set(error, ENOTSUP,
7951                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7952                                                   NULL,
7953                                                   "unsupported action RSS");
7954                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
7955                         return rte_flow_error_set(error, EINVAL,
7956                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7957                                                   actions,
7958                                                   "no fate action is found");
7959         } else {
7960                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
7961                         return rte_flow_error_set(error, EINVAL,
7962                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7963                                                   actions,
7964                                                   "no fate action is found");
7965         }
7966         /*
7967          * Continue validation for Xcap and VLAN actions.
7968          * If hairpin is working in explicit TX rule mode, there is no actions
7969          * splitting and the validation of hairpin ingress flow should be the
7970          * same as other standard flows.
7971          */
7972         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
7973                              MLX5_FLOW_VLAN_ACTIONS)) &&
7974             (queue_index == 0xFFFF ||
7975              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN ||
7976              ((conf = mlx5_rxq_get_hairpin_conf(dev, queue_index)) != NULL &&
7977              conf->tx_explicit != 0))) {
7978                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
7979                     MLX5_FLOW_XCAP_ACTIONS)
7980                         return rte_flow_error_set(error, ENOTSUP,
7981                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7982                                                   NULL, "encap and decap "
7983                                                   "combination aren't supported");
7984                 if (!attr->transfer && attr->ingress) {
7985                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
7986                                 return rte_flow_error_set
7987                                                 (error, ENOTSUP,
7988                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7989                                                  NULL, "encap is not supported"
7990                                                  " for ingress traffic");
7991                         else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
7992                                 return rte_flow_error_set
7993                                                 (error, ENOTSUP,
7994                                                  RTE_FLOW_ERROR_TYPE_ACTION,
7995                                                  NULL, "push VLAN action not "
7996                                                  "supported for ingress");
7997                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
7998                                         MLX5_FLOW_VLAN_ACTIONS)
7999                                 return rte_flow_error_set
8000                                                 (error, ENOTSUP,
8001                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8002                                                  NULL, "no support for "
8003                                                  "multiple VLAN actions");
8004                 }
8005         }
8006         if (action_flags & MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY) {
8007                 if ((action_flags & (MLX5_FLOW_FATE_ACTIONS &
8008                         ~MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)) &&
8009                         attr->ingress)
8010                         return rte_flow_error_set
8011                                 (error, ENOTSUP,
8012                                 RTE_FLOW_ERROR_TYPE_ACTION,
8013                                 NULL, "fate action not supported for "
8014                                 "meter with policy");
8015                 if (attr->egress) {
8016                         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
8017                                 return rte_flow_error_set
8018                                         (error, ENOTSUP,
8019                                         RTE_FLOW_ERROR_TYPE_ACTION,
8020                                         NULL, "modify header action in egress "
8021                                         "cannot be done before meter action");
8022                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
8023                                 return rte_flow_error_set
8024                                         (error, ENOTSUP,
8025                                         RTE_FLOW_ERROR_TYPE_ACTION,
8026                                         NULL, "encap action in egress "
8027                                         "cannot be done before meter action");
8028                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
8029                                 return rte_flow_error_set
8030                                         (error, ENOTSUP,
8031                                         RTE_FLOW_ERROR_TYPE_ACTION,
8032                                         NULL, "push vlan action in egress "
8033                                         "cannot be done before meter action");
8034                 }
8035         }
8036         /*
8037          * Hairpin flow will add one more TAG action in TX implicit mode.
8038          * In TX explicit mode, there will be no hairpin flow ID.
8039          */
8040         if (hairpin > 0)
8041                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
8042         /* extra metadata enabled: one more TAG action will be add. */
8043         if (dev_conf->dv_flow_en &&
8044             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
8045             mlx5_flow_ext_mreg_supported(dev))
8046                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
8047         if (rw_act_num >
8048                         flow_dv_modify_hdr_action_max(dev, is_root)) {
8049                 return rte_flow_error_set(error, ENOTSUP,
8050                                           RTE_FLOW_ERROR_TYPE_ACTION,
8051                                           NULL, "too many header modify"
8052                                           " actions to support");
8053         }
8054         /* Eswitch egress mirror and modify flow has limitation on CX5 */
8055         if (fdb_mirror_limit && modify_after_mirror)
8056                 return rte_flow_error_set(error, EINVAL,
8057                                 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
8058                                 "sample before modify action is not supported");
8059         return 0;
8060 }
8061
8062 /**
8063  * Internal preparation function. Allocates the DV flow size,
8064  * this size is constant.
8065  *
8066  * @param[in] dev
8067  *   Pointer to the rte_eth_dev structure.
8068  * @param[in] attr
8069  *   Pointer to the flow attributes.
8070  * @param[in] items
8071  *   Pointer to the list of items.
8072  * @param[in] actions
8073  *   Pointer to the list of actions.
8074  * @param[out] error
8075  *   Pointer to the error structure.
8076  *
8077  * @return
8078  *   Pointer to mlx5_flow object on success,
8079  *   otherwise NULL and rte_errno is set.
8080  */
8081 static struct mlx5_flow *
8082 flow_dv_prepare(struct rte_eth_dev *dev,
8083                 const struct rte_flow_attr *attr __rte_unused,
8084                 const struct rte_flow_item items[] __rte_unused,
8085                 const struct rte_flow_action actions[] __rte_unused,
8086                 struct rte_flow_error *error)
8087 {
8088         uint32_t handle_idx = 0;
8089         struct mlx5_flow *dev_flow;
8090         struct mlx5_flow_handle *dev_handle;
8091         struct mlx5_priv *priv = dev->data->dev_private;
8092         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
8093
8094         MLX5_ASSERT(wks);
8095         wks->skip_matcher_reg = 0;
8096         wks->policy = NULL;
8097         wks->final_policy = NULL;
8098         /* In case of corrupting the memory. */
8099         if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
8100                 rte_flow_error_set(error, ENOSPC,
8101                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8102                                    "not free temporary device flow");
8103                 return NULL;
8104         }
8105         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
8106                                    &handle_idx);
8107         if (!dev_handle) {
8108                 rte_flow_error_set(error, ENOMEM,
8109                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8110                                    "not enough memory to create flow handle");
8111                 return NULL;
8112         }
8113         MLX5_ASSERT(wks->flow_idx < RTE_DIM(wks->flows));
8114         dev_flow = &wks->flows[wks->flow_idx++];
8115         memset(dev_flow, 0, sizeof(*dev_flow));
8116         dev_flow->handle = dev_handle;
8117         dev_flow->handle_idx = handle_idx;
8118         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
8119         dev_flow->ingress = attr->ingress;
8120         dev_flow->dv.transfer = attr->transfer;
8121         return dev_flow;
8122 }
8123
8124 #ifdef RTE_LIBRTE_MLX5_DEBUG
8125 /**
8126  * Sanity check for match mask and value. Similar to check_valid_spec() in
8127  * kernel driver. If unmasked bit is present in value, it returns failure.
8128  *
8129  * @param match_mask
8130  *   pointer to match mask buffer.
8131  * @param match_value
8132  *   pointer to match value buffer.
8133  *
8134  * @return
8135  *   0 if valid, -EINVAL otherwise.
8136  */
8137 static int
8138 flow_dv_check_valid_spec(void *match_mask, void *match_value)
8139 {
8140         uint8_t *m = match_mask;
8141         uint8_t *v = match_value;
8142         unsigned int i;
8143
8144         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
8145                 if (v[i] & ~m[i]) {
8146                         DRV_LOG(ERR,
8147                                 "match_value differs from match_criteria"
8148                                 " %p[%u] != %p[%u]",
8149                                 match_value, i, match_mask, i);
8150                         return -EINVAL;
8151                 }
8152         }
8153         return 0;
8154 }
8155 #endif
8156
8157 /**
8158  * Add match of ip_version.
8159  *
8160  * @param[in] group
8161  *   Flow group.
8162  * @param[in] headers_v
8163  *   Values header pointer.
8164  * @param[in] headers_m
8165  *   Masks header pointer.
8166  * @param[in] ip_version
8167  *   The IP version to set.
8168  */
8169 static inline void
8170 flow_dv_set_match_ip_version(uint32_t group,
8171                              void *headers_v,
8172                              void *headers_m,
8173                              uint8_t ip_version)
8174 {
8175         if (group == 0)
8176                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
8177         else
8178                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
8179                          ip_version);
8180         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
8181         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
8182         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
8183 }
8184
8185 /**
8186  * Add Ethernet item to matcher and to the value.
8187  *
8188  * @param[in, out] matcher
8189  *   Flow matcher.
8190  * @param[in, out] key
8191  *   Flow matcher value.
8192  * @param[in] item
8193  *   Flow pattern to translate.
8194  * @param[in] inner
8195  *   Item is inner pattern.
8196  */
8197 static void
8198 flow_dv_translate_item_eth(void *matcher, void *key,
8199                            const struct rte_flow_item *item, int inner,
8200                            uint32_t group)
8201 {
8202         const struct rte_flow_item_eth *eth_m = item->mask;
8203         const struct rte_flow_item_eth *eth_v = item->spec;
8204         const struct rte_flow_item_eth nic_mask = {
8205                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
8206                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
8207                 .type = RTE_BE16(0xffff),
8208                 .has_vlan = 0,
8209         };
8210         void *hdrs_m;
8211         void *hdrs_v;
8212         char *l24_v;
8213         unsigned int i;
8214
8215         if (!eth_v)
8216                 return;
8217         if (!eth_m)
8218                 eth_m = &nic_mask;
8219         if (inner) {
8220                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8221                                          inner_headers);
8222                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8223         } else {
8224                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8225                                          outer_headers);
8226                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8227         }
8228         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, dmac_47_16),
8229                &eth_m->dst, sizeof(eth_m->dst));
8230         /* The value must be in the range of the mask. */
8231         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, dmac_47_16);
8232         for (i = 0; i < sizeof(eth_m->dst); ++i)
8233                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
8234         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, smac_47_16),
8235                &eth_m->src, sizeof(eth_m->src));
8236         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, smac_47_16);
8237         /* The value must be in the range of the mask. */
8238         for (i = 0; i < sizeof(eth_m->dst); ++i)
8239                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
8240         /*
8241          * HW supports match on one Ethertype, the Ethertype following the last
8242          * VLAN tag of the packet (see PRM).
8243          * Set match on ethertype only if ETH header is not followed by VLAN.
8244          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8245          * ethertype, and use ip_version field instead.
8246          * eCPRI over Ether layer will use type value 0xAEFE.
8247          */
8248         if (eth_m->type == 0xFFFF) {
8249                 /* Set cvlan_tag mask for any single\multi\un-tagged case. */
8250                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8251                 switch (eth_v->type) {
8252                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8253                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8254                         return;
8255                 case RTE_BE16(RTE_ETHER_TYPE_QINQ):
8256                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8257                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8258                         return;
8259                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8260                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8261                         return;
8262                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8263                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8264                         return;
8265                 default:
8266                         break;
8267                 }
8268         }
8269         if (eth_m->has_vlan) {
8270                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8271                 if (eth_v->has_vlan) {
8272                         /*
8273                          * Here, when also has_more_vlan field in VLAN item is
8274                          * not set, only single-tagged packets will be matched.
8275                          */
8276                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8277                         return;
8278                 }
8279         }
8280         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8281                  rte_be_to_cpu_16(eth_m->type));
8282         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype);
8283         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
8284 }
8285
8286 /**
8287  * Add VLAN item to matcher and to the value.
8288  *
8289  * @param[in, out] dev_flow
8290  *   Flow descriptor.
8291  * @param[in, out] matcher
8292  *   Flow matcher.
8293  * @param[in, out] key
8294  *   Flow matcher value.
8295  * @param[in] item
8296  *   Flow pattern to translate.
8297  * @param[in] inner
8298  *   Item is inner pattern.
8299  */
8300 static void
8301 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
8302                             void *matcher, void *key,
8303                             const struct rte_flow_item *item,
8304                             int inner, uint32_t group)
8305 {
8306         const struct rte_flow_item_vlan *vlan_m = item->mask;
8307         const struct rte_flow_item_vlan *vlan_v = item->spec;
8308         void *hdrs_m;
8309         void *hdrs_v;
8310         uint16_t tci_m;
8311         uint16_t tci_v;
8312
8313         if (inner) {
8314                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8315                                          inner_headers);
8316                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8317         } else {
8318                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
8319                                          outer_headers);
8320                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8321                 /*
8322                  * This is workaround, masks are not supported,
8323                  * and pre-validated.
8324                  */
8325                 if (vlan_v)
8326                         dev_flow->handle->vf_vlan.tag =
8327                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
8328         }
8329         /*
8330          * When VLAN item exists in flow, mark packet as tagged,
8331          * even if TCI is not specified.
8332          */
8333         if (!MLX5_GET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag)) {
8334                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
8335                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
8336         }
8337         if (!vlan_v)
8338                 return;
8339         if (!vlan_m)
8340                 vlan_m = &rte_flow_item_vlan_mask;
8341         tci_m = rte_be_to_cpu_16(vlan_m->tci);
8342         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
8343         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_vid, tci_m);
8344         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_vid, tci_v);
8345         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_cfi, tci_m >> 12);
8346         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_cfi, tci_v >> 12);
8347         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_prio, tci_m >> 13);
8348         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_prio, tci_v >> 13);
8349         /*
8350          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
8351          * ethertype, and use ip_version field instead.
8352          */
8353         if (vlan_m->inner_type == 0xFFFF) {
8354                 switch (vlan_v->inner_type) {
8355                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
8356                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8357                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8358                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8359                         return;
8360                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
8361                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
8362                         return;
8363                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
8364                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
8365                         return;
8366                 default:
8367                         break;
8368                 }
8369         }
8370         if (vlan_m->has_more_vlan && vlan_v->has_more_vlan) {
8371                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
8372                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
8373                 /* Only one vlan_tag bit can be set. */
8374                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
8375                 return;
8376         }
8377         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
8378                  rte_be_to_cpu_16(vlan_m->inner_type));
8379         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, ethertype,
8380                  rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
8381 }
8382
8383 /**
8384  * Add IPV4 item to matcher and to the value.
8385  *
8386  * @param[in, out] matcher
8387  *   Flow matcher.
8388  * @param[in, out] key
8389  *   Flow matcher value.
8390  * @param[in] item
8391  *   Flow pattern to translate.
8392  * @param[in] inner
8393  *   Item is inner pattern.
8394  * @param[in] group
8395  *   The group to insert the rule.
8396  */
8397 static void
8398 flow_dv_translate_item_ipv4(void *matcher, void *key,
8399                             const struct rte_flow_item *item,
8400                             int inner, uint32_t group)
8401 {
8402         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
8403         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
8404         const struct rte_flow_item_ipv4 nic_mask = {
8405                 .hdr = {
8406                         .src_addr = RTE_BE32(0xffffffff),
8407                         .dst_addr = RTE_BE32(0xffffffff),
8408                         .type_of_service = 0xff,
8409                         .next_proto_id = 0xff,
8410                         .time_to_live = 0xff,
8411                 },
8412         };
8413         void *headers_m;
8414         void *headers_v;
8415         char *l24_m;
8416         char *l24_v;
8417         uint8_t tos, ihl_m, ihl_v;
8418
8419         if (inner) {
8420                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8421                                          inner_headers);
8422                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8423         } else {
8424                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8425                                          outer_headers);
8426                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8427         }
8428         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
8429         if (!ipv4_v)
8430                 return;
8431         if (!ipv4_m)
8432                 ipv4_m = &nic_mask;
8433         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8434                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8435         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8436                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
8437         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
8438         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
8439         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8440                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8441         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8442                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
8443         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
8444         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
8445         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
8446         ihl_m = ipv4_m->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK;
8447         ihl_v = ipv4_v->hdr.version_ihl & RTE_IPV4_HDR_IHL_MASK;
8448         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_ihl, ihl_m);
8449         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_ihl, ihl_m & ihl_v);
8450         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
8451                  ipv4_m->hdr.type_of_service);
8452         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
8453         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
8454                  ipv4_m->hdr.type_of_service >> 2);
8455         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
8456         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8457                  ipv4_m->hdr.next_proto_id);
8458         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8459                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
8460         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8461                  ipv4_m->hdr.time_to_live);
8462         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8463                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
8464         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8465                  !!(ipv4_m->hdr.fragment_offset));
8466         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8467                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
8468 }
8469
8470 /**
8471  * Add IPV6 item to matcher and to the value.
8472  *
8473  * @param[in, out] matcher
8474  *   Flow matcher.
8475  * @param[in, out] key
8476  *   Flow matcher value.
8477  * @param[in] item
8478  *   Flow pattern to translate.
8479  * @param[in] inner
8480  *   Item is inner pattern.
8481  * @param[in] group
8482  *   The group to insert the rule.
8483  */
8484 static void
8485 flow_dv_translate_item_ipv6(void *matcher, void *key,
8486                             const struct rte_flow_item *item,
8487                             int inner, uint32_t group)
8488 {
8489         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
8490         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
8491         const struct rte_flow_item_ipv6 nic_mask = {
8492                 .hdr = {
8493                         .src_addr =
8494                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8495                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8496                         .dst_addr =
8497                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
8498                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
8499                         .vtc_flow = RTE_BE32(0xffffffff),
8500                         .proto = 0xff,
8501                         .hop_limits = 0xff,
8502                 },
8503         };
8504         void *headers_m;
8505         void *headers_v;
8506         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8507         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8508         char *l24_m;
8509         char *l24_v;
8510         uint32_t vtc_m;
8511         uint32_t vtc_v;
8512         int i;
8513         int size;
8514
8515         if (inner) {
8516                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8517                                          inner_headers);
8518                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8519         } else {
8520                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8521                                          outer_headers);
8522                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8523         }
8524         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
8525         if (!ipv6_v)
8526                 return;
8527         if (!ipv6_m)
8528                 ipv6_m = &nic_mask;
8529         size = sizeof(ipv6_m->hdr.dst_addr);
8530         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8531                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8532         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8533                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
8534         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
8535         for (i = 0; i < size; ++i)
8536                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
8537         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
8538                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8539         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
8540                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
8541         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
8542         for (i = 0; i < size; ++i)
8543                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
8544         /* TOS. */
8545         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
8546         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
8547         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
8548         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
8549         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
8550         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
8551         /* Label. */
8552         if (inner) {
8553                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
8554                          vtc_m);
8555                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
8556                          vtc_v);
8557         } else {
8558                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
8559                          vtc_m);
8560                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
8561                          vtc_v);
8562         }
8563         /* Protocol. */
8564         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8565                  ipv6_m->hdr.proto);
8566         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8567                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
8568         /* Hop limit. */
8569         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
8570                  ipv6_m->hdr.hop_limits);
8571         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
8572                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
8573         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
8574                  !!(ipv6_m->has_frag_ext));
8575         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
8576                  !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext));
8577 }
8578
8579 /**
8580  * Add IPV6 fragment extension item to matcher and to the value.
8581  *
8582  * @param[in, out] matcher
8583  *   Flow matcher.
8584  * @param[in, out] key
8585  *   Flow matcher value.
8586  * @param[in] item
8587  *   Flow pattern to translate.
8588  * @param[in] inner
8589  *   Item is inner pattern.
8590  */
8591 static void
8592 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key,
8593                                      const struct rte_flow_item *item,
8594                                      int inner)
8595 {
8596         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask;
8597         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec;
8598         const struct rte_flow_item_ipv6_frag_ext nic_mask = {
8599                 .hdr = {
8600                         .next_header = 0xff,
8601                         .frag_data = RTE_BE16(0xffff),
8602                 },
8603         };
8604         void *headers_m;
8605         void *headers_v;
8606
8607         if (inner) {
8608                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8609                                          inner_headers);
8610                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8611         } else {
8612                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8613                                          outer_headers);
8614                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8615         }
8616         /* IPv6 fragment extension item exists, so packet is IP fragment. */
8617         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
8618         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1);
8619         if (!ipv6_frag_ext_v)
8620                 return;
8621         if (!ipv6_frag_ext_m)
8622                 ipv6_frag_ext_m = &nic_mask;
8623         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
8624                  ipv6_frag_ext_m->hdr.next_header);
8625         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
8626                  ipv6_frag_ext_v->hdr.next_header &
8627                  ipv6_frag_ext_m->hdr.next_header);
8628 }
8629
8630 /**
8631  * Add TCP item to matcher and to the value.
8632  *
8633  * @param[in, out] matcher
8634  *   Flow matcher.
8635  * @param[in, out] key
8636  *   Flow matcher value.
8637  * @param[in] item
8638  *   Flow pattern to translate.
8639  * @param[in] inner
8640  *   Item is inner pattern.
8641  */
8642 static void
8643 flow_dv_translate_item_tcp(void *matcher, void *key,
8644                            const struct rte_flow_item *item,
8645                            int inner)
8646 {
8647         const struct rte_flow_item_tcp *tcp_m = item->mask;
8648         const struct rte_flow_item_tcp *tcp_v = item->spec;
8649         void *headers_m;
8650         void *headers_v;
8651
8652         if (inner) {
8653                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8654                                          inner_headers);
8655                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8656         } else {
8657                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8658                                          outer_headers);
8659                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8660         }
8661         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8662         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
8663         if (!tcp_v)
8664                 return;
8665         if (!tcp_m)
8666                 tcp_m = &rte_flow_item_tcp_mask;
8667         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
8668                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
8669         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
8670                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
8671         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
8672                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
8673         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
8674                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
8675         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
8676                  tcp_m->hdr.tcp_flags);
8677         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
8678                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
8679 }
8680
8681 /**
8682  * Add UDP item to matcher and to the value.
8683  *
8684  * @param[in, out] matcher
8685  *   Flow matcher.
8686  * @param[in, out] key
8687  *   Flow matcher value.
8688  * @param[in] item
8689  *   Flow pattern to translate.
8690  * @param[in] inner
8691  *   Item is inner pattern.
8692  */
8693 static void
8694 flow_dv_translate_item_udp(void *matcher, void *key,
8695                            const struct rte_flow_item *item,
8696                            int inner)
8697 {
8698         const struct rte_flow_item_udp *udp_m = item->mask;
8699         const struct rte_flow_item_udp *udp_v = item->spec;
8700         void *headers_m;
8701         void *headers_v;
8702
8703         if (inner) {
8704                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8705                                          inner_headers);
8706                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8707         } else {
8708                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8709                                          outer_headers);
8710                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8711         }
8712         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8713         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
8714         if (!udp_v)
8715                 return;
8716         if (!udp_m)
8717                 udp_m = &rte_flow_item_udp_mask;
8718         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
8719                  rte_be_to_cpu_16(udp_m->hdr.src_port));
8720         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
8721                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
8722         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
8723                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
8724         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
8725                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
8726 }
8727
8728 /**
8729  * Add GRE optional Key item to matcher and to the value.
8730  *
8731  * @param[in, out] matcher
8732  *   Flow matcher.
8733  * @param[in, out] key
8734  *   Flow matcher value.
8735  * @param[in] item
8736  *   Flow pattern to translate.
8737  * @param[in] inner
8738  *   Item is inner pattern.
8739  */
8740 static void
8741 flow_dv_translate_item_gre_key(void *matcher, void *key,
8742                                    const struct rte_flow_item *item)
8743 {
8744         const rte_be32_t *key_m = item->mask;
8745         const rte_be32_t *key_v = item->spec;
8746         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8747         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8748         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
8749
8750         /* GRE K bit must be on and should already be validated */
8751         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
8752         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
8753         if (!key_v)
8754                 return;
8755         if (!key_m)
8756                 key_m = &gre_key_default_mask;
8757         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
8758                  rte_be_to_cpu_32(*key_m) >> 8);
8759         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
8760                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
8761         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
8762                  rte_be_to_cpu_32(*key_m) & 0xFF);
8763         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
8764                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
8765 }
8766
8767 /**
8768  * Add GRE item to matcher and to the value.
8769  *
8770  * @param[in, out] matcher
8771  *   Flow matcher.
8772  * @param[in, out] key
8773  *   Flow matcher value.
8774  * @param[in] item
8775  *   Flow pattern to translate.
8776  * @param[in] inner
8777  *   Item is inner pattern.
8778  */
8779 static void
8780 flow_dv_translate_item_gre(void *matcher, void *key,
8781                            const struct rte_flow_item *item,
8782                            int inner)
8783 {
8784         const struct rte_flow_item_gre *gre_m = item->mask;
8785         const struct rte_flow_item_gre *gre_v = item->spec;
8786         void *headers_m;
8787         void *headers_v;
8788         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8789         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8790         struct {
8791                 union {
8792                         __extension__
8793                         struct {
8794                                 uint16_t version:3;
8795                                 uint16_t rsvd0:9;
8796                                 uint16_t s_present:1;
8797                                 uint16_t k_present:1;
8798                                 uint16_t rsvd_bit1:1;
8799                                 uint16_t c_present:1;
8800                         };
8801                         uint16_t value;
8802                 };
8803         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
8804
8805         if (inner) {
8806                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8807                                          inner_headers);
8808                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8809         } else {
8810                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8811                                          outer_headers);
8812                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8813         }
8814         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
8815         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
8816         if (!gre_v)
8817                 return;
8818         if (!gre_m)
8819                 gre_m = &rte_flow_item_gre_mask;
8820         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
8821                  rte_be_to_cpu_16(gre_m->protocol));
8822         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
8823                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
8824         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
8825         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
8826         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
8827                  gre_crks_rsvd0_ver_m.c_present);
8828         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
8829                  gre_crks_rsvd0_ver_v.c_present &
8830                  gre_crks_rsvd0_ver_m.c_present);
8831         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
8832                  gre_crks_rsvd0_ver_m.k_present);
8833         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
8834                  gre_crks_rsvd0_ver_v.k_present &
8835                  gre_crks_rsvd0_ver_m.k_present);
8836         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
8837                  gre_crks_rsvd0_ver_m.s_present);
8838         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
8839                  gre_crks_rsvd0_ver_v.s_present &
8840                  gre_crks_rsvd0_ver_m.s_present);
8841 }
8842
8843 /**
8844  * Add NVGRE item to matcher and to the value.
8845  *
8846  * @param[in, out] matcher
8847  *   Flow matcher.
8848  * @param[in, out] key
8849  *   Flow matcher value.
8850  * @param[in] item
8851  *   Flow pattern to translate.
8852  * @param[in] inner
8853  *   Item is inner pattern.
8854  */
8855 static void
8856 flow_dv_translate_item_nvgre(void *matcher, void *key,
8857                              const struct rte_flow_item *item,
8858                              int inner)
8859 {
8860         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
8861         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
8862         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8863         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8864         const char *tni_flow_id_m;
8865         const char *tni_flow_id_v;
8866         char *gre_key_m;
8867         char *gre_key_v;
8868         int size;
8869         int i;
8870
8871         /* For NVGRE, GRE header fields must be set with defined values. */
8872         const struct rte_flow_item_gre gre_spec = {
8873                 .c_rsvd0_ver = RTE_BE16(0x2000),
8874                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
8875         };
8876         const struct rte_flow_item_gre gre_mask = {
8877                 .c_rsvd0_ver = RTE_BE16(0xB000),
8878                 .protocol = RTE_BE16(UINT16_MAX),
8879         };
8880         const struct rte_flow_item gre_item = {
8881                 .spec = &gre_spec,
8882                 .mask = &gre_mask,
8883                 .last = NULL,
8884         };
8885         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
8886         if (!nvgre_v)
8887                 return;
8888         if (!nvgre_m)
8889                 nvgre_m = &rte_flow_item_nvgre_mask;
8890         tni_flow_id_m = (const char *)nvgre_m->tni;
8891         tni_flow_id_v = (const char *)nvgre_v->tni;
8892         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
8893         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
8894         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
8895         memcpy(gre_key_m, tni_flow_id_m, size);
8896         for (i = 0; i < size; ++i)
8897                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
8898 }
8899
8900 /**
8901  * Add VXLAN item to matcher and to the value.
8902  *
8903  * @param[in] dev
8904  *   Pointer to the Ethernet device structure.
8905  * @param[in] attr
8906  *   Flow rule attributes.
8907  * @param[in, out] matcher
8908  *   Flow matcher.
8909  * @param[in, out] key
8910  *   Flow matcher value.
8911  * @param[in] item
8912  *   Flow pattern to translate.
8913  * @param[in] inner
8914  *   Item is inner pattern.
8915  */
8916 static void
8917 flow_dv_translate_item_vxlan(struct rte_eth_dev *dev,
8918                              const struct rte_flow_attr *attr,
8919                              void *matcher, void *key,
8920                              const struct rte_flow_item *item,
8921                              int inner)
8922 {
8923         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
8924         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
8925         void *headers_m;
8926         void *headers_v;
8927         void *misc5_m;
8928         void *misc5_v;
8929         uint32_t *tunnel_header_v;
8930         uint32_t *tunnel_header_m;
8931         uint16_t dport;
8932         struct mlx5_priv *priv = dev->data->dev_private;
8933         const struct rte_flow_item_vxlan nic_mask = {
8934                 .vni = "\xff\xff\xff",
8935                 .rsvd1 = 0xff,
8936         };
8937
8938         if (inner) {
8939                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8940                                          inner_headers);
8941                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
8942         } else {
8943                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
8944                                          outer_headers);
8945                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
8946         }
8947         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
8948                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
8949         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
8950                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
8951                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
8952         }
8953         if (!vxlan_v)
8954                 return;
8955         if (!vxlan_m) {
8956                 if ((!attr->group && !priv->sh->tunnel_header_0_1) ||
8957                     (attr->group && !priv->sh->misc5_cap))
8958                         vxlan_m = &rte_flow_item_vxlan_mask;
8959                 else
8960                         vxlan_m = &nic_mask;
8961         }
8962         if ((!attr->group && !attr->transfer && !priv->sh->tunnel_header_0_1) ||
8963             ((attr->group || attr->transfer) && !priv->sh->misc5_cap)) {
8964                 void *misc_m;
8965                 void *misc_v;
8966                 char *vni_m;
8967                 char *vni_v;
8968                 int size;
8969                 int i;
8970                 misc_m = MLX5_ADDR_OF(fte_match_param,
8971                                       matcher, misc_parameters);
8972                 misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8973                 size = sizeof(vxlan_m->vni);
8974                 vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
8975                 vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
8976                 memcpy(vni_m, vxlan_m->vni, size);
8977                 for (i = 0; i < size; ++i)
8978                         vni_v[i] = vni_m[i] & vxlan_v->vni[i];
8979                 return;
8980         }
8981         misc5_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_5);
8982         misc5_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_5);
8983         tunnel_header_v = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5,
8984                                                    misc5_v,
8985                                                    tunnel_header_1);
8986         tunnel_header_m = (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc5,
8987                                                    misc5_m,
8988                                                    tunnel_header_1);
8989         *tunnel_header_v = (vxlan_v->vni[0] & vxlan_m->vni[0]) |
8990                            (vxlan_v->vni[1] & vxlan_m->vni[1]) << 8 |
8991                            (vxlan_v->vni[2] & vxlan_m->vni[2]) << 16;
8992         if (*tunnel_header_v)
8993                 *tunnel_header_m = vxlan_m->vni[0] |
8994                         vxlan_m->vni[1] << 8 |
8995                         vxlan_m->vni[2] << 16;
8996         else
8997                 *tunnel_header_m = 0x0;
8998         *tunnel_header_v |= (vxlan_v->rsvd1 & vxlan_m->rsvd1) << 24;
8999         if (vxlan_v->rsvd1 & vxlan_m->rsvd1)
9000                 *tunnel_header_m |= vxlan_m->rsvd1 << 24;
9001 }
9002
9003 /**
9004  * Add VXLAN-GPE item to matcher and to the value.
9005  *
9006  * @param[in, out] matcher
9007  *   Flow matcher.
9008  * @param[in, out] key
9009  *   Flow matcher value.
9010  * @param[in] item
9011  *   Flow pattern to translate.
9012  * @param[in] inner
9013  *   Item is inner pattern.
9014  */
9015
9016 static void
9017 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
9018                                  const struct rte_flow_item *item, int inner)
9019 {
9020         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
9021         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
9022         void *headers_m;
9023         void *headers_v;
9024         void *misc_m =
9025                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
9026         void *misc_v =
9027                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9028         char *vni_m;
9029         char *vni_v;
9030         uint16_t dport;
9031         int size;
9032         int i;
9033         uint8_t flags_m = 0xff;
9034         uint8_t flags_v = 0xc;
9035
9036         if (inner) {
9037                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9038                                          inner_headers);
9039                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9040         } else {
9041                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9042                                          outer_headers);
9043                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9044         }
9045         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
9046                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
9047         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9048                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9049                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
9050         }
9051         if (!vxlan_v)
9052                 return;
9053         if (!vxlan_m)
9054                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
9055         size = sizeof(vxlan_m->vni);
9056         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
9057         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
9058         memcpy(vni_m, vxlan_m->vni, size);
9059         for (i = 0; i < size; ++i)
9060                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
9061         if (vxlan_m->flags) {
9062                 flags_m = vxlan_m->flags;
9063                 flags_v = vxlan_v->flags;
9064         }
9065         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
9066         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
9067         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
9068                  vxlan_m->protocol);
9069         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
9070                  vxlan_v->protocol);
9071 }
9072
9073 /**
9074  * Add Geneve item to matcher and to the value.
9075  *
9076  * @param[in, out] matcher
9077  *   Flow matcher.
9078  * @param[in, out] key
9079  *   Flow matcher value.
9080  * @param[in] item
9081  *   Flow pattern to translate.
9082  * @param[in] inner
9083  *   Item is inner pattern.
9084  */
9085
9086 static void
9087 flow_dv_translate_item_geneve(void *matcher, void *key,
9088                               const struct rte_flow_item *item, int inner)
9089 {
9090         const struct rte_flow_item_geneve *geneve_m = item->mask;
9091         const struct rte_flow_item_geneve *geneve_v = item->spec;
9092         void *headers_m;
9093         void *headers_v;
9094         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9095         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9096         uint16_t dport;
9097         uint16_t gbhdr_m;
9098         uint16_t gbhdr_v;
9099         char *vni_m;
9100         char *vni_v;
9101         size_t size, i;
9102
9103         if (inner) {
9104                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9105                                          inner_headers);
9106                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9107         } else {
9108                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9109                                          outer_headers);
9110                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9111         }
9112         dport = MLX5_UDP_PORT_GENEVE;
9113         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9114                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9115                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
9116         }
9117         if (!geneve_v)
9118                 return;
9119         if (!geneve_m)
9120                 geneve_m = &rte_flow_item_geneve_mask;
9121         size = sizeof(geneve_m->vni);
9122         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
9123         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
9124         memcpy(vni_m, geneve_m->vni, size);
9125         for (i = 0; i < size; ++i)
9126                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
9127         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
9128                  rte_be_to_cpu_16(geneve_m->protocol));
9129         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
9130                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
9131         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
9132         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
9133         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
9134                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
9135         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
9136                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
9137         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
9138                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
9139         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
9140                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
9141                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
9142 }
9143
9144 /**
9145  * Create Geneve TLV option resource.
9146  *
9147  * @param dev[in, out]
9148  *   Pointer to rte_eth_dev structure.
9149  * @param[in, out] tag_be24
9150  *   Tag value in big endian then R-shift 8.
9151  * @parm[in, out] dev_flow
9152  *   Pointer to the dev_flow.
9153  * @param[out] error
9154  *   pointer to error structure.
9155  *
9156  * @return
9157  *   0 on success otherwise -errno and errno is set.
9158  */
9159
9160 int
9161 flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
9162                                              const struct rte_flow_item *item,
9163                                              struct rte_flow_error *error)
9164 {
9165         struct mlx5_priv *priv = dev->data->dev_private;
9166         struct mlx5_dev_ctx_shared *sh = priv->sh;
9167         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
9168                         sh->geneve_tlv_option_resource;
9169         struct mlx5_devx_obj *obj;
9170         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
9171         int ret = 0;
9172
9173         if (!geneve_opt_v)
9174                 return -1;
9175         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
9176         if (geneve_opt_resource != NULL) {
9177                 if (geneve_opt_resource->option_class ==
9178                         geneve_opt_v->option_class &&
9179                         geneve_opt_resource->option_type ==
9180                         geneve_opt_v->option_type &&
9181                         geneve_opt_resource->length ==
9182                         geneve_opt_v->option_len) {
9183                         /* We already have GENVE TLV option obj allocated. */
9184                         __atomic_fetch_add(&geneve_opt_resource->refcnt, 1,
9185                                            __ATOMIC_RELAXED);
9186                 } else {
9187                         ret = rte_flow_error_set(error, ENOMEM,
9188                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9189                                 "Only one GENEVE TLV option supported");
9190                         goto exit;
9191                 }
9192         } else {
9193                 /* Create a GENEVE TLV object and resource. */
9194                 obj = mlx5_devx_cmd_create_geneve_tlv_option(sh->ctx,
9195                                 geneve_opt_v->option_class,
9196                                 geneve_opt_v->option_type,
9197                                 geneve_opt_v->option_len);
9198                 if (!obj) {
9199                         ret = rte_flow_error_set(error, ENODATA,
9200                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9201                                 "Failed to create GENEVE TLV Devx object");
9202                         goto exit;
9203                 }
9204                 sh->geneve_tlv_option_resource =
9205                                 mlx5_malloc(MLX5_MEM_ZERO,
9206                                                 sizeof(*geneve_opt_resource),
9207                                                 0, SOCKET_ID_ANY);
9208                 if (!sh->geneve_tlv_option_resource) {
9209                         claim_zero(mlx5_devx_cmd_destroy(obj));
9210                         ret = rte_flow_error_set(error, ENOMEM,
9211                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
9212                                 "GENEVE TLV object memory allocation failed");
9213                         goto exit;
9214                 }
9215                 geneve_opt_resource = sh->geneve_tlv_option_resource;
9216                 geneve_opt_resource->obj = obj;
9217                 geneve_opt_resource->option_class = geneve_opt_v->option_class;
9218                 geneve_opt_resource->option_type = geneve_opt_v->option_type;
9219                 geneve_opt_resource->length = geneve_opt_v->option_len;
9220                 __atomic_store_n(&geneve_opt_resource->refcnt, 1,
9221                                 __ATOMIC_RELAXED);
9222         }
9223 exit:
9224         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
9225         return ret;
9226 }
9227
9228 /**
9229  * Add Geneve TLV option item to matcher.
9230  *
9231  * @param[in, out] dev
9232  *   Pointer to rte_eth_dev structure.
9233  * @param[in, out] matcher
9234  *   Flow matcher.
9235  * @param[in, out] key
9236  *   Flow matcher value.
9237  * @param[in] item
9238  *   Flow pattern to translate.
9239  * @param[out] error
9240  *   Pointer to error structure.
9241  */
9242 static int
9243 flow_dv_translate_item_geneve_opt(struct rte_eth_dev *dev, void *matcher,
9244                                   void *key, const struct rte_flow_item *item,
9245                                   struct rte_flow_error *error)
9246 {
9247         const struct rte_flow_item_geneve_opt *geneve_opt_m = item->mask;
9248         const struct rte_flow_item_geneve_opt *geneve_opt_v = item->spec;
9249         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9250         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9251         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9252                         misc_parameters_3);
9253         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9254         rte_be32_t opt_data_key = 0, opt_data_mask = 0;
9255         int ret = 0;
9256
9257         if (!geneve_opt_v)
9258                 return -1;
9259         if (!geneve_opt_m)
9260                 geneve_opt_m = &rte_flow_item_geneve_opt_mask;
9261         ret = flow_dev_geneve_tlv_option_resource_register(dev, item,
9262                                                            error);
9263         if (ret) {
9264                 DRV_LOG(ERR, "Failed to create geneve_tlv_obj");
9265                 return ret;
9266         }
9267         /*
9268          * Set the option length in GENEVE header if not requested.
9269          * The GENEVE TLV option length is expressed by the option length field
9270          * in the GENEVE header.
9271          * If the option length was not requested but the GENEVE TLV option item
9272          * is present we set the option length field implicitly.
9273          */
9274         if (!MLX5_GET16(fte_match_set_misc, misc_m, geneve_opt_len)) {
9275                 MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
9276                          MLX5_GENEVE_OPTLEN_MASK);
9277                 MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
9278                          geneve_opt_v->option_len + 1);
9279         }
9280         /* Set the data. */
9281         if (geneve_opt_v->data) {
9282                 memcpy(&opt_data_key, geneve_opt_v->data,
9283                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
9284                                 sizeof(opt_data_key)));
9285                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
9286                                 sizeof(opt_data_key));
9287                 memcpy(&opt_data_mask, geneve_opt_m->data,
9288                         RTE_MIN((uint32_t)(geneve_opt_v->option_len * 4),
9289                                 sizeof(opt_data_mask)));
9290                 MLX5_ASSERT((uint32_t)(geneve_opt_v->option_len * 4) <=
9291                                 sizeof(opt_data_mask));
9292                 MLX5_SET(fte_match_set_misc3, misc3_m,
9293                                 geneve_tlv_option_0_data,
9294                                 rte_be_to_cpu_32(opt_data_mask));
9295                 MLX5_SET(fte_match_set_misc3, misc3_v,
9296                                 geneve_tlv_option_0_data,
9297                         rte_be_to_cpu_32(opt_data_key & opt_data_mask));
9298         }
9299         return ret;
9300 }
9301
9302 /**
9303  * Add MPLS item to matcher and to the value.
9304  *
9305  * @param[in, out] matcher
9306  *   Flow matcher.
9307  * @param[in, out] key
9308  *   Flow matcher value.
9309  * @param[in] item
9310  *   Flow pattern to translate.
9311  * @param[in] prev_layer
9312  *   The protocol layer indicated in previous item.
9313  * @param[in] inner
9314  *   Item is inner pattern.
9315  */
9316 static void
9317 flow_dv_translate_item_mpls(void *matcher, void *key,
9318                             const struct rte_flow_item *item,
9319                             uint64_t prev_layer,
9320                             int inner)
9321 {
9322         const uint32_t *in_mpls_m = item->mask;
9323         const uint32_t *in_mpls_v = item->spec;
9324         uint32_t *out_mpls_m = 0;
9325         uint32_t *out_mpls_v = 0;
9326         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9327         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9328         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
9329                                      misc_parameters_2);
9330         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9331         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
9332         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9333
9334         switch (prev_layer) {
9335         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9336                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
9337                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
9338                          MLX5_UDP_PORT_MPLS);
9339                 break;
9340         case MLX5_FLOW_LAYER_GRE:
9341                 /* Fall-through. */
9342         case MLX5_FLOW_LAYER_GRE_KEY:
9343                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
9344                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
9345                          RTE_ETHER_TYPE_MPLS);
9346                 break;
9347         default:
9348                 break;
9349         }
9350         if (!in_mpls_v)
9351                 return;
9352         if (!in_mpls_m)
9353                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
9354         switch (prev_layer) {
9355         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
9356                 out_mpls_m =
9357                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9358                                                  outer_first_mpls_over_udp);
9359                 out_mpls_v =
9360                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9361                                                  outer_first_mpls_over_udp);
9362                 break;
9363         case MLX5_FLOW_LAYER_GRE:
9364                 out_mpls_m =
9365                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
9366                                                  outer_first_mpls_over_gre);
9367                 out_mpls_v =
9368                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
9369                                                  outer_first_mpls_over_gre);
9370                 break;
9371         default:
9372                 /* Inner MPLS not over GRE is not supported. */
9373                 if (!inner) {
9374                         out_mpls_m =
9375                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9376                                                          misc2_m,
9377                                                          outer_first_mpls);
9378                         out_mpls_v =
9379                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
9380                                                          misc2_v,
9381                                                          outer_first_mpls);
9382                 }
9383                 break;
9384         }
9385         if (out_mpls_m && out_mpls_v) {
9386                 *out_mpls_m = *in_mpls_m;
9387                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
9388         }
9389 }
9390
9391 /**
9392  * Add metadata register item to matcher
9393  *
9394  * @param[in, out] matcher
9395  *   Flow matcher.
9396  * @param[in, out] key
9397  *   Flow matcher value.
9398  * @param[in] reg_type
9399  *   Type of device metadata register
9400  * @param[in] value
9401  *   Register value
9402  * @param[in] mask
9403  *   Register mask
9404  */
9405 static void
9406 flow_dv_match_meta_reg(void *matcher, void *key,
9407                        enum modify_reg reg_type,
9408                        uint32_t data, uint32_t mask)
9409 {
9410         void *misc2_m =
9411                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
9412         void *misc2_v =
9413                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
9414         uint32_t temp;
9415
9416         data &= mask;
9417         switch (reg_type) {
9418         case REG_A:
9419                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
9420                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
9421                 break;
9422         case REG_B:
9423                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
9424                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
9425                 break;
9426         case REG_C_0:
9427                 /*
9428                  * The metadata register C0 field might be divided into
9429                  * source vport index and META item value, we should set
9430                  * this field according to specified mask, not as whole one.
9431                  */
9432                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
9433                 temp |= mask;
9434                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
9435                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
9436                 temp &= ~mask;
9437                 temp |= data;
9438                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
9439                 break;
9440         case REG_C_1:
9441                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
9442                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
9443                 break;
9444         case REG_C_2:
9445                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
9446                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
9447                 break;
9448         case REG_C_3:
9449                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
9450                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
9451                 break;
9452         case REG_C_4:
9453                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
9454                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
9455                 break;
9456         case REG_C_5:
9457                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
9458                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
9459                 break;
9460         case REG_C_6:
9461                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
9462                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
9463                 break;
9464         case REG_C_7:
9465                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
9466                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
9467                 break;
9468         default:
9469                 MLX5_ASSERT(false);
9470                 break;
9471         }
9472 }
9473
9474 /**
9475  * Add MARK item to matcher
9476  *
9477  * @param[in] dev
9478  *   The device to configure through.
9479  * @param[in, out] matcher
9480  *   Flow matcher.
9481  * @param[in, out] key
9482  *   Flow matcher value.
9483  * @param[in] item
9484  *   Flow pattern to translate.
9485  */
9486 static void
9487 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
9488                             void *matcher, void *key,
9489                             const struct rte_flow_item *item)
9490 {
9491         struct mlx5_priv *priv = dev->data->dev_private;
9492         const struct rte_flow_item_mark *mark;
9493         uint32_t value;
9494         uint32_t mask;
9495
9496         mark = item->mask ? (const void *)item->mask :
9497                             &rte_flow_item_mark_mask;
9498         mask = mark->id & priv->sh->dv_mark_mask;
9499         mark = (const void *)item->spec;
9500         MLX5_ASSERT(mark);
9501         value = mark->id & priv->sh->dv_mark_mask & mask;
9502         if (mask) {
9503                 enum modify_reg reg;
9504
9505                 /* Get the metadata register index for the mark. */
9506                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
9507                 MLX5_ASSERT(reg > 0);
9508                 if (reg == REG_C_0) {
9509                         struct mlx5_priv *priv = dev->data->dev_private;
9510                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9511                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9512
9513                         mask &= msk_c0;
9514                         mask <<= shl_c0;
9515                         value <<= shl_c0;
9516                 }
9517                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9518         }
9519 }
9520
9521 /**
9522  * Add META item to matcher
9523  *
9524  * @param[in] dev
9525  *   The devich to configure through.
9526  * @param[in, out] matcher
9527  *   Flow matcher.
9528  * @param[in, out] key
9529  *   Flow matcher value.
9530  * @param[in] attr
9531  *   Attributes of flow that includes this item.
9532  * @param[in] item
9533  *   Flow pattern to translate.
9534  */
9535 static void
9536 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
9537                             void *matcher, void *key,
9538                             const struct rte_flow_attr *attr,
9539                             const struct rte_flow_item *item)
9540 {
9541         const struct rte_flow_item_meta *meta_m;
9542         const struct rte_flow_item_meta *meta_v;
9543
9544         meta_m = (const void *)item->mask;
9545         if (!meta_m)
9546                 meta_m = &rte_flow_item_meta_mask;
9547         meta_v = (const void *)item->spec;
9548         if (meta_v) {
9549                 int reg;
9550                 uint32_t value = meta_v->data;
9551                 uint32_t mask = meta_m->data;
9552
9553                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
9554                 if (reg < 0)
9555                         return;
9556                 MLX5_ASSERT(reg != REG_NON);
9557                 if (reg == REG_C_0) {
9558                         struct mlx5_priv *priv = dev->data->dev_private;
9559                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9560                         uint32_t shl_c0 = rte_bsf32(msk_c0);
9561
9562                         mask &= msk_c0;
9563                         mask <<= shl_c0;
9564                         value <<= shl_c0;
9565                 }
9566                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
9567         }
9568 }
9569
9570 /**
9571  * Add vport metadata Reg C0 item to matcher
9572  *
9573  * @param[in, out] matcher
9574  *   Flow matcher.
9575  * @param[in, out] key
9576  *   Flow matcher value.
9577  * @param[in] reg
9578  *   Flow pattern to translate.
9579  */
9580 static void
9581 flow_dv_translate_item_meta_vport(void *matcher, void *key,
9582                                   uint32_t value, uint32_t mask)
9583 {
9584         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
9585 }
9586
9587 /**
9588  * Add tag item to matcher
9589  *
9590  * @param[in] dev
9591  *   The devich to configure through.
9592  * @param[in, out] matcher
9593  *   Flow matcher.
9594  * @param[in, out] key
9595  *   Flow matcher value.
9596  * @param[in] item
9597  *   Flow pattern to translate.
9598  */
9599 static void
9600 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
9601                                 void *matcher, void *key,
9602                                 const struct rte_flow_item *item)
9603 {
9604         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
9605         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
9606         uint32_t mask, value;
9607
9608         MLX5_ASSERT(tag_v);
9609         value = tag_v->data;
9610         mask = tag_m ? tag_m->data : UINT32_MAX;
9611         if (tag_v->id == REG_C_0) {
9612                 struct mlx5_priv *priv = dev->data->dev_private;
9613                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
9614                 uint32_t shl_c0 = rte_bsf32(msk_c0);
9615
9616                 mask &= msk_c0;
9617                 mask <<= shl_c0;
9618                 value <<= shl_c0;
9619         }
9620         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
9621 }
9622
9623 /**
9624  * Add TAG item to matcher
9625  *
9626  * @param[in] dev
9627  *   The devich to configure through.
9628  * @param[in, out] matcher
9629  *   Flow matcher.
9630  * @param[in, out] key
9631  *   Flow matcher value.
9632  * @param[in] item
9633  *   Flow pattern to translate.
9634  */
9635 static void
9636 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
9637                            void *matcher, void *key,
9638                            const struct rte_flow_item *item)
9639 {
9640         const struct rte_flow_item_tag *tag_v = item->spec;
9641         const struct rte_flow_item_tag *tag_m = item->mask;
9642         enum modify_reg reg;
9643
9644         MLX5_ASSERT(tag_v);
9645         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
9646         /* Get the metadata register index for the tag. */
9647         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
9648         MLX5_ASSERT(reg > 0);
9649         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
9650 }
9651
9652 /**
9653  * Add source vport match to the specified matcher.
9654  *
9655  * @param[in, out] matcher
9656  *   Flow matcher.
9657  * @param[in, out] key
9658  *   Flow matcher value.
9659  * @param[in] port
9660  *   Source vport value to match
9661  * @param[in] mask
9662  *   Mask
9663  */
9664 static void
9665 flow_dv_translate_item_source_vport(void *matcher, void *key,
9666                                     int16_t port, uint16_t mask)
9667 {
9668         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
9669         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
9670
9671         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
9672         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
9673 }
9674
9675 /**
9676  * Translate port-id item to eswitch match on  port-id.
9677  *
9678  * @param[in] dev
9679  *   The devich to configure through.
9680  * @param[in, out] matcher
9681  *   Flow matcher.
9682  * @param[in, out] key
9683  *   Flow matcher value.
9684  * @param[in] item
9685  *   Flow pattern to translate.
9686  * @param[in]
9687  *   Flow attributes.
9688  *
9689  * @return
9690  *   0 on success, a negative errno value otherwise.
9691  */
9692 static int
9693 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
9694                                void *key, const struct rte_flow_item *item,
9695                                const struct rte_flow_attr *attr)
9696 {
9697         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
9698         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
9699         struct mlx5_priv *priv;
9700         uint16_t mask, id;
9701
9702         mask = pid_m ? pid_m->id : 0xffff;
9703         id = pid_v ? pid_v->id : dev->data->port_id;
9704         priv = mlx5_port_to_eswitch_info(id, item == NULL);
9705         if (!priv)
9706                 return -rte_errno;
9707         /*
9708          * Translate to vport field or to metadata, depending on mode.
9709          * Kernel can use either misc.source_port or half of C0 metadata
9710          * register.
9711          */
9712         if (priv->vport_meta_mask) {
9713                 /*
9714                  * Provide the hint for SW steering library
9715                  * to insert the flow into ingress domain and
9716                  * save the extra vport match.
9717                  */
9718                 if (mask == 0xffff && priv->vport_id == 0xffff &&
9719                     priv->pf_bond < 0 && attr->transfer)
9720                         flow_dv_translate_item_source_vport
9721                                 (matcher, key, priv->vport_id, mask);
9722                 /*
9723                  * We should always set the vport metadata register,
9724                  * otherwise the SW steering library can drop
9725                  * the rule if wire vport metadata value is not zero,
9726                  * it depends on kernel configuration.
9727                  */
9728                 flow_dv_translate_item_meta_vport(matcher, key,
9729                                                   priv->vport_meta_tag,
9730                                                   priv->vport_meta_mask);
9731         } else {
9732                 flow_dv_translate_item_source_vport(matcher, key,
9733                                                     priv->vport_id, mask);
9734         }
9735         return 0;
9736 }
9737
9738 /**
9739  * Add ICMP6 item to matcher and to the value.
9740  *
9741  * @param[in, out] matcher
9742  *   Flow matcher.
9743  * @param[in, out] key
9744  *   Flow matcher value.
9745  * @param[in] item
9746  *   Flow pattern to translate.
9747  * @param[in] inner
9748  *   Item is inner pattern.
9749  */
9750 static void
9751 flow_dv_translate_item_icmp6(void *matcher, void *key,
9752                               const struct rte_flow_item *item,
9753                               int inner)
9754 {
9755         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
9756         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
9757         void *headers_m;
9758         void *headers_v;
9759         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9760                                      misc_parameters_3);
9761         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9762         if (inner) {
9763                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9764                                          inner_headers);
9765                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9766         } else {
9767                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9768                                          outer_headers);
9769                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9770         }
9771         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
9772         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
9773         if (!icmp6_v)
9774                 return;
9775         if (!icmp6_m)
9776                 icmp6_m = &rte_flow_item_icmp6_mask;
9777         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
9778         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
9779                  icmp6_v->type & icmp6_m->type);
9780         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
9781         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
9782                  icmp6_v->code & icmp6_m->code);
9783 }
9784
9785 /**
9786  * Add ICMP item to matcher and to the value.
9787  *
9788  * @param[in, out] matcher
9789  *   Flow matcher.
9790  * @param[in, out] key
9791  *   Flow matcher value.
9792  * @param[in] item
9793  *   Flow pattern to translate.
9794  * @param[in] inner
9795  *   Item is inner pattern.
9796  */
9797 static void
9798 flow_dv_translate_item_icmp(void *matcher, void *key,
9799                             const struct rte_flow_item *item,
9800                             int inner)
9801 {
9802         const struct rte_flow_item_icmp *icmp_m = item->mask;
9803         const struct rte_flow_item_icmp *icmp_v = item->spec;
9804         uint32_t icmp_header_data_m = 0;
9805         uint32_t icmp_header_data_v = 0;
9806         void *headers_m;
9807         void *headers_v;
9808         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9809                                      misc_parameters_3);
9810         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9811         if (inner) {
9812                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9813                                          inner_headers);
9814                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9815         } else {
9816                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9817                                          outer_headers);
9818                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9819         }
9820         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
9821         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
9822         if (!icmp_v)
9823                 return;
9824         if (!icmp_m)
9825                 icmp_m = &rte_flow_item_icmp_mask;
9826         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
9827                  icmp_m->hdr.icmp_type);
9828         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
9829                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
9830         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
9831                  icmp_m->hdr.icmp_code);
9832         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
9833                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
9834         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
9835         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
9836         if (icmp_header_data_m) {
9837                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
9838                 icmp_header_data_v |=
9839                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
9840                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
9841                          icmp_header_data_m);
9842                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
9843                          icmp_header_data_v & icmp_header_data_m);
9844         }
9845 }
9846
9847 /**
9848  * Add GTP item to matcher and to the value.
9849  *
9850  * @param[in, out] matcher
9851  *   Flow matcher.
9852  * @param[in, out] key
9853  *   Flow matcher value.
9854  * @param[in] item
9855  *   Flow pattern to translate.
9856  * @param[in] inner
9857  *   Item is inner pattern.
9858  */
9859 static void
9860 flow_dv_translate_item_gtp(void *matcher, void *key,
9861                            const struct rte_flow_item *item, int inner)
9862 {
9863         const struct rte_flow_item_gtp *gtp_m = item->mask;
9864         const struct rte_flow_item_gtp *gtp_v = item->spec;
9865         void *headers_m;
9866         void *headers_v;
9867         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9868                                      misc_parameters_3);
9869         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9870         uint16_t dport = RTE_GTPU_UDP_PORT;
9871
9872         if (inner) {
9873                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9874                                          inner_headers);
9875                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
9876         } else {
9877                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
9878                                          outer_headers);
9879                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
9880         }
9881         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
9882                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
9883                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
9884         }
9885         if (!gtp_v)
9886                 return;
9887         if (!gtp_m)
9888                 gtp_m = &rte_flow_item_gtp_mask;
9889         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
9890                  gtp_m->v_pt_rsv_flags);
9891         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
9892                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
9893         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
9894         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
9895                  gtp_v->msg_type & gtp_m->msg_type);
9896         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
9897                  rte_be_to_cpu_32(gtp_m->teid));
9898         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
9899                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
9900 }
9901
9902 /**
9903  * Add GTP PSC item to matcher.
9904  *
9905  * @param[in, out] matcher
9906  *   Flow matcher.
9907  * @param[in, out] key
9908  *   Flow matcher value.
9909  * @param[in] item
9910  *   Flow pattern to translate.
9911  */
9912 static int
9913 flow_dv_translate_item_gtp_psc(void *matcher, void *key,
9914                                const struct rte_flow_item *item)
9915 {
9916         const struct rte_flow_item_gtp_psc *gtp_psc_m = item->mask;
9917         const struct rte_flow_item_gtp_psc *gtp_psc_v = item->spec;
9918         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
9919                         misc_parameters_3);
9920         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
9921         union {
9922                 uint32_t w32;
9923                 struct {
9924                         uint16_t seq_num;
9925                         uint8_t npdu_num;
9926                         uint8_t next_ext_header_type;
9927                 };
9928         } dw_2;
9929         uint8_t gtp_flags;
9930
9931         /* Always set E-flag match on one, regardless of GTP item settings. */
9932         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_m, gtpu_msg_flags);
9933         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
9934         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags, gtp_flags);
9935         gtp_flags = MLX5_GET(fte_match_set_misc3, misc3_v, gtpu_msg_flags);
9936         gtp_flags |= MLX5_GTP_EXT_HEADER_FLAG;
9937         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags, gtp_flags);
9938         /*Set next extension header type. */
9939         dw_2.seq_num = 0;
9940         dw_2.npdu_num = 0;
9941         dw_2.next_ext_header_type = 0xff;
9942         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_dw_2,
9943                  rte_cpu_to_be_32(dw_2.w32));
9944         dw_2.seq_num = 0;
9945         dw_2.npdu_num = 0;
9946         dw_2.next_ext_header_type = 0x85;
9947         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_dw_2,
9948                  rte_cpu_to_be_32(dw_2.w32));
9949         if (gtp_psc_v) {
9950                 union {
9951                         uint32_t w32;
9952                         struct {
9953                                 uint8_t len;
9954                                 uint8_t type_flags;
9955                                 uint8_t qfi;
9956                                 uint8_t reserved;
9957                         };
9958                 } dw_0;
9959
9960                 /*Set extension header PDU type and Qos. */
9961                 if (!gtp_psc_m)
9962                         gtp_psc_m = &rte_flow_item_gtp_psc_mask;
9963                 dw_0.w32 = 0;
9964                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_m->pdu_type);
9965                 dw_0.qfi = gtp_psc_m->qfi;
9966                 MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_first_ext_dw_0,
9967                          rte_cpu_to_be_32(dw_0.w32));
9968                 dw_0.w32 = 0;
9969                 dw_0.type_flags = MLX5_GTP_PDU_TYPE_SHIFT(gtp_psc_v->pdu_type &
9970                                                         gtp_psc_m->pdu_type);
9971                 dw_0.qfi = gtp_psc_v->qfi & gtp_psc_m->qfi;
9972                 MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_first_ext_dw_0,
9973                          rte_cpu_to_be_32(dw_0.w32));
9974         }
9975         return 0;
9976 }
9977
9978 /**
9979  * Add eCPRI item to matcher and to the value.
9980  *
9981  * @param[in] dev
9982  *   The devich to configure through.
9983  * @param[in, out] matcher
9984  *   Flow matcher.
9985  * @param[in, out] key
9986  *   Flow matcher value.
9987  * @param[in] item
9988  *   Flow pattern to translate.
9989  * @param[in] samples
9990  *   Sample IDs to be used in the matching.
9991  */
9992 static void
9993 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
9994                              void *key, const struct rte_flow_item *item)
9995 {
9996         struct mlx5_priv *priv = dev->data->dev_private;
9997         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
9998         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
9999         struct rte_ecpri_common_hdr common;
10000         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
10001                                      misc_parameters_4);
10002         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
10003         uint32_t *samples;
10004         void *dw_m;
10005         void *dw_v;
10006
10007         if (!ecpri_v)
10008                 return;
10009         if (!ecpri_m)
10010                 ecpri_m = &rte_flow_item_ecpri_mask;
10011         /*
10012          * Maximal four DW samples are supported in a single matching now.
10013          * Two are used now for a eCPRI matching:
10014          * 1. Type: one byte, mask should be 0x00ff0000 in network order
10015          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
10016          *    if any.
10017          */
10018         if (!ecpri_m->hdr.common.u32)
10019                 return;
10020         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
10021         /* Need to take the whole DW as the mask to fill the entry. */
10022         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
10023                             prog_sample_field_value_0);
10024         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
10025                             prog_sample_field_value_0);
10026         /* Already big endian (network order) in the header. */
10027         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
10028         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32 & ecpri_m->hdr.common.u32;
10029         /* Sample#0, used for matching type, offset 0. */
10030         MLX5_SET(fte_match_set_misc4, misc4_m,
10031                  prog_sample_field_id_0, samples[0]);
10032         /* It makes no sense to set the sample ID in the mask field. */
10033         MLX5_SET(fte_match_set_misc4, misc4_v,
10034                  prog_sample_field_id_0, samples[0]);
10035         /*
10036          * Checking if message body part needs to be matched.
10037          * Some wildcard rules only matching type field should be supported.
10038          */
10039         if (ecpri_m->hdr.dummy[0]) {
10040                 common.u32 = rte_be_to_cpu_32(ecpri_v->hdr.common.u32);
10041                 switch (common.type) {
10042                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
10043                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
10044                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
10045                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
10046                                             prog_sample_field_value_1);
10047                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
10048                                             prog_sample_field_value_1);
10049                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
10050                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0] &
10051                                             ecpri_m->hdr.dummy[0];
10052                         /* Sample#1, to match message body, offset 4. */
10053                         MLX5_SET(fte_match_set_misc4, misc4_m,
10054                                  prog_sample_field_id_1, samples[1]);
10055                         MLX5_SET(fte_match_set_misc4, misc4_v,
10056                                  prog_sample_field_id_1, samples[1]);
10057                         break;
10058                 default:
10059                         /* Others, do not match any sample ID. */
10060                         break;
10061                 }
10062         }
10063 }
10064
10065 /*
10066  * Add connection tracking status item to matcher
10067  *
10068  * @param[in] dev
10069  *   The devich to configure through.
10070  * @param[in, out] matcher
10071  *   Flow matcher.
10072  * @param[in, out] key
10073  *   Flow matcher value.
10074  * @param[in] item
10075  *   Flow pattern to translate.
10076  */
10077 static void
10078 flow_dv_translate_item_aso_ct(struct rte_eth_dev *dev,
10079                               void *matcher, void *key,
10080                               const struct rte_flow_item *item)
10081 {
10082         uint32_t reg_value = 0;
10083         int reg_id;
10084         /* 8LSB 0b 11/0000/11, middle 4 bits are reserved. */
10085         uint32_t reg_mask = 0;
10086         const struct rte_flow_item_conntrack *spec = item->spec;
10087         const struct rte_flow_item_conntrack *mask = item->mask;
10088         uint32_t flags;
10089         struct rte_flow_error error;
10090
10091         if (!mask)
10092                 mask = &rte_flow_item_conntrack_mask;
10093         if (!spec || !mask->flags)
10094                 return;
10095         flags = spec->flags & mask->flags;
10096         /* The conflict should be checked in the validation. */
10097         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_VALID)
10098                 reg_value |= MLX5_CT_SYNDROME_VALID;
10099         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
10100                 reg_value |= MLX5_CT_SYNDROME_STATE_CHANGE;
10101         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_INVALID)
10102                 reg_value |= MLX5_CT_SYNDROME_INVALID;
10103         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED)
10104                 reg_value |= MLX5_CT_SYNDROME_TRAP;
10105         if (flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
10106                 reg_value |= MLX5_CT_SYNDROME_BAD_PACKET;
10107         if (mask->flags & (RTE_FLOW_CONNTRACK_PKT_STATE_VALID |
10108                            RTE_FLOW_CONNTRACK_PKT_STATE_INVALID |
10109                            RTE_FLOW_CONNTRACK_PKT_STATE_DISABLED))
10110                 reg_mask |= 0xc0;
10111         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_CHANGED)
10112                 reg_mask |= MLX5_CT_SYNDROME_STATE_CHANGE;
10113         if (mask->flags & RTE_FLOW_CONNTRACK_PKT_STATE_BAD)
10114                 reg_mask |= MLX5_CT_SYNDROME_BAD_PACKET;
10115         /* The REG_C_x value could be saved during startup. */
10116         reg_id = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, &error);
10117         if (reg_id == REG_NON)
10118                 return;
10119         flow_dv_match_meta_reg(matcher, key, (enum modify_reg)reg_id,
10120                                reg_value, reg_mask);
10121 }
10122
10123 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
10124
10125 #define HEADER_IS_ZERO(match_criteria, headers)                              \
10126         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
10127                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
10128
10129 /**
10130  * Calculate flow matcher enable bitmap.
10131  *
10132  * @param match_criteria
10133  *   Pointer to flow matcher criteria.
10134  *
10135  * @return
10136  *   Bitmap of enabled fields.
10137  */
10138 static uint8_t
10139 flow_dv_matcher_enable(uint32_t *match_criteria)
10140 {
10141         uint8_t match_criteria_enable;
10142
10143         match_criteria_enable =
10144                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
10145                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
10146         match_criteria_enable |=
10147                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
10148                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
10149         match_criteria_enable |=
10150                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
10151                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
10152         match_criteria_enable |=
10153                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
10154                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
10155         match_criteria_enable |=
10156                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
10157                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
10158         match_criteria_enable |=
10159                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
10160                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
10161         match_criteria_enable |=
10162                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_5)) <<
10163                 MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT;
10164         return match_criteria_enable;
10165 }
10166
10167 static void
10168 __flow_dv_adjust_buf_size(size_t *size, uint8_t match_criteria)
10169 {
10170         /*
10171          * Check flow matching criteria first, subtract misc5/4 length if flow
10172          * doesn't own misc5/4 parameters. In some old rdma-core releases,
10173          * misc5/4 are not supported, and matcher creation failure is expected
10174          * w/o subtration. If misc5 is provided, misc4 must be counted in since
10175          * misc5 is right after misc4.
10176          */
10177         if (!(match_criteria & (1 << MLX5_MATCH_CRITERIA_ENABLE_MISC5_BIT))) {
10178                 *size = MLX5_ST_SZ_BYTES(fte_match_param) -
10179                         MLX5_ST_SZ_BYTES(fte_match_set_misc5);
10180                 if (!(match_criteria & (1 <<
10181                         MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT))) {
10182                         *size -= MLX5_ST_SZ_BYTES(fte_match_set_misc4);
10183                 }
10184         }
10185 }
10186
10187 static struct mlx5_list_entry *
10188 flow_dv_matcher_clone_cb(void *tool_ctx __rte_unused,
10189                          struct mlx5_list_entry *entry, void *cb_ctx)
10190 {
10191         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10192         struct mlx5_flow_dv_matcher *ref = ctx->data;
10193         struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
10194                                                             typeof(*tbl), tbl);
10195         struct mlx5_flow_dv_matcher *resource = mlx5_malloc(MLX5_MEM_ANY,
10196                                                             sizeof(*resource),
10197                                                             0, SOCKET_ID_ANY);
10198
10199         if (!resource) {
10200                 rte_flow_error_set(ctx->error, ENOMEM,
10201                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10202                                    "cannot create matcher");
10203                 return NULL;
10204         }
10205         memcpy(resource, entry, sizeof(*resource));
10206         resource->tbl = &tbl->tbl;
10207         return &resource->entry;
10208 }
10209
10210 static void
10211 flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused,
10212                              struct mlx5_list_entry *entry)
10213 {
10214         mlx5_free(entry);
10215 }
10216
10217 struct mlx5_list_entry *
10218 flow_dv_tbl_create_cb(void *tool_ctx, void *cb_ctx)
10219 {
10220         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10221         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10222         struct rte_eth_dev *dev = ctx->dev;
10223         struct mlx5_flow_tbl_data_entry *tbl_data;
10224         struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data2;
10225         struct rte_flow_error *error = ctx->error;
10226         union mlx5_flow_tbl_key key = { .v64 = *(uint64_t *)(ctx->data) };
10227         struct mlx5_flow_tbl_resource *tbl;
10228         void *domain;
10229         uint32_t idx = 0;
10230         int ret;
10231
10232         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
10233         if (!tbl_data) {
10234                 rte_flow_error_set(error, ENOMEM,
10235                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10236                                    NULL,
10237                                    "cannot allocate flow table data entry");
10238                 return NULL;
10239         }
10240         tbl_data->idx = idx;
10241         tbl_data->tunnel = tt_prm->tunnel;
10242         tbl_data->group_id = tt_prm->group_id;
10243         tbl_data->external = !!tt_prm->external;
10244         tbl_data->tunnel_offload = is_tunnel_offload_active(dev);
10245         tbl_data->is_egress = !!key.is_egress;
10246         tbl_data->is_transfer = !!key.is_fdb;
10247         tbl_data->dummy = !!key.dummy;
10248         tbl_data->level = key.level;
10249         tbl_data->id = key.id;
10250         tbl = &tbl_data->tbl;
10251         if (key.dummy)
10252                 return &tbl_data->entry;
10253         if (key.is_fdb)
10254                 domain = sh->fdb_domain;
10255         else if (key.is_egress)
10256                 domain = sh->tx_domain;
10257         else
10258                 domain = sh->rx_domain;
10259         ret = mlx5_flow_os_create_flow_tbl(domain, key.level, &tbl->obj);
10260         if (ret) {
10261                 rte_flow_error_set(error, ENOMEM,
10262                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10263                                    NULL, "cannot create flow table object");
10264                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10265                 return NULL;
10266         }
10267         if (key.level != 0) {
10268                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
10269                                         (tbl->obj, &tbl_data->jump.action);
10270                 if (ret) {
10271                         rte_flow_error_set(error, ENOMEM,
10272                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10273                                            NULL,
10274                                            "cannot create flow jump action");
10275                         mlx5_flow_os_destroy_flow_tbl(tbl->obj);
10276                         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10277                         return NULL;
10278                 }
10279         }
10280         MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
10281               key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
10282               key.level, key.id);
10283         tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
10284                                               flow_dv_matcher_create_cb,
10285                                               flow_dv_matcher_match_cb,
10286                                               flow_dv_matcher_remove_cb,
10287                                               flow_dv_matcher_clone_cb,
10288                                               flow_dv_matcher_clone_free_cb);
10289         if (!tbl_data->matchers) {
10290                 rte_flow_error_set(error, ENOMEM,
10291                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10292                                    NULL,
10293                                    "cannot create tbl matcher list");
10294                 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
10295                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
10296                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
10297                 return NULL;
10298         }
10299         return &tbl_data->entry;
10300 }
10301
10302 int
10303 flow_dv_tbl_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
10304                      void *cb_ctx)
10305 {
10306         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10307         struct mlx5_flow_tbl_data_entry *tbl_data =
10308                 container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10309         union mlx5_flow_tbl_key key = { .v64 =  *(uint64_t *)(ctx->data) };
10310
10311         return tbl_data->level != key.level ||
10312                tbl_data->id != key.id ||
10313                tbl_data->dummy != key.dummy ||
10314                tbl_data->is_transfer != !!key.is_fdb ||
10315                tbl_data->is_egress != !!key.is_egress;
10316 }
10317
10318 struct mlx5_list_entry *
10319 flow_dv_tbl_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
10320                       void *cb_ctx)
10321 {
10322         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10323         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10324         struct mlx5_flow_tbl_data_entry *tbl_data;
10325         struct rte_flow_error *error = ctx->error;
10326         uint32_t idx = 0;
10327
10328         tbl_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
10329         if (!tbl_data) {
10330                 rte_flow_error_set(error, ENOMEM,
10331                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10332                                    NULL,
10333                                    "cannot allocate flow table data entry");
10334                 return NULL;
10335         }
10336         memcpy(tbl_data, oentry, sizeof(*tbl_data));
10337         tbl_data->idx = idx;
10338         return &tbl_data->entry;
10339 }
10340
10341 void
10342 flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10343 {
10344         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10345         struct mlx5_flow_tbl_data_entry *tbl_data =
10346                     container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10347
10348         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
10349 }
10350
10351 /**
10352  * Get a flow table.
10353  *
10354  * @param[in, out] dev
10355  *   Pointer to rte_eth_dev structure.
10356  * @param[in] table_level
10357  *   Table level to use.
10358  * @param[in] egress
10359  *   Direction of the table.
10360  * @param[in] transfer
10361  *   E-Switch or NIC flow.
10362  * @param[in] dummy
10363  *   Dummy entry for dv API.
10364  * @param[in] table_id
10365  *   Table id to use.
10366  * @param[out] error
10367  *   pointer to error structure.
10368  *
10369  * @return
10370  *   Returns tables resource based on the index, NULL in case of failed.
10371  */
10372 struct mlx5_flow_tbl_resource *
10373 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
10374                          uint32_t table_level, uint8_t egress,
10375                          uint8_t transfer,
10376                          bool external,
10377                          const struct mlx5_flow_tunnel *tunnel,
10378                          uint32_t group_id, uint8_t dummy,
10379                          uint32_t table_id,
10380                          struct rte_flow_error *error)
10381 {
10382         struct mlx5_priv *priv = dev->data->dev_private;
10383         union mlx5_flow_tbl_key table_key = {
10384                 {
10385                         .level = table_level,
10386                         .id = table_id,
10387                         .reserved = 0,
10388                         .dummy = !!dummy,
10389                         .is_fdb = !!transfer,
10390                         .is_egress = !!egress,
10391                 }
10392         };
10393         struct mlx5_flow_tbl_tunnel_prm tt_prm = {
10394                 .tunnel = tunnel,
10395                 .group_id = group_id,
10396                 .external = external,
10397         };
10398         struct mlx5_flow_cb_ctx ctx = {
10399                 .dev = dev,
10400                 .error = error,
10401                 .data = &table_key.v64,
10402                 .data2 = &tt_prm,
10403         };
10404         struct mlx5_list_entry *entry;
10405         struct mlx5_flow_tbl_data_entry *tbl_data;
10406
10407         entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx);
10408         if (!entry) {
10409                 rte_flow_error_set(error, ENOMEM,
10410                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10411                                    "cannot get table");
10412                 return NULL;
10413         }
10414         DRV_LOG(DEBUG, "table_level %u table_id %u "
10415                 "tunnel %u group %u registered.",
10416                 table_level, table_id,
10417                 tunnel ? tunnel->tunnel_id : 0, group_id);
10418         tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10419         return &tbl_data->tbl;
10420 }
10421
10422 void
10423 flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10424 {
10425         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10426         struct mlx5_flow_tbl_data_entry *tbl_data =
10427                     container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
10428
10429         MLX5_ASSERT(entry && sh);
10430         if (tbl_data->jump.action)
10431                 mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
10432         if (tbl_data->tbl.obj)
10433                 mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj);
10434         if (tbl_data->tunnel_offload && tbl_data->external) {
10435                 struct mlx5_list_entry *he;
10436                 struct mlx5_hlist *tunnel_grp_hash;
10437                 struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
10438                 union tunnel_tbl_key tunnel_key = {
10439                         .tunnel_id = tbl_data->tunnel ?
10440                                         tbl_data->tunnel->tunnel_id : 0,
10441                         .group = tbl_data->group_id
10442                 };
10443                 uint32_t table_level = tbl_data->level;
10444                 struct mlx5_flow_cb_ctx ctx = {
10445                         .data = (void *)&tunnel_key.val,
10446                 };
10447
10448                 tunnel_grp_hash = tbl_data->tunnel ?
10449                                         tbl_data->tunnel->groups :
10450                                         thub->groups;
10451                 he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, &ctx);
10452                 if (he)
10453                         mlx5_hlist_unregister(tunnel_grp_hash, he);
10454                 DRV_LOG(DEBUG,
10455                         "table_level %u id %u tunnel %u group %u released.",
10456                         table_level,
10457                         tbl_data->id,
10458                         tbl_data->tunnel ?
10459                         tbl_data->tunnel->tunnel_id : 0,
10460                         tbl_data->group_id);
10461         }
10462         mlx5_list_destroy(tbl_data->matchers);
10463         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
10464 }
10465
10466 /**
10467  * Release a flow table.
10468  *
10469  * @param[in] sh
10470  *   Pointer to device shared structure.
10471  * @param[in] tbl
10472  *   Table resource to be released.
10473  *
10474  * @return
10475  *   Returns 0 if table was released, else return 1;
10476  */
10477 static int
10478 flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
10479                              struct mlx5_flow_tbl_resource *tbl)
10480 {
10481         struct mlx5_flow_tbl_data_entry *tbl_data =
10482                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10483
10484         if (!tbl)
10485                 return 0;
10486         return mlx5_hlist_unregister(sh->flow_tbls, &tbl_data->entry);
10487 }
10488
10489 int
10490 flow_dv_matcher_match_cb(void *tool_ctx __rte_unused,
10491                          struct mlx5_list_entry *entry, void *cb_ctx)
10492 {
10493         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10494         struct mlx5_flow_dv_matcher *ref = ctx->data;
10495         struct mlx5_flow_dv_matcher *cur = container_of(entry, typeof(*cur),
10496                                                         entry);
10497
10498         return cur->crc != ref->crc ||
10499                cur->priority != ref->priority ||
10500                memcmp((const void *)cur->mask.buf,
10501                       (const void *)ref->mask.buf, ref->mask.size);
10502 }
10503
10504 struct mlx5_list_entry *
10505 flow_dv_matcher_create_cb(void *tool_ctx, void *cb_ctx)
10506 {
10507         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10508         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10509         struct mlx5_flow_dv_matcher *ref = ctx->data;
10510         struct mlx5_flow_dv_matcher *resource;
10511         struct mlx5dv_flow_matcher_attr dv_attr = {
10512                 .type = IBV_FLOW_ATTR_NORMAL,
10513                 .match_mask = (void *)&ref->mask,
10514         };
10515         struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
10516                                                             typeof(*tbl), tbl);
10517         int ret;
10518
10519         resource = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*resource), 0,
10520                                SOCKET_ID_ANY);
10521         if (!resource) {
10522                 rte_flow_error_set(ctx->error, ENOMEM,
10523                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10524                                    "cannot create matcher");
10525                 return NULL;
10526         }
10527         *resource = *ref;
10528         dv_attr.match_criteria_enable =
10529                 flow_dv_matcher_enable(resource->mask.buf);
10530         __flow_dv_adjust_buf_size(&ref->mask.size,
10531                                   dv_attr.match_criteria_enable);
10532         dv_attr.priority = ref->priority;
10533         if (tbl->is_egress)
10534                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
10535         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->tbl.obj,
10536                                                &resource->matcher_object);
10537         if (ret) {
10538                 mlx5_free(resource);
10539                 rte_flow_error_set(ctx->error, ENOMEM,
10540                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10541                                    "cannot create matcher");
10542                 return NULL;
10543         }
10544         return &resource->entry;
10545 }
10546
10547 /**
10548  * Register the flow matcher.
10549  *
10550  * @param[in, out] dev
10551  *   Pointer to rte_eth_dev structure.
10552  * @param[in, out] matcher
10553  *   Pointer to flow matcher.
10554  * @param[in, out] key
10555  *   Pointer to flow table key.
10556  * @parm[in, out] dev_flow
10557  *   Pointer to the dev_flow.
10558  * @param[out] error
10559  *   pointer to error structure.
10560  *
10561  * @return
10562  *   0 on success otherwise -errno and errno is set.
10563  */
10564 static int
10565 flow_dv_matcher_register(struct rte_eth_dev *dev,
10566                          struct mlx5_flow_dv_matcher *ref,
10567                          union mlx5_flow_tbl_key *key,
10568                          struct mlx5_flow *dev_flow,
10569                          const struct mlx5_flow_tunnel *tunnel,
10570                          uint32_t group_id,
10571                          struct rte_flow_error *error)
10572 {
10573         struct mlx5_list_entry *entry;
10574         struct mlx5_flow_dv_matcher *resource;
10575         struct mlx5_flow_tbl_resource *tbl;
10576         struct mlx5_flow_tbl_data_entry *tbl_data;
10577         struct mlx5_flow_cb_ctx ctx = {
10578                 .error = error,
10579                 .data = ref,
10580         };
10581         /**
10582          * tunnel offload API requires this registration for cases when
10583          * tunnel match rule was inserted before tunnel set rule.
10584          */
10585         tbl = flow_dv_tbl_resource_get(dev, key->level,
10586                                        key->is_egress, key->is_fdb,
10587                                        dev_flow->external, tunnel,
10588                                        group_id, 0, key->id, error);
10589         if (!tbl)
10590                 return -rte_errno;      /* No need to refill the error info */
10591         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
10592         ref->tbl = tbl;
10593         entry = mlx5_list_register(tbl_data->matchers, &ctx);
10594         if (!entry) {
10595                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
10596                 return rte_flow_error_set(error, ENOMEM,
10597                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10598                                           "cannot allocate ref memory");
10599         }
10600         resource = container_of(entry, typeof(*resource), entry);
10601         dev_flow->handle->dvh.matcher = resource;
10602         return 0;
10603 }
10604
10605 struct mlx5_list_entry *
10606 flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx)
10607 {
10608         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10609         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10610         struct mlx5_flow_dv_tag_resource *entry;
10611         uint32_t idx = 0;
10612         int ret;
10613
10614         entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
10615         if (!entry) {
10616                 rte_flow_error_set(ctx->error, ENOMEM,
10617                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10618                                    "cannot allocate resource memory");
10619                 return NULL;
10620         }
10621         entry->idx = idx;
10622         entry->tag_id = *(uint32_t *)(ctx->data);
10623         ret = mlx5_flow_os_create_flow_action_tag(entry->tag_id,
10624                                                   &entry->action);
10625         if (ret) {
10626                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx);
10627                 rte_flow_error_set(ctx->error, ENOMEM,
10628                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10629                                    NULL, "cannot create action");
10630                 return NULL;
10631         }
10632         return &entry->entry;
10633 }
10634
10635 int
10636 flow_dv_tag_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
10637                      void *cb_ctx)
10638 {
10639         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10640         struct mlx5_flow_dv_tag_resource *tag =
10641                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10642
10643         return *(uint32_t *)(ctx->data) != tag->tag_id;
10644 }
10645
10646 struct mlx5_list_entry *
10647 flow_dv_tag_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
10648                      void *cb_ctx)
10649 {
10650         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10651         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
10652         struct mlx5_flow_dv_tag_resource *entry;
10653         uint32_t idx = 0;
10654
10655         entry = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
10656         if (!entry) {
10657                 rte_flow_error_set(ctx->error, ENOMEM,
10658                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10659                                    "cannot allocate tag resource memory");
10660                 return NULL;
10661         }
10662         memcpy(entry, oentry, sizeof(*entry));
10663         entry->idx = idx;
10664         return &entry->entry;
10665 }
10666
10667 void
10668 flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10669 {
10670         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10671         struct mlx5_flow_dv_tag_resource *tag =
10672                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10673
10674         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
10675 }
10676
10677 /**
10678  * Find existing tag resource or create and register a new one.
10679  *
10680  * @param dev[in, out]
10681  *   Pointer to rte_eth_dev structure.
10682  * @param[in, out] tag_be24
10683  *   Tag value in big endian then R-shift 8.
10684  * @parm[in, out] dev_flow
10685  *   Pointer to the dev_flow.
10686  * @param[out] error
10687  *   pointer to error structure.
10688  *
10689  * @return
10690  *   0 on success otherwise -errno and errno is set.
10691  */
10692 static int
10693 flow_dv_tag_resource_register
10694                         (struct rte_eth_dev *dev,
10695                          uint32_t tag_be24,
10696                          struct mlx5_flow *dev_flow,
10697                          struct rte_flow_error *error)
10698 {
10699         struct mlx5_priv *priv = dev->data->dev_private;
10700         struct mlx5_flow_dv_tag_resource *resource;
10701         struct mlx5_list_entry *entry;
10702         struct mlx5_flow_cb_ctx ctx = {
10703                                         .error = error,
10704                                         .data = &tag_be24,
10705                                         };
10706         struct mlx5_hlist *tag_table;
10707
10708         tag_table = flow_dv_hlist_prepare(priv->sh, &priv->sh->tag_table,
10709                                       "tags",
10710                                       MLX5_TAGS_HLIST_ARRAY_SIZE,
10711                                       false, false, priv->sh,
10712                                       flow_dv_tag_create_cb,
10713                                       flow_dv_tag_match_cb,
10714                                       flow_dv_tag_remove_cb,
10715                                       flow_dv_tag_clone_cb,
10716                                       flow_dv_tag_clone_free_cb);
10717         if (unlikely(!tag_table))
10718                 return -rte_errno;
10719         entry = mlx5_hlist_register(tag_table, tag_be24, &ctx);
10720         if (entry) {
10721                 resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
10722                                         entry);
10723                 dev_flow->handle->dvh.rix_tag = resource->idx;
10724                 dev_flow->dv.tag_resource = resource;
10725                 return 0;
10726         }
10727         return -rte_errno;
10728 }
10729
10730 void
10731 flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
10732 {
10733         struct mlx5_dev_ctx_shared *sh = tool_ctx;
10734         struct mlx5_flow_dv_tag_resource *tag =
10735                    container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
10736
10737         MLX5_ASSERT(tag && sh && tag->action);
10738         claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
10739         DRV_LOG(DEBUG, "Tag %p: removed.", (void *)tag);
10740         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
10741 }
10742
10743 /**
10744  * Release the tag.
10745  *
10746  * @param dev
10747  *   Pointer to Ethernet device.
10748  * @param tag_idx
10749  *   Tag index.
10750  *
10751  * @return
10752  *   1 while a reference on it exists, 0 when freed.
10753  */
10754 static int
10755 flow_dv_tag_release(struct rte_eth_dev *dev,
10756                     uint32_t tag_idx)
10757 {
10758         struct mlx5_priv *priv = dev->data->dev_private;
10759         struct mlx5_flow_dv_tag_resource *tag;
10760
10761         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
10762         if (!tag)
10763                 return 0;
10764         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
10765                 dev->data->port_id, (void *)tag, tag->entry.ref_cnt);
10766         return mlx5_hlist_unregister(priv->sh->tag_table, &tag->entry);
10767 }
10768
10769 /**
10770  * Translate port ID action to vport.
10771  *
10772  * @param[in] dev
10773  *   Pointer to rte_eth_dev structure.
10774  * @param[in] action
10775  *   Pointer to the port ID action.
10776  * @param[out] dst_port_id
10777  *   The target port ID.
10778  * @param[out] error
10779  *   Pointer to the error structure.
10780  *
10781  * @return
10782  *   0 on success, a negative errno value otherwise and rte_errno is set.
10783  */
10784 static int
10785 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
10786                                  const struct rte_flow_action *action,
10787                                  uint32_t *dst_port_id,
10788                                  struct rte_flow_error *error)
10789 {
10790         uint32_t port;
10791         struct mlx5_priv *priv;
10792         const struct rte_flow_action_port_id *conf =
10793                         (const struct rte_flow_action_port_id *)action->conf;
10794
10795         port = conf->original ? dev->data->port_id : conf->id;
10796         priv = mlx5_port_to_eswitch_info(port, false);
10797         if (!priv)
10798                 return rte_flow_error_set(error, -rte_errno,
10799                                           RTE_FLOW_ERROR_TYPE_ACTION,
10800                                           NULL,
10801                                           "No eswitch info was found for port");
10802 #ifdef HAVE_MLX5DV_DR_CREATE_DEST_IB_PORT
10803         /*
10804          * This parameter is transferred to
10805          * mlx5dv_dr_action_create_dest_ib_port().
10806          */
10807         *dst_port_id = priv->dev_port;
10808 #else
10809         /*
10810          * Legacy mode, no LAG configurations is supported.
10811          * This parameter is transferred to
10812          * mlx5dv_dr_action_create_dest_vport().
10813          */
10814         *dst_port_id = priv->vport_id;
10815 #endif
10816         return 0;
10817 }
10818
10819 /**
10820  * Create a counter with aging configuration.
10821  *
10822  * @param[in] dev
10823  *   Pointer to rte_eth_dev structure.
10824  * @param[in] dev_flow
10825  *   Pointer to the mlx5_flow.
10826  * @param[out] count
10827  *   Pointer to the counter action configuration.
10828  * @param[in] age
10829  *   Pointer to the aging action configuration.
10830  *
10831  * @return
10832  *   Index to flow counter on success, 0 otherwise.
10833  */
10834 static uint32_t
10835 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
10836                                 struct mlx5_flow *dev_flow,
10837                                 const struct rte_flow_action_count *count,
10838                                 const struct rte_flow_action_age *age)
10839 {
10840         uint32_t counter;
10841         struct mlx5_age_param *age_param;
10842
10843         if (count && count->shared)
10844                 counter = flow_dv_counter_get_shared(dev, count->id);
10845         else
10846                 counter = flow_dv_counter_alloc(dev, !!age);
10847         if (!counter || age == NULL)
10848                 return counter;
10849         age_param = flow_dv_counter_idx_get_age(dev, counter);
10850         age_param->context = age->context ? age->context :
10851                 (void *)(uintptr_t)(dev_flow->flow_idx);
10852         age_param->timeout = age->timeout;
10853         age_param->port_id = dev->data->port_id;
10854         __atomic_store_n(&age_param->sec_since_last_hit, 0, __ATOMIC_RELAXED);
10855         __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED);
10856         return counter;
10857 }
10858
10859 /**
10860  * Add Tx queue matcher
10861  *
10862  * @param[in] dev
10863  *   Pointer to the dev struct.
10864  * @param[in, out] matcher
10865  *   Flow matcher.
10866  * @param[in, out] key
10867  *   Flow matcher value.
10868  * @param[in] item
10869  *   Flow pattern to translate.
10870  * @param[in] inner
10871  *   Item is inner pattern.
10872  */
10873 static void
10874 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
10875                                 void *matcher, void *key,
10876                                 const struct rte_flow_item *item)
10877 {
10878         const struct mlx5_rte_flow_item_tx_queue *queue_m;
10879         const struct mlx5_rte_flow_item_tx_queue *queue_v;
10880         void *misc_m =
10881                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
10882         void *misc_v =
10883                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
10884         struct mlx5_txq_ctrl *txq;
10885         uint32_t queue;
10886
10887
10888         queue_m = (const void *)item->mask;
10889         if (!queue_m)
10890                 return;
10891         queue_v = (const void *)item->spec;
10892         if (!queue_v)
10893                 return;
10894         txq = mlx5_txq_get(dev, queue_v->queue);
10895         if (!txq)
10896                 return;
10897         queue = txq->obj->sq->id;
10898         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
10899         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
10900                  queue & queue_m->queue);
10901         mlx5_txq_release(dev, queue_v->queue);
10902 }
10903
10904 /**
10905  * Set the hash fields according to the @p flow information.
10906  *
10907  * @param[in] dev_flow
10908  *   Pointer to the mlx5_flow.
10909  * @param[in] rss_desc
10910  *   Pointer to the mlx5_flow_rss_desc.
10911  */
10912 static void
10913 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
10914                        struct mlx5_flow_rss_desc *rss_desc)
10915 {
10916         uint64_t items = dev_flow->handle->layers;
10917         int rss_inner = 0;
10918         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
10919
10920         dev_flow->hash_fields = 0;
10921 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
10922         if (rss_desc->level >= 2) {
10923                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
10924                 rss_inner = 1;
10925         }
10926 #endif
10927         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
10928             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
10929                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
10930                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
10931                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
10932                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
10933                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
10934                         else
10935                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
10936                 }
10937         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
10938                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
10939                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
10940                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
10941                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
10942                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
10943                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
10944                         else
10945                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
10946                 }
10947         }
10948         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
10949             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
10950                 if (rss_types & ETH_RSS_UDP) {
10951                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
10952                                 dev_flow->hash_fields |=
10953                                                 IBV_RX_HASH_SRC_PORT_UDP;
10954                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
10955                                 dev_flow->hash_fields |=
10956                                                 IBV_RX_HASH_DST_PORT_UDP;
10957                         else
10958                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
10959                 }
10960         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
10961                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
10962                 if (rss_types & ETH_RSS_TCP) {
10963                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
10964                                 dev_flow->hash_fields |=
10965                                                 IBV_RX_HASH_SRC_PORT_TCP;
10966                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
10967                                 dev_flow->hash_fields |=
10968                                                 IBV_RX_HASH_DST_PORT_TCP;
10969                         else
10970                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
10971                 }
10972         }
10973 }
10974
10975 /**
10976  * Prepare an Rx Hash queue.
10977  *
10978  * @param dev
10979  *   Pointer to Ethernet device.
10980  * @param[in] dev_flow
10981  *   Pointer to the mlx5_flow.
10982  * @param[in] rss_desc
10983  *   Pointer to the mlx5_flow_rss_desc.
10984  * @param[out] hrxq_idx
10985  *   Hash Rx queue index.
10986  *
10987  * @return
10988  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
10989  */
10990 static struct mlx5_hrxq *
10991 flow_dv_hrxq_prepare(struct rte_eth_dev *dev,
10992                      struct mlx5_flow *dev_flow,
10993                      struct mlx5_flow_rss_desc *rss_desc,
10994                      uint32_t *hrxq_idx)
10995 {
10996         struct mlx5_priv *priv = dev->data->dev_private;
10997         struct mlx5_flow_handle *dh = dev_flow->handle;
10998         struct mlx5_hrxq *hrxq;
10999
11000         MLX5_ASSERT(rss_desc->queue_num);
11001         rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN;
11002         rss_desc->hash_fields = dev_flow->hash_fields;
11003         rss_desc->tunnel = !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL);
11004         rss_desc->shared_rss = 0;
11005         *hrxq_idx = mlx5_hrxq_get(dev, rss_desc);
11006         if (!*hrxq_idx)
11007                 return NULL;
11008         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
11009                               *hrxq_idx);
11010         return hrxq;
11011 }
11012
11013 /**
11014  * Release sample sub action resource.
11015  *
11016  * @param[in, out] dev
11017  *   Pointer to rte_eth_dev structure.
11018  * @param[in] act_res
11019  *   Pointer to sample sub action resource.
11020  */
11021 static void
11022 flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
11023                                    struct mlx5_flow_sub_actions_idx *act_res)
11024 {
11025         if (act_res->rix_hrxq) {
11026                 mlx5_hrxq_release(dev, act_res->rix_hrxq);
11027                 act_res->rix_hrxq = 0;
11028         }
11029         if (act_res->rix_encap_decap) {
11030                 flow_dv_encap_decap_resource_release(dev,
11031                                                      act_res->rix_encap_decap);
11032                 act_res->rix_encap_decap = 0;
11033         }
11034         if (act_res->rix_port_id_action) {
11035                 flow_dv_port_id_action_resource_release(dev,
11036                                                 act_res->rix_port_id_action);
11037                 act_res->rix_port_id_action = 0;
11038         }
11039         if (act_res->rix_tag) {
11040                 flow_dv_tag_release(dev, act_res->rix_tag);
11041                 act_res->rix_tag = 0;
11042         }
11043         if (act_res->rix_jump) {
11044                 flow_dv_jump_tbl_resource_release(dev, act_res->rix_jump);
11045                 act_res->rix_jump = 0;
11046         }
11047 }
11048
11049 int
11050 flow_dv_sample_match_cb(void *tool_ctx __rte_unused,
11051                         struct mlx5_list_entry *entry, void *cb_ctx)
11052 {
11053         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11054         struct rte_eth_dev *dev = ctx->dev;
11055         struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
11056         struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
11057                                                               typeof(*resource),
11058                                                               entry);
11059
11060         if (ctx_resource->ratio == resource->ratio &&
11061             ctx_resource->ft_type == resource->ft_type &&
11062             ctx_resource->ft_id == resource->ft_id &&
11063             ctx_resource->set_action == resource->set_action &&
11064             !memcmp((void *)&ctx_resource->sample_act,
11065                     (void *)&resource->sample_act,
11066                     sizeof(struct mlx5_flow_sub_actions_list))) {
11067                 /*
11068                  * Existing sample action should release the prepared
11069                  * sub-actions reference counter.
11070                  */
11071                 flow_dv_sample_sub_actions_release(dev,
11072                                                    &ctx_resource->sample_idx);
11073                 return 0;
11074         }
11075         return 1;
11076 }
11077
11078 struct mlx5_list_entry *
11079 flow_dv_sample_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
11080 {
11081         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11082         struct rte_eth_dev *dev = ctx->dev;
11083         struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
11084         void **sample_dv_actions = ctx_resource->sub_actions;
11085         struct mlx5_flow_dv_sample_resource *resource;
11086         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
11087         struct mlx5_priv *priv = dev->data->dev_private;
11088         struct mlx5_dev_ctx_shared *sh = priv->sh;
11089         struct mlx5_flow_tbl_resource *tbl;
11090         uint32_t idx = 0;
11091         const uint32_t next_ft_step = 1;
11092         uint32_t next_ft_id = ctx_resource->ft_id + next_ft_step;
11093         uint8_t is_egress = 0;
11094         uint8_t is_transfer = 0;
11095         struct rte_flow_error *error = ctx->error;
11096
11097         /* Register new sample resource. */
11098         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
11099         if (!resource) {
11100                 rte_flow_error_set(error, ENOMEM,
11101                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11102                                           NULL,
11103                                           "cannot allocate resource memory");
11104                 return NULL;
11105         }
11106         *resource = *ctx_resource;
11107         /* Create normal path table level */
11108         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
11109                 is_transfer = 1;
11110         else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
11111                 is_egress = 1;
11112         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
11113                                         is_egress, is_transfer,
11114                                         true, NULL, 0, 0, 0, error);
11115         if (!tbl) {
11116                 rte_flow_error_set(error, ENOMEM,
11117                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11118                                           NULL,
11119                                           "fail to create normal path table "
11120                                           "for sample");
11121                 goto error;
11122         }
11123         resource->normal_path_tbl = tbl;
11124         if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
11125                 if (!sh->default_miss_action) {
11126                         rte_flow_error_set(error, ENOMEM,
11127                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11128                                                 NULL,
11129                                                 "default miss action was not "
11130                                                 "created");
11131                         goto error;
11132                 }
11133                 sample_dv_actions[ctx_resource->sample_act.actions_num++] =
11134                                                 sh->default_miss_action;
11135         }
11136         /* Create a DR sample action */
11137         sampler_attr.sample_ratio = resource->ratio;
11138         sampler_attr.default_next_table = tbl->obj;
11139         sampler_attr.num_sample_actions = ctx_resource->sample_act.actions_num;
11140         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
11141                                                         &sample_dv_actions[0];
11142         sampler_attr.action = resource->set_action;
11143         if (mlx5_os_flow_dr_create_flow_action_sampler
11144                         (&sampler_attr, &resource->verbs_action)) {
11145                 rte_flow_error_set(error, ENOMEM,
11146                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11147                                         NULL, "cannot create sample action");
11148                 goto error;
11149         }
11150         resource->idx = idx;
11151         resource->dev = dev;
11152         return &resource->entry;
11153 error:
11154         if (resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
11155                 flow_dv_sample_sub_actions_release(dev,
11156                                                    &resource->sample_idx);
11157         if (resource->normal_path_tbl)
11158                 flow_dv_tbl_resource_release(MLX5_SH(dev),
11159                                 resource->normal_path_tbl);
11160         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx);
11161         return NULL;
11162
11163 }
11164
11165 struct mlx5_list_entry *
11166 flow_dv_sample_clone_cb(void *tool_ctx __rte_unused,
11167                          struct mlx5_list_entry *entry __rte_unused,
11168                          void *cb_ctx)
11169 {
11170         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11171         struct rte_eth_dev *dev = ctx->dev;
11172         struct mlx5_flow_dv_sample_resource *resource;
11173         struct mlx5_priv *priv = dev->data->dev_private;
11174         struct mlx5_dev_ctx_shared *sh = priv->sh;
11175         uint32_t idx = 0;
11176
11177         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
11178         if (!resource) {
11179                 rte_flow_error_set(ctx->error, ENOMEM,
11180                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11181                                           NULL,
11182                                           "cannot allocate resource memory");
11183                 return NULL;
11184         }
11185         memcpy(resource, entry, sizeof(*resource));
11186         resource->idx = idx;
11187         resource->dev = dev;
11188         return &resource->entry;
11189 }
11190
11191 void
11192 flow_dv_sample_clone_free_cb(void *tool_ctx __rte_unused,
11193                              struct mlx5_list_entry *entry)
11194 {
11195         struct mlx5_flow_dv_sample_resource *resource =
11196                                   container_of(entry, typeof(*resource), entry);
11197         struct rte_eth_dev *dev = resource->dev;
11198         struct mlx5_priv *priv = dev->data->dev_private;
11199
11200         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
11201 }
11202
11203 /**
11204  * Find existing sample resource or create and register a new one.
11205  *
11206  * @param[in, out] dev
11207  *   Pointer to rte_eth_dev structure.
11208  * @param[in] ref
11209  *   Pointer to sample resource reference.
11210  * @parm[in, out] dev_flow
11211  *   Pointer to the dev_flow.
11212  * @param[out] error
11213  *   pointer to error structure.
11214  *
11215  * @return
11216  *   0 on success otherwise -errno and errno is set.
11217  */
11218 static int
11219 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
11220                          struct mlx5_flow_dv_sample_resource *ref,
11221                          struct mlx5_flow *dev_flow,
11222                          struct rte_flow_error *error)
11223 {
11224         struct mlx5_flow_dv_sample_resource *resource;
11225         struct mlx5_list_entry *entry;
11226         struct mlx5_priv *priv = dev->data->dev_private;
11227         struct mlx5_flow_cb_ctx ctx = {
11228                 .dev = dev,
11229                 .error = error,
11230                 .data = ref,
11231         };
11232
11233         entry = mlx5_list_register(priv->sh->sample_action_list, &ctx);
11234         if (!entry)
11235                 return -rte_errno;
11236         resource = container_of(entry, typeof(*resource), entry);
11237         dev_flow->handle->dvh.rix_sample = resource->idx;
11238         dev_flow->dv.sample_res = resource;
11239         return 0;
11240 }
11241
11242 int
11243 flow_dv_dest_array_match_cb(void *tool_ctx __rte_unused,
11244                             struct mlx5_list_entry *entry, void *cb_ctx)
11245 {
11246         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11247         struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
11248         struct rte_eth_dev *dev = ctx->dev;
11249         struct mlx5_flow_dv_dest_array_resource *resource =
11250                                   container_of(entry, typeof(*resource), entry);
11251         uint32_t idx = 0;
11252
11253         if (ctx_resource->num_of_dest == resource->num_of_dest &&
11254             ctx_resource->ft_type == resource->ft_type &&
11255             !memcmp((void *)resource->sample_act,
11256                     (void *)ctx_resource->sample_act,
11257                    (ctx_resource->num_of_dest *
11258                    sizeof(struct mlx5_flow_sub_actions_list)))) {
11259                 /*
11260                  * Existing sample action should release the prepared
11261                  * sub-actions reference counter.
11262                  */
11263                 for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
11264                         flow_dv_sample_sub_actions_release(dev,
11265                                         &ctx_resource->sample_idx[idx]);
11266                 return 0;
11267         }
11268         return 1;
11269 }
11270
11271 struct mlx5_list_entry *
11272 flow_dv_dest_array_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
11273 {
11274         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11275         struct rte_eth_dev *dev = ctx->dev;
11276         struct mlx5_flow_dv_dest_array_resource *resource;
11277         struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
11278         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
11279         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
11280         struct mlx5_priv *priv = dev->data->dev_private;
11281         struct mlx5_dev_ctx_shared *sh = priv->sh;
11282         struct mlx5_flow_sub_actions_list *sample_act;
11283         struct mlx5dv_dr_domain *domain;
11284         uint32_t idx = 0, res_idx = 0;
11285         struct rte_flow_error *error = ctx->error;
11286         uint64_t action_flags;
11287         int ret;
11288
11289         /* Register new destination array resource. */
11290         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
11291                                             &res_idx);
11292         if (!resource) {
11293                 rte_flow_error_set(error, ENOMEM,
11294                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11295                                           NULL,
11296                                           "cannot allocate resource memory");
11297                 return NULL;
11298         }
11299         *resource = *ctx_resource;
11300         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
11301                 domain = sh->fdb_domain;
11302         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
11303                 domain = sh->rx_domain;
11304         else
11305                 domain = sh->tx_domain;
11306         for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
11307                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
11308                                  mlx5_malloc(MLX5_MEM_ZERO,
11309                                  sizeof(struct mlx5dv_dr_action_dest_attr),
11310                                  0, SOCKET_ID_ANY);
11311                 if (!dest_attr[idx]) {
11312                         rte_flow_error_set(error, ENOMEM,
11313                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11314                                            NULL,
11315                                            "cannot allocate resource memory");
11316                         goto error;
11317                 }
11318                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
11319                 sample_act = &ctx_resource->sample_act[idx];
11320                 action_flags = sample_act->action_flags;
11321                 switch (action_flags) {
11322                 case MLX5_FLOW_ACTION_QUEUE:
11323                         dest_attr[idx]->dest = sample_act->dr_queue_action;
11324                         break;
11325                 case (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP):
11326                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
11327                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
11328                         dest_attr[idx]->dest_reformat->reformat =
11329                                         sample_act->dr_encap_action;
11330                         dest_attr[idx]->dest_reformat->dest =
11331                                         sample_act->dr_port_id_action;
11332                         break;
11333                 case MLX5_FLOW_ACTION_PORT_ID:
11334                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
11335                         break;
11336                 case MLX5_FLOW_ACTION_JUMP:
11337                         dest_attr[idx]->dest = sample_act->dr_jump_action;
11338                         break;
11339                 default:
11340                         rte_flow_error_set(error, EINVAL,
11341                                            RTE_FLOW_ERROR_TYPE_ACTION,
11342                                            NULL,
11343                                            "unsupported actions type");
11344                         goto error;
11345                 }
11346         }
11347         /* create a dest array actioin */
11348         ret = mlx5_os_flow_dr_create_flow_action_dest_array
11349                                                 (domain,
11350                                                  resource->num_of_dest,
11351                                                  dest_attr,
11352                                                  &resource->action);
11353         if (ret) {
11354                 rte_flow_error_set(error, ENOMEM,
11355                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11356                                    NULL,
11357                                    "cannot create destination array action");
11358                 goto error;
11359         }
11360         resource->idx = res_idx;
11361         resource->dev = dev;
11362         for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
11363                 mlx5_free(dest_attr[idx]);
11364         return &resource->entry;
11365 error:
11366         for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
11367                 flow_dv_sample_sub_actions_release(dev,
11368                                                    &resource->sample_idx[idx]);
11369                 if (dest_attr[idx])
11370                         mlx5_free(dest_attr[idx]);
11371         }
11372         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx);
11373         return NULL;
11374 }
11375
11376 struct mlx5_list_entry *
11377 flow_dv_dest_array_clone_cb(void *tool_ctx __rte_unused,
11378                             struct mlx5_list_entry *entry __rte_unused,
11379                             void *cb_ctx)
11380 {
11381         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
11382         struct rte_eth_dev *dev = ctx->dev;
11383         struct mlx5_flow_dv_dest_array_resource *resource;
11384         struct mlx5_priv *priv = dev->data->dev_private;
11385         struct mlx5_dev_ctx_shared *sh = priv->sh;
11386         uint32_t res_idx = 0;
11387         struct rte_flow_error *error = ctx->error;
11388
11389         resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
11390                                       &res_idx);
11391         if (!resource) {
11392                 rte_flow_error_set(error, ENOMEM,
11393                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11394                                           NULL,
11395                                           "cannot allocate dest-array memory");
11396                 return NULL;
11397         }
11398         memcpy(resource, entry, sizeof(*resource));
11399         resource->idx = res_idx;
11400         resource->dev = dev;
11401         return &resource->entry;
11402 }
11403
11404 void
11405 flow_dv_dest_array_clone_free_cb(void *tool_ctx __rte_unused,
11406                                  struct mlx5_list_entry *entry)
11407 {
11408         struct mlx5_flow_dv_dest_array_resource *resource =
11409                         container_of(entry, typeof(*resource), entry);
11410         struct rte_eth_dev *dev = resource->dev;
11411         struct mlx5_priv *priv = dev->data->dev_private;
11412
11413         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
11414 }
11415
11416 /**
11417  * Find existing destination array resource or create and register a new one.
11418  *
11419  * @param[in, out] dev
11420  *   Pointer to rte_eth_dev structure.
11421  * @param[in] ref
11422  *   Pointer to destination array resource reference.
11423  * @parm[in, out] dev_flow
11424  *   Pointer to the dev_flow.
11425  * @param[out] error
11426  *   pointer to error structure.
11427  *
11428  * @return
11429  *   0 on success otherwise -errno and errno is set.
11430  */
11431 static int
11432 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
11433                          struct mlx5_flow_dv_dest_array_resource *ref,
11434                          struct mlx5_flow *dev_flow,
11435                          struct rte_flow_error *error)
11436 {
11437         struct mlx5_flow_dv_dest_array_resource *resource;
11438         struct mlx5_priv *priv = dev->data->dev_private;
11439         struct mlx5_list_entry *entry;
11440         struct mlx5_flow_cb_ctx ctx = {
11441                 .dev = dev,
11442                 .error = error,
11443                 .data = ref,
11444         };
11445
11446         entry = mlx5_list_register(priv->sh->dest_array_list, &ctx);
11447         if (!entry)
11448                 return -rte_errno;
11449         resource = container_of(entry, typeof(*resource), entry);
11450         dev_flow->handle->dvh.rix_dest_array = resource->idx;
11451         dev_flow->dv.dest_array_res = resource;
11452         return 0;
11453 }
11454
11455 /**
11456  * Convert Sample action to DV specification.
11457  *
11458  * @param[in] dev
11459  *   Pointer to rte_eth_dev structure.
11460  * @param[in] action
11461  *   Pointer to sample action structure.
11462  * @param[in, out] dev_flow
11463  *   Pointer to the mlx5_flow.
11464  * @param[in] attr
11465  *   Pointer to the flow attributes.
11466  * @param[in, out] num_of_dest
11467  *   Pointer to the num of destination.
11468  * @param[in, out] sample_actions
11469  *   Pointer to sample actions list.
11470  * @param[in, out] res
11471  *   Pointer to sample resource.
11472  * @param[out] error
11473  *   Pointer to the error structure.
11474  *
11475  * @return
11476  *   0 on success, a negative errno value otherwise and rte_errno is set.
11477  */
11478 static int
11479 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
11480                                 const struct rte_flow_action_sample *action,
11481                                 struct mlx5_flow *dev_flow,
11482                                 const struct rte_flow_attr *attr,
11483                                 uint32_t *num_of_dest,
11484                                 void **sample_actions,
11485                                 struct mlx5_flow_dv_sample_resource *res,
11486                                 struct rte_flow_error *error)
11487 {
11488         struct mlx5_priv *priv = dev->data->dev_private;
11489         const struct rte_flow_action *sub_actions;
11490         struct mlx5_flow_sub_actions_list *sample_act;
11491         struct mlx5_flow_sub_actions_idx *sample_idx;
11492         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
11493         struct rte_flow *flow = dev_flow->flow;
11494         struct mlx5_flow_rss_desc *rss_desc;
11495         uint64_t action_flags = 0;
11496
11497         MLX5_ASSERT(wks);
11498         rss_desc = &wks->rss_desc;
11499         sample_act = &res->sample_act;
11500         sample_idx = &res->sample_idx;
11501         res->ratio = action->ratio;
11502         sub_actions = action->actions;
11503         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
11504                 int type = sub_actions->type;
11505                 uint32_t pre_rix = 0;
11506                 void *pre_r;
11507                 switch (type) {
11508                 case RTE_FLOW_ACTION_TYPE_QUEUE:
11509                 {
11510                         const struct rte_flow_action_queue *queue;
11511                         struct mlx5_hrxq *hrxq;
11512                         uint32_t hrxq_idx;
11513
11514                         queue = sub_actions->conf;
11515                         rss_desc->queue_num = 1;
11516                         rss_desc->queue[0] = queue->index;
11517                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11518                                                     rss_desc, &hrxq_idx);
11519                         if (!hrxq)
11520                                 return rte_flow_error_set
11521                                         (error, rte_errno,
11522                                          RTE_FLOW_ERROR_TYPE_ACTION,
11523                                          NULL,
11524                                          "cannot create fate queue");
11525                         sample_act->dr_queue_action = hrxq->action;
11526                         sample_idx->rix_hrxq = hrxq_idx;
11527                         sample_actions[sample_act->actions_num++] =
11528                                                 hrxq->action;
11529                         (*num_of_dest)++;
11530                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
11531                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11532                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11533                         dev_flow->handle->fate_action =
11534                                         MLX5_FLOW_FATE_QUEUE;
11535                         break;
11536                 }
11537                 case RTE_FLOW_ACTION_TYPE_RSS:
11538                 {
11539                         struct mlx5_hrxq *hrxq;
11540                         uint32_t hrxq_idx;
11541                         const struct rte_flow_action_rss *rss;
11542                         const uint8_t *rss_key;
11543
11544                         rss = sub_actions->conf;
11545                         memcpy(rss_desc->queue, rss->queue,
11546                                rss->queue_num * sizeof(uint16_t));
11547                         rss_desc->queue_num = rss->queue_num;
11548                         /* NULL RSS key indicates default RSS key. */
11549                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
11550                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
11551                         /*
11552                          * rss->level and rss.types should be set in advance
11553                          * when expanding items for RSS.
11554                          */
11555                         flow_dv_hashfields_set(dev_flow, rss_desc);
11556                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11557                                                     rss_desc, &hrxq_idx);
11558                         if (!hrxq)
11559                                 return rte_flow_error_set
11560                                         (error, rte_errno,
11561                                          RTE_FLOW_ERROR_TYPE_ACTION,
11562                                          NULL,
11563                                          "cannot create fate queue");
11564                         sample_act->dr_queue_action = hrxq->action;
11565                         sample_idx->rix_hrxq = hrxq_idx;
11566                         sample_actions[sample_act->actions_num++] =
11567                                                 hrxq->action;
11568                         (*num_of_dest)++;
11569                         action_flags |= MLX5_FLOW_ACTION_RSS;
11570                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11571                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11572                         dev_flow->handle->fate_action =
11573                                         MLX5_FLOW_FATE_QUEUE;
11574                         break;
11575                 }
11576                 case RTE_FLOW_ACTION_TYPE_MARK:
11577                 {
11578                         uint32_t tag_be = mlx5_flow_mark_set
11579                                 (((const struct rte_flow_action_mark *)
11580                                 (sub_actions->conf))->id);
11581
11582                         dev_flow->handle->mark = 1;
11583                         pre_rix = dev_flow->handle->dvh.rix_tag;
11584                         /* Save the mark resource before sample */
11585                         pre_r = dev_flow->dv.tag_resource;
11586                         if (flow_dv_tag_resource_register(dev, tag_be,
11587                                                   dev_flow, error))
11588                                 return -rte_errno;
11589                         MLX5_ASSERT(dev_flow->dv.tag_resource);
11590                         sample_act->dr_tag_action =
11591                                 dev_flow->dv.tag_resource->action;
11592                         sample_idx->rix_tag =
11593                                 dev_flow->handle->dvh.rix_tag;
11594                         sample_actions[sample_act->actions_num++] =
11595                                                 sample_act->dr_tag_action;
11596                         /* Recover the mark resource after sample */
11597                         dev_flow->dv.tag_resource = pre_r;
11598                         dev_flow->handle->dvh.rix_tag = pre_rix;
11599                         action_flags |= MLX5_FLOW_ACTION_MARK;
11600                         break;
11601                 }
11602                 case RTE_FLOW_ACTION_TYPE_COUNT:
11603                 {
11604                         if (!flow->counter) {
11605                                 flow->counter =
11606                                         flow_dv_translate_create_counter(dev,
11607                                                 dev_flow, sub_actions->conf,
11608                                                 0);
11609                                 if (!flow->counter)
11610                                         return rte_flow_error_set
11611                                                 (error, rte_errno,
11612                                                 RTE_FLOW_ERROR_TYPE_ACTION,
11613                                                 NULL,
11614                                                 "cannot create counter"
11615                                                 " object.");
11616                         }
11617                         sample_act->dr_cnt_action =
11618                                   (flow_dv_counter_get_by_idx(dev,
11619                                   flow->counter, NULL))->action;
11620                         sample_actions[sample_act->actions_num++] =
11621                                                 sample_act->dr_cnt_action;
11622                         action_flags |= MLX5_FLOW_ACTION_COUNT;
11623                         break;
11624                 }
11625                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
11626                 {
11627                         struct mlx5_flow_dv_port_id_action_resource
11628                                         port_id_resource;
11629                         uint32_t port_id = 0;
11630
11631                         memset(&port_id_resource, 0, sizeof(port_id_resource));
11632                         /* Save the port id resource before sample */
11633                         pre_rix = dev_flow->handle->rix_port_id_action;
11634                         pre_r = dev_flow->dv.port_id_action;
11635                         if (flow_dv_translate_action_port_id(dev, sub_actions,
11636                                                              &port_id, error))
11637                                 return -rte_errno;
11638                         port_id_resource.port_id = port_id;
11639                         if (flow_dv_port_id_action_resource_register
11640                             (dev, &port_id_resource, dev_flow, error))
11641                                 return -rte_errno;
11642                         sample_act->dr_port_id_action =
11643                                 dev_flow->dv.port_id_action->action;
11644                         sample_idx->rix_port_id_action =
11645                                 dev_flow->handle->rix_port_id_action;
11646                         sample_actions[sample_act->actions_num++] =
11647                                                 sample_act->dr_port_id_action;
11648                         /* Recover the port id resource after sample */
11649                         dev_flow->dv.port_id_action = pre_r;
11650                         dev_flow->handle->rix_port_id_action = pre_rix;
11651                         (*num_of_dest)++;
11652                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
11653                         break;
11654                 }
11655                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
11656                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
11657                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
11658                         /* Save the encap resource before sample */
11659                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
11660                         pre_r = dev_flow->dv.encap_decap;
11661                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
11662                                                            dev_flow,
11663                                                            attr->transfer,
11664                                                            error))
11665                                 return -rte_errno;
11666                         sample_act->dr_encap_action =
11667                                 dev_flow->dv.encap_decap->action;
11668                         sample_idx->rix_encap_decap =
11669                                 dev_flow->handle->dvh.rix_encap_decap;
11670                         sample_actions[sample_act->actions_num++] =
11671                                                 sample_act->dr_encap_action;
11672                         /* Recover the encap resource after sample */
11673                         dev_flow->dv.encap_decap = pre_r;
11674                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
11675                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
11676                         break;
11677                 default:
11678                         return rte_flow_error_set(error, EINVAL,
11679                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11680                                 NULL,
11681                                 "Not support for sampler action");
11682                 }
11683         }
11684         sample_act->action_flags = action_flags;
11685         res->ft_id = dev_flow->dv.group;
11686         if (attr->transfer) {
11687                 union {
11688                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
11689                         uint64_t set_action;
11690                 } action_ctx = { .set_action = 0 };
11691
11692                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
11693                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
11694                          MLX5_MODIFICATION_TYPE_SET);
11695                 MLX5_SET(set_action_in, action_ctx.action_in, field,
11696                          MLX5_MODI_META_REG_C_0);
11697                 MLX5_SET(set_action_in, action_ctx.action_in, data,
11698                          priv->vport_meta_tag);
11699                 res->set_action = action_ctx.set_action;
11700         } else if (attr->ingress) {
11701                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
11702         } else {
11703                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX;
11704         }
11705         return 0;
11706 }
11707
11708 /**
11709  * Convert Sample action to DV specification.
11710  *
11711  * @param[in] dev
11712  *   Pointer to rte_eth_dev structure.
11713  * @param[in, out] dev_flow
11714  *   Pointer to the mlx5_flow.
11715  * @param[in] num_of_dest
11716  *   The num of destination.
11717  * @param[in, out] res
11718  *   Pointer to sample resource.
11719  * @param[in, out] mdest_res
11720  *   Pointer to destination array resource.
11721  * @param[in] sample_actions
11722  *   Pointer to sample path actions list.
11723  * @param[in] action_flags
11724  *   Holds the actions detected until now.
11725  * @param[out] error
11726  *   Pointer to the error structure.
11727  *
11728  * @return
11729  *   0 on success, a negative errno value otherwise and rte_errno is set.
11730  */
11731 static int
11732 flow_dv_create_action_sample(struct rte_eth_dev *dev,
11733                              struct mlx5_flow *dev_flow,
11734                              uint32_t num_of_dest,
11735                              struct mlx5_flow_dv_sample_resource *res,
11736                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
11737                              void **sample_actions,
11738                              uint64_t action_flags,
11739                              struct rte_flow_error *error)
11740 {
11741         /* update normal path action resource into last index of array */
11742         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
11743         struct mlx5_flow_sub_actions_list *sample_act =
11744                                         &mdest_res->sample_act[dest_index];
11745         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
11746         struct mlx5_flow_rss_desc *rss_desc;
11747         uint32_t normal_idx = 0;
11748         struct mlx5_hrxq *hrxq;
11749         uint32_t hrxq_idx;
11750
11751         MLX5_ASSERT(wks);
11752         rss_desc = &wks->rss_desc;
11753         if (num_of_dest > 1) {
11754                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
11755                         /* Handle QP action for mirroring */
11756                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow,
11757                                                     rss_desc, &hrxq_idx);
11758                         if (!hrxq)
11759                                 return rte_flow_error_set
11760                                      (error, rte_errno,
11761                                       RTE_FLOW_ERROR_TYPE_ACTION,
11762                                       NULL,
11763                                       "cannot create rx queue");
11764                         normal_idx++;
11765                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
11766                         sample_act->dr_queue_action = hrxq->action;
11767                         if (action_flags & MLX5_FLOW_ACTION_MARK)
11768                                 dev_flow->handle->rix_hrxq = hrxq_idx;
11769                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
11770                 }
11771                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
11772                         normal_idx++;
11773                         mdest_res->sample_idx[dest_index].rix_encap_decap =
11774                                 dev_flow->handle->dvh.rix_encap_decap;
11775                         sample_act->dr_encap_action =
11776                                 dev_flow->dv.encap_decap->action;
11777                         dev_flow->handle->dvh.rix_encap_decap = 0;
11778                 }
11779                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
11780                         normal_idx++;
11781                         mdest_res->sample_idx[dest_index].rix_port_id_action =
11782                                 dev_flow->handle->rix_port_id_action;
11783                         sample_act->dr_port_id_action =
11784                                 dev_flow->dv.port_id_action->action;
11785                         dev_flow->handle->rix_port_id_action = 0;
11786                 }
11787                 if (sample_act->action_flags & MLX5_FLOW_ACTION_JUMP) {
11788                         normal_idx++;
11789                         mdest_res->sample_idx[dest_index].rix_jump =
11790                                 dev_flow->handle->rix_jump;
11791                         sample_act->dr_jump_action =
11792                                 dev_flow->dv.jump->action;
11793                         dev_flow->handle->rix_jump = 0;
11794                 }
11795                 sample_act->actions_num = normal_idx;
11796                 /* update sample action resource into first index of array */
11797                 mdest_res->ft_type = res->ft_type;
11798                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
11799                                 sizeof(struct mlx5_flow_sub_actions_idx));
11800                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
11801                                 sizeof(struct mlx5_flow_sub_actions_list));
11802                 mdest_res->num_of_dest = num_of_dest;
11803                 if (flow_dv_dest_array_resource_register(dev, mdest_res,
11804                                                          dev_flow, error))
11805                         return rte_flow_error_set(error, EINVAL,
11806                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11807                                                   NULL, "can't create sample "
11808                                                   "action");
11809         } else {
11810                 res->sub_actions = sample_actions;
11811                 if (flow_dv_sample_resource_register(dev, res, dev_flow, error))
11812                         return rte_flow_error_set(error, EINVAL,
11813                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11814                                                   NULL,
11815                                                   "can't create sample action");
11816         }
11817         return 0;
11818 }
11819
11820 /**
11821  * Remove an ASO age action from age actions list.
11822  *
11823  * @param[in] dev
11824  *   Pointer to the Ethernet device structure.
11825  * @param[in] age
11826  *   Pointer to the aso age action handler.
11827  */
11828 static void
11829 flow_dv_aso_age_remove_from_age(struct rte_eth_dev *dev,
11830                                 struct mlx5_aso_age_action *age)
11831 {
11832         struct mlx5_age_info *age_info;
11833         struct mlx5_age_param *age_param = &age->age_params;
11834         struct mlx5_priv *priv = dev->data->dev_private;
11835         uint16_t expected = AGE_CANDIDATE;
11836
11837         age_info = GET_PORT_AGE_INFO(priv);
11838         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
11839                                          AGE_FREE, false, __ATOMIC_RELAXED,
11840                                          __ATOMIC_RELAXED)) {
11841                 /**
11842                  * We need the lock even it is age timeout,
11843                  * since age action may still in process.
11844                  */
11845                 rte_spinlock_lock(&age_info->aged_sl);
11846                 LIST_REMOVE(age, next);
11847                 rte_spinlock_unlock(&age_info->aged_sl);
11848                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
11849         }
11850 }
11851
11852 /**
11853  * Release an ASO age action.
11854  *
11855  * @param[in] dev
11856  *   Pointer to the Ethernet device structure.
11857  * @param[in] age_idx
11858  *   Index of ASO age action to release.
11859  * @param[in] flow
11860  *   True if the release operation is during flow destroy operation.
11861  *   False if the release operation is during action destroy operation.
11862  *
11863  * @return
11864  *   0 when age action was removed, otherwise the number of references.
11865  */
11866 static int
11867 flow_dv_aso_age_release(struct rte_eth_dev *dev, uint32_t age_idx)
11868 {
11869         struct mlx5_priv *priv = dev->data->dev_private;
11870         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11871         struct mlx5_aso_age_action *age = flow_aso_age_get_by_idx(dev, age_idx);
11872         uint32_t ret = __atomic_sub_fetch(&age->refcnt, 1, __ATOMIC_RELAXED);
11873
11874         if (!ret) {
11875                 flow_dv_aso_age_remove_from_age(dev, age);
11876                 rte_spinlock_lock(&mng->free_sl);
11877                 LIST_INSERT_HEAD(&mng->free, age, next);
11878                 rte_spinlock_unlock(&mng->free_sl);
11879         }
11880         return ret;
11881 }
11882
11883 /**
11884  * Resize the ASO age pools array by MLX5_CNT_CONTAINER_RESIZE pools.
11885  *
11886  * @param[in] dev
11887  *   Pointer to the Ethernet device structure.
11888  *
11889  * @return
11890  *   0 on success, otherwise negative errno value and rte_errno is set.
11891  */
11892 static int
11893 flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev)
11894 {
11895         struct mlx5_priv *priv = dev->data->dev_private;
11896         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11897         void *old_pools = mng->pools;
11898         uint32_t resize = mng->n + MLX5_CNT_CONTAINER_RESIZE;
11899         uint32_t mem_size = sizeof(struct mlx5_aso_age_pool *) * resize;
11900         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
11901
11902         if (!pools) {
11903                 rte_errno = ENOMEM;
11904                 return -ENOMEM;
11905         }
11906         if (old_pools) {
11907                 memcpy(pools, old_pools,
11908                        mng->n * sizeof(struct mlx5_flow_counter_pool *));
11909                 mlx5_free(old_pools);
11910         } else {
11911                 /* First ASO flow hit allocation - starting ASO data-path. */
11912                 int ret = mlx5_aso_flow_hit_queue_poll_start(priv->sh);
11913
11914                 if (ret) {
11915                         mlx5_free(pools);
11916                         return ret;
11917                 }
11918         }
11919         mng->n = resize;
11920         mng->pools = pools;
11921         return 0;
11922 }
11923
11924 /**
11925  * Create and initialize a new ASO aging pool.
11926  *
11927  * @param[in] dev
11928  *   Pointer to the Ethernet device structure.
11929  * @param[out] age_free
11930  *   Where to put the pointer of a new age action.
11931  *
11932  * @return
11933  *   The age actions pool pointer and @p age_free is set on success,
11934  *   NULL otherwise and rte_errno is set.
11935  */
11936 static struct mlx5_aso_age_pool *
11937 flow_dv_age_pool_create(struct rte_eth_dev *dev,
11938                         struct mlx5_aso_age_action **age_free)
11939 {
11940         struct mlx5_priv *priv = dev->data->dev_private;
11941         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
11942         struct mlx5_aso_age_pool *pool = NULL;
11943         struct mlx5_devx_obj *obj = NULL;
11944         uint32_t i;
11945
11946         obj = mlx5_devx_cmd_create_flow_hit_aso_obj(priv->sh->ctx,
11947                                                     priv->sh->pdn);
11948         if (!obj) {
11949                 rte_errno = ENODATA;
11950                 DRV_LOG(ERR, "Failed to create flow_hit_aso_obj using DevX.");
11951                 return NULL;
11952         }
11953         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
11954         if (!pool) {
11955                 claim_zero(mlx5_devx_cmd_destroy(obj));
11956                 rte_errno = ENOMEM;
11957                 return NULL;
11958         }
11959         pool->flow_hit_aso_obj = obj;
11960         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
11961         rte_spinlock_lock(&mng->resize_sl);
11962         pool->index = mng->next;
11963         /* Resize pools array if there is no room for the new pool in it. */
11964         if (pool->index == mng->n && flow_dv_aso_age_pools_resize(dev)) {
11965                 claim_zero(mlx5_devx_cmd_destroy(obj));
11966                 mlx5_free(pool);
11967                 rte_spinlock_unlock(&mng->resize_sl);
11968                 return NULL;
11969         }
11970         mng->pools[pool->index] = pool;
11971         mng->next++;
11972         rte_spinlock_unlock(&mng->resize_sl);
11973         /* Assign the first action in the new pool, the rest go to free list. */
11974         *age_free = &pool->actions[0];
11975         for (i = 1; i < MLX5_ASO_AGE_ACTIONS_PER_POOL; i++) {
11976                 pool->actions[i].offset = i;
11977                 LIST_INSERT_HEAD(&mng->free, &pool->actions[i], next);
11978         }
11979         return pool;
11980 }
11981
11982 /**
11983  * Allocate a ASO aging bit.
11984  *
11985  * @param[in] dev
11986  *   Pointer to the Ethernet device structure.
11987  * @param[out] error
11988  *   Pointer to the error structure.
11989  *
11990  * @return
11991  *   Index to ASO age action on success, 0 otherwise and rte_errno is set.
11992  */
11993 static uint32_t
11994 flow_dv_aso_age_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
11995 {
11996         struct mlx5_priv *priv = dev->data->dev_private;
11997         const struct mlx5_aso_age_pool *pool;
11998         struct mlx5_aso_age_action *age_free = NULL;
11999         struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
12000
12001         MLX5_ASSERT(mng);
12002         /* Try to get the next free age action bit. */
12003         rte_spinlock_lock(&mng->free_sl);
12004         age_free = LIST_FIRST(&mng->free);
12005         if (age_free) {
12006                 LIST_REMOVE(age_free, next);
12007         } else if (!flow_dv_age_pool_create(dev, &age_free)) {
12008                 rte_spinlock_unlock(&mng->free_sl);
12009                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
12010                                    NULL, "failed to create ASO age pool");
12011                 return 0; /* 0 is an error. */
12012         }
12013         rte_spinlock_unlock(&mng->free_sl);
12014         pool = container_of
12015           ((const struct mlx5_aso_age_action (*)[MLX5_ASO_AGE_ACTIONS_PER_POOL])
12016                   (age_free - age_free->offset), const struct mlx5_aso_age_pool,
12017                                                                        actions);
12018         if (!age_free->dr_action) {
12019                 int reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_FLOW_HIT, 0,
12020                                                  error);
12021
12022                 if (reg_c < 0) {
12023                         rte_flow_error_set(error, rte_errno,
12024                                            RTE_FLOW_ERROR_TYPE_ACTION,
12025                                            NULL, "failed to get reg_c "
12026                                            "for ASO flow hit");
12027                         return 0; /* 0 is an error. */
12028                 }
12029 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
12030                 age_free->dr_action = mlx5_glue->dv_create_flow_action_aso
12031                                 (priv->sh->rx_domain,
12032                                  pool->flow_hit_aso_obj->obj, age_free->offset,
12033                                  MLX5DV_DR_ACTION_FLAGS_ASO_FIRST_HIT_SET,
12034                                  (reg_c - REG_C_0));
12035 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
12036                 if (!age_free->dr_action) {
12037                         rte_errno = errno;
12038                         rte_spinlock_lock(&mng->free_sl);
12039                         LIST_INSERT_HEAD(&mng->free, age_free, next);
12040                         rte_spinlock_unlock(&mng->free_sl);
12041                         rte_flow_error_set(error, rte_errno,
12042                                            RTE_FLOW_ERROR_TYPE_ACTION,
12043                                            NULL, "failed to create ASO "
12044                                            "flow hit action");
12045                         return 0; /* 0 is an error. */
12046                 }
12047         }
12048         __atomic_store_n(&age_free->refcnt, 1, __ATOMIC_RELAXED);
12049         return pool->index | ((age_free->offset + 1) << 16);
12050 }
12051
12052 /**
12053  * Initialize flow ASO age parameters.
12054  *
12055  * @param[in] dev
12056  *   Pointer to rte_eth_dev structure.
12057  * @param[in] age_idx
12058  *   Index of ASO age action.
12059  * @param[in] context
12060  *   Pointer to flow counter age context.
12061  * @param[in] timeout
12062  *   Aging timeout in seconds.
12063  *
12064  */
12065 static void
12066 flow_dv_aso_age_params_init(struct rte_eth_dev *dev,
12067                             uint32_t age_idx,
12068                             void *context,
12069                             uint32_t timeout)
12070 {
12071         struct mlx5_aso_age_action *aso_age;
12072
12073         aso_age = flow_aso_age_get_by_idx(dev, age_idx);
12074         MLX5_ASSERT(aso_age);
12075         aso_age->age_params.context = context;
12076         aso_age->age_params.timeout = timeout;
12077         aso_age->age_params.port_id = dev->data->port_id;
12078         __atomic_store_n(&aso_age->age_params.sec_since_last_hit, 0,
12079                          __ATOMIC_RELAXED);
12080         __atomic_store_n(&aso_age->age_params.state, AGE_CANDIDATE,
12081                          __ATOMIC_RELAXED);
12082 }
12083
12084 static void
12085 flow_dv_translate_integrity_l4(const struct rte_flow_item_integrity *mask,
12086                                const struct rte_flow_item_integrity *value,
12087                                void *headers_m, void *headers_v)
12088 {
12089         if (mask->l4_ok) {
12090                 /* application l4_ok filter aggregates all hardware l4 filters
12091                  * therefore hw l4_checksum_ok must be implicitly added here.
12092                  */
12093                 struct rte_flow_item_integrity local_item;
12094
12095                 local_item.l4_csum_ok = 1;
12096                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok,
12097                          local_item.l4_csum_ok);
12098                 if (value->l4_ok) {
12099                         /* application l4_ok = 1 matches sets both hw flags
12100                          * l4_ok and l4_checksum_ok flags to 1.
12101                          */
12102                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
12103                                  l4_checksum_ok, local_item.l4_csum_ok);
12104                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_ok,
12105                                  mask->l4_ok);
12106                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_ok,
12107                                  value->l4_ok);
12108                 } else {
12109                         /* application l4_ok = 0 matches on hw flag
12110                          * l4_checksum_ok = 0 only.
12111                          */
12112                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
12113                                  l4_checksum_ok, 0);
12114                 }
12115         } else if (mask->l4_csum_ok) {
12116                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, l4_checksum_ok,
12117                          mask->l4_csum_ok);
12118                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, l4_checksum_ok,
12119                          value->l4_csum_ok);
12120         }
12121 }
12122
12123 static void
12124 flow_dv_translate_integrity_l3(const struct rte_flow_item_integrity *mask,
12125                                const struct rte_flow_item_integrity *value,
12126                                void *headers_m, void *headers_v,
12127                                bool is_ipv4)
12128 {
12129         if (mask->l3_ok) {
12130                 /* application l3_ok filter aggregates all hardware l3 filters
12131                  * therefore hw ipv4_checksum_ok must be implicitly added here.
12132                  */
12133                 struct rte_flow_item_integrity local_item;
12134
12135                 local_item.ipv4_csum_ok = !!is_ipv4;
12136                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok,
12137                          local_item.ipv4_csum_ok);
12138                 if (value->l3_ok) {
12139                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
12140                                  ipv4_checksum_ok, local_item.ipv4_csum_ok);
12141                         MLX5_SET(fte_match_set_lyr_2_4, headers_m, l3_ok,
12142                                  mask->l3_ok);
12143                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, l3_ok,
12144                                  value->l3_ok);
12145                 } else {
12146                         MLX5_SET(fte_match_set_lyr_2_4, headers_v,
12147                                  ipv4_checksum_ok, 0);
12148                 }
12149         } else if (mask->ipv4_csum_ok) {
12150                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ipv4_checksum_ok,
12151                          mask->ipv4_csum_ok);
12152                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ipv4_checksum_ok,
12153                          value->ipv4_csum_ok);
12154         }
12155 }
12156
12157 static void
12158 flow_dv_translate_item_integrity(void *matcher, void *key,
12159                                  const struct rte_flow_item *head_item,
12160                                  const struct rte_flow_item *integrity_item)
12161 {
12162         const struct rte_flow_item_integrity *mask = integrity_item->mask;
12163         const struct rte_flow_item_integrity *value = integrity_item->spec;
12164         const struct rte_flow_item *tunnel_item, *end_item, *item;
12165         void *headers_m;
12166         void *headers_v;
12167         uint32_t l3_protocol;
12168
12169         if (!value)
12170                 return;
12171         if (!mask)
12172                 mask = &rte_flow_item_integrity_mask;
12173         if (value->level > 1) {
12174                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
12175                                          inner_headers);
12176                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
12177         } else {
12178                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
12179                                          outer_headers);
12180                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
12181         }
12182         tunnel_item = mlx5_flow_find_tunnel_item(head_item);
12183         if (value->level > 1) {
12184                 /* tunnel item was verified during the item validation */
12185                 item = tunnel_item;
12186                 end_item = mlx5_find_end_item(tunnel_item);
12187         } else {
12188                 item = head_item;
12189                 end_item = tunnel_item ? tunnel_item :
12190                            mlx5_find_end_item(integrity_item);
12191         }
12192         l3_protocol = mask->l3_ok ?
12193                       mlx5_flow_locate_proto_l3(&item, end_item) : 0;
12194         flow_dv_translate_integrity_l3(mask, value, headers_m, headers_v,
12195                                        l3_protocol == RTE_ETHER_TYPE_IPV4);
12196         flow_dv_translate_integrity_l4(mask, value, headers_m, headers_v);
12197 }
12198
12199 /**
12200  * Prepares DV flow counter with aging configuration.
12201  * Gets it by index when exists, creates a new one when doesn't.
12202  *
12203  * @param[in] dev
12204  *   Pointer to rte_eth_dev structure.
12205  * @param[in] dev_flow
12206  *   Pointer to the mlx5_flow.
12207  * @param[in, out] flow
12208  *   Pointer to the sub flow.
12209  * @param[in] count
12210  *   Pointer to the counter action configuration.
12211  * @param[in] age
12212  *   Pointer to the aging action configuration.
12213  * @param[out] error
12214  *   Pointer to the error structure.
12215  *
12216  * @return
12217  *   Pointer to the counter, NULL otherwise.
12218  */
12219 static struct mlx5_flow_counter *
12220 flow_dv_prepare_counter(struct rte_eth_dev *dev,
12221                         struct mlx5_flow *dev_flow,
12222                         struct rte_flow *flow,
12223                         const struct rte_flow_action_count *count,
12224                         const struct rte_flow_action_age *age,
12225                         struct rte_flow_error *error)
12226 {
12227         if (!flow->counter) {
12228                 flow->counter = flow_dv_translate_create_counter(dev, dev_flow,
12229                                                                  count, age);
12230                 if (!flow->counter) {
12231                         rte_flow_error_set(error, rte_errno,
12232                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12233                                            "cannot create counter object.");
12234                         return NULL;
12235                 }
12236         }
12237         return flow_dv_counter_get_by_idx(dev, flow->counter, NULL);
12238 }
12239
12240 /*
12241  * Release an ASO CT action by its own device.
12242  *
12243  * @param[in] dev
12244  *   Pointer to the Ethernet device structure.
12245  * @param[in] idx
12246  *   Index of ASO CT action to release.
12247  *
12248  * @return
12249  *   0 when CT action was removed, otherwise the number of references.
12250  */
12251 static inline int
12252 flow_dv_aso_ct_dev_release(struct rte_eth_dev *dev, uint32_t idx)
12253 {
12254         struct mlx5_priv *priv = dev->data->dev_private;
12255         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12256         uint32_t ret;
12257         struct mlx5_aso_ct_action *ct = flow_aso_ct_get_by_dev_idx(dev, idx);
12258         enum mlx5_aso_ct_state state =
12259                         __atomic_load_n(&ct->state, __ATOMIC_RELAXED);
12260
12261         /* Cannot release when CT is in the ASO SQ. */
12262         if (state == ASO_CONNTRACK_WAIT || state == ASO_CONNTRACK_QUERY)
12263                 return -1;
12264         ret = __atomic_sub_fetch(&ct->refcnt, 1, __ATOMIC_RELAXED);
12265         if (!ret) {
12266                 if (ct->dr_action_orig) {
12267 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12268                         claim_zero(mlx5_glue->destroy_flow_action
12269                                         (ct->dr_action_orig));
12270 #endif
12271                         ct->dr_action_orig = NULL;
12272                 }
12273                 if (ct->dr_action_rply) {
12274 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12275                         claim_zero(mlx5_glue->destroy_flow_action
12276                                         (ct->dr_action_rply));
12277 #endif
12278                         ct->dr_action_rply = NULL;
12279                 }
12280                 /* Clear the state to free, no need in 1st allocation. */
12281                 MLX5_ASO_CT_UPDATE_STATE(ct, ASO_CONNTRACK_FREE);
12282                 rte_spinlock_lock(&mng->ct_sl);
12283                 LIST_INSERT_HEAD(&mng->free_cts, ct, next);
12284                 rte_spinlock_unlock(&mng->ct_sl);
12285         }
12286         return (int)ret;
12287 }
12288
12289 static inline int
12290 flow_dv_aso_ct_release(struct rte_eth_dev *dev, uint32_t own_idx)
12291 {
12292         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(own_idx);
12293         uint32_t idx = MLX5_INDIRECT_ACT_CT_GET_IDX(own_idx);
12294         struct rte_eth_dev *owndev = &rte_eth_devices[owner];
12295         RTE_SET_USED(dev);
12296
12297         MLX5_ASSERT(owner < RTE_MAX_ETHPORTS);
12298         if (dev->data->dev_started != 1)
12299                 return -1;
12300         return flow_dv_aso_ct_dev_release(owndev, idx);
12301 }
12302
12303 /*
12304  * Resize the ASO CT pools array by 64 pools.
12305  *
12306  * @param[in] dev
12307  *   Pointer to the Ethernet device structure.
12308  *
12309  * @return
12310  *   0 on success, otherwise negative errno value and rte_errno is set.
12311  */
12312 static int
12313 flow_dv_aso_ct_pools_resize(struct rte_eth_dev *dev)
12314 {
12315         struct mlx5_priv *priv = dev->data->dev_private;
12316         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12317         void *old_pools = mng->pools;
12318         /* Magic number now, need a macro. */
12319         uint32_t resize = mng->n + 64;
12320         uint32_t mem_size = sizeof(struct mlx5_aso_ct_pool *) * resize;
12321         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
12322
12323         if (!pools) {
12324                 rte_errno = ENOMEM;
12325                 return -rte_errno;
12326         }
12327         rte_rwlock_write_lock(&mng->resize_rwl);
12328         /* ASO SQ/QP was already initialized in the startup. */
12329         if (old_pools) {
12330                 /* Realloc could be an alternative choice. */
12331                 rte_memcpy(pools, old_pools,
12332                            mng->n * sizeof(struct mlx5_aso_ct_pool *));
12333                 mlx5_free(old_pools);
12334         }
12335         mng->n = resize;
12336         mng->pools = pools;
12337         rte_rwlock_write_unlock(&mng->resize_rwl);
12338         return 0;
12339 }
12340
12341 /*
12342  * Create and initialize a new ASO CT pool.
12343  *
12344  * @param[in] dev
12345  *   Pointer to the Ethernet device structure.
12346  * @param[out] ct_free
12347  *   Where to put the pointer of a new CT action.
12348  *
12349  * @return
12350  *   The CT actions pool pointer and @p ct_free is set on success,
12351  *   NULL otherwise and rte_errno is set.
12352  */
12353 static struct mlx5_aso_ct_pool *
12354 flow_dv_ct_pool_create(struct rte_eth_dev *dev,
12355                        struct mlx5_aso_ct_action **ct_free)
12356 {
12357         struct mlx5_priv *priv = dev->data->dev_private;
12358         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12359         struct mlx5_aso_ct_pool *pool = NULL;
12360         struct mlx5_devx_obj *obj = NULL;
12361         uint32_t i;
12362         uint32_t log_obj_size = rte_log2_u32(MLX5_ASO_CT_ACTIONS_PER_POOL);
12363
12364         obj = mlx5_devx_cmd_create_conn_track_offload_obj(priv->sh->ctx,
12365                                                 priv->sh->pdn, log_obj_size);
12366         if (!obj) {
12367                 rte_errno = ENODATA;
12368                 DRV_LOG(ERR, "Failed to create conn_track_offload_obj using DevX.");
12369                 return NULL;
12370         }
12371         pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
12372         if (!pool) {
12373                 rte_errno = ENOMEM;
12374                 claim_zero(mlx5_devx_cmd_destroy(obj));
12375                 return NULL;
12376         }
12377         pool->devx_obj = obj;
12378         pool->index = mng->next;
12379         /* Resize pools array if there is no room for the new pool in it. */
12380         if (pool->index == mng->n && flow_dv_aso_ct_pools_resize(dev)) {
12381                 claim_zero(mlx5_devx_cmd_destroy(obj));
12382                 mlx5_free(pool);
12383                 return NULL;
12384         }
12385         mng->pools[pool->index] = pool;
12386         mng->next++;
12387         /* Assign the first action in the new pool, the rest go to free list. */
12388         *ct_free = &pool->actions[0];
12389         /* Lock outside, the list operation is safe here. */
12390         for (i = 1; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) {
12391                 /* refcnt is 0 when allocating the memory. */
12392                 pool->actions[i].offset = i;
12393                 LIST_INSERT_HEAD(&mng->free_cts, &pool->actions[i], next);
12394         }
12395         return pool;
12396 }
12397
12398 /*
12399  * Allocate a ASO CT action from free list.
12400  *
12401  * @param[in] dev
12402  *   Pointer to the Ethernet device structure.
12403  * @param[out] error
12404  *   Pointer to the error structure.
12405  *
12406  * @return
12407  *   Index to ASO CT action on success, 0 otherwise and rte_errno is set.
12408  */
12409 static uint32_t
12410 flow_dv_aso_ct_alloc(struct rte_eth_dev *dev, struct rte_flow_error *error)
12411 {
12412         struct mlx5_priv *priv = dev->data->dev_private;
12413         struct mlx5_aso_ct_pools_mng *mng = priv->sh->ct_mng;
12414         struct mlx5_aso_ct_action *ct = NULL;
12415         struct mlx5_aso_ct_pool *pool;
12416         uint8_t reg_c;
12417         uint32_t ct_idx;
12418
12419         MLX5_ASSERT(mng);
12420         if (!priv->config.devx) {
12421                 rte_errno = ENOTSUP;
12422                 return 0;
12423         }
12424         /* Get a free CT action, if no, a new pool will be created. */
12425         rte_spinlock_lock(&mng->ct_sl);
12426         ct = LIST_FIRST(&mng->free_cts);
12427         if (ct) {
12428                 LIST_REMOVE(ct, next);
12429         } else if (!flow_dv_ct_pool_create(dev, &ct)) {
12430                 rte_spinlock_unlock(&mng->ct_sl);
12431                 rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION,
12432                                    NULL, "failed to create ASO CT pool");
12433                 return 0;
12434         }
12435         rte_spinlock_unlock(&mng->ct_sl);
12436         pool = container_of(ct, struct mlx5_aso_ct_pool, actions[ct->offset]);
12437         ct_idx = MLX5_MAKE_CT_IDX(pool->index, ct->offset);
12438         /* 0: inactive, 1: created, 2+: used by flows. */
12439         __atomic_store_n(&ct->refcnt, 1, __ATOMIC_RELAXED);
12440         reg_c = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, error);
12441         if (!ct->dr_action_orig) {
12442 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12443                 ct->dr_action_orig = mlx5_glue->dv_create_flow_action_aso
12444                         (priv->sh->rx_domain, pool->devx_obj->obj,
12445                          ct->offset,
12446                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_INITIATOR,
12447                          reg_c - REG_C_0);
12448 #else
12449                 RTE_SET_USED(reg_c);
12450 #endif
12451                 if (!ct->dr_action_orig) {
12452                         flow_dv_aso_ct_dev_release(dev, ct_idx);
12453                         rte_flow_error_set(error, rte_errno,
12454                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12455                                            "failed to create ASO CT action");
12456                         return 0;
12457                 }
12458         }
12459         if (!ct->dr_action_rply) {
12460 #ifdef HAVE_MLX5_DR_ACTION_ASO_CT
12461                 ct->dr_action_rply = mlx5_glue->dv_create_flow_action_aso
12462                         (priv->sh->rx_domain, pool->devx_obj->obj,
12463                          ct->offset,
12464                          MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_RESPONDER,
12465                          reg_c - REG_C_0);
12466 #endif
12467                 if (!ct->dr_action_rply) {
12468                         flow_dv_aso_ct_dev_release(dev, ct_idx);
12469                         rte_flow_error_set(error, rte_errno,
12470                                            RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12471                                            "failed to create ASO CT action");
12472                         return 0;
12473                 }
12474         }
12475         return ct_idx;
12476 }
12477
12478 /*
12479  * Create a conntrack object with context and actions by using ASO mechanism.
12480  *
12481  * @param[in] dev
12482  *   Pointer to rte_eth_dev structure.
12483  * @param[in] pro
12484  *   Pointer to conntrack information profile.
12485  * @param[out] error
12486  *   Pointer to the error structure.
12487  *
12488  * @return
12489  *   Index to conntrack object on success, 0 otherwise.
12490  */
12491 static uint32_t
12492 flow_dv_translate_create_conntrack(struct rte_eth_dev *dev,
12493                                    const struct rte_flow_action_conntrack *pro,
12494                                    struct rte_flow_error *error)
12495 {
12496         struct mlx5_priv *priv = dev->data->dev_private;
12497         struct mlx5_dev_ctx_shared *sh = priv->sh;
12498         struct mlx5_aso_ct_action *ct;
12499         uint32_t idx;
12500
12501         if (!sh->ct_aso_en)
12502                 return rte_flow_error_set(error, ENOTSUP,
12503                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12504                                           "Connection is not supported");
12505         idx = flow_dv_aso_ct_alloc(dev, error);
12506         if (!idx)
12507                 return rte_flow_error_set(error, rte_errno,
12508                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12509                                           "Failed to allocate CT object");
12510         ct = flow_aso_ct_get_by_dev_idx(dev, idx);
12511         if (mlx5_aso_ct_update_by_wqe(sh, ct, pro))
12512                 return rte_flow_error_set(error, EBUSY,
12513                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
12514                                           "Failed to update CT");
12515         ct->is_original = !!pro->is_original_dir;
12516         ct->peer = pro->peer_port;
12517         return idx;
12518 }
12519
12520 /**
12521  * Fill the flow with DV spec, lock free
12522  * (mutex should be acquired by caller).
12523  *
12524  * @param[in] dev
12525  *   Pointer to rte_eth_dev structure.
12526  * @param[in, out] dev_flow
12527  *   Pointer to the sub flow.
12528  * @param[in] attr
12529  *   Pointer to the flow attributes.
12530  * @param[in] items
12531  *   Pointer to the list of items.
12532  * @param[in] actions
12533  *   Pointer to the list of actions.
12534  * @param[out] error
12535  *   Pointer to the error structure.
12536  *
12537  * @return
12538  *   0 on success, a negative errno value otherwise and rte_errno is set.
12539  */
12540 static int
12541 flow_dv_translate(struct rte_eth_dev *dev,
12542                   struct mlx5_flow *dev_flow,
12543                   const struct rte_flow_attr *attr,
12544                   const struct rte_flow_item items[],
12545                   const struct rte_flow_action actions[],
12546                   struct rte_flow_error *error)
12547 {
12548         struct mlx5_priv *priv = dev->data->dev_private;
12549         struct mlx5_dev_config *dev_conf = &priv->config;
12550         struct rte_flow *flow = dev_flow->flow;
12551         struct mlx5_flow_handle *handle = dev_flow->handle;
12552         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
12553         struct mlx5_flow_rss_desc *rss_desc;
12554         uint64_t item_flags = 0;
12555         uint64_t last_item = 0;
12556         uint64_t action_flags = 0;
12557         struct mlx5_flow_dv_matcher matcher = {
12558                 .mask = {
12559                         .size = sizeof(matcher.mask.buf),
12560                 },
12561         };
12562         int actions_n = 0;
12563         bool actions_end = false;
12564         union {
12565                 struct mlx5_flow_dv_modify_hdr_resource res;
12566                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
12567                             sizeof(struct mlx5_modification_cmd) *
12568                             (MLX5_MAX_MODIFY_NUM + 1)];
12569         } mhdr_dummy;
12570         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
12571         const struct rte_flow_action_count *count = NULL;
12572         const struct rte_flow_action_age *non_shared_age = NULL;
12573         union flow_dv_attr flow_attr = { .attr = 0 };
12574         uint32_t tag_be;
12575         union mlx5_flow_tbl_key tbl_key;
12576         uint32_t modify_action_position = UINT32_MAX;
12577         void *match_mask = matcher.mask.buf;
12578         void *match_value = dev_flow->dv.value.buf;
12579         uint8_t next_protocol = 0xff;
12580         struct rte_vlan_hdr vlan = { 0 };
12581         struct mlx5_flow_dv_dest_array_resource mdest_res;
12582         struct mlx5_flow_dv_sample_resource sample_res;
12583         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
12584         const struct rte_flow_action_sample *sample = NULL;
12585         struct mlx5_flow_sub_actions_list *sample_act;
12586         uint32_t sample_act_pos = UINT32_MAX;
12587         uint32_t age_act_pos = UINT32_MAX;
12588         uint32_t num_of_dest = 0;
12589         int tmp_actions_n = 0;
12590         uint32_t table;
12591         int ret = 0;
12592         const struct mlx5_flow_tunnel *tunnel = NULL;
12593         struct flow_grp_info grp_info = {
12594                 .external = !!dev_flow->external,
12595                 .transfer = !!attr->transfer,
12596                 .fdb_def_rule = !!priv->fdb_def_rule,
12597                 .skip_scale = dev_flow->skip_scale &
12598                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
12599                 .std_tbl_fix = true,
12600         };
12601         const struct rte_flow_item *head_item = items;
12602
12603         if (!wks)
12604                 return rte_flow_error_set(error, ENOMEM,
12605                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12606                                           NULL,
12607                                           "failed to push flow workspace");
12608         rss_desc = &wks->rss_desc;
12609         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
12610         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
12611         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12612                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12613         /* update normal path action resource into last index of array */
12614         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
12615         if (is_tunnel_offload_active(dev)) {
12616                 if (dev_flow->tunnel) {
12617                         RTE_VERIFY(dev_flow->tof_type ==
12618                                    MLX5_TUNNEL_OFFLOAD_MISS_RULE);
12619                         tunnel = dev_flow->tunnel;
12620                 } else {
12621                         tunnel = mlx5_get_tof(items, actions,
12622                                               &dev_flow->tof_type);
12623                         dev_flow->tunnel = tunnel;
12624                 }
12625                 grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
12626                                         (dev, attr, tunnel, dev_flow->tof_type);
12627         }
12628         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
12629                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
12630         ret = mlx5_flow_group_to_table(dev, tunnel, attr->group, &table,
12631                                        &grp_info, error);
12632         if (ret)
12633                 return ret;
12634         dev_flow->dv.group = table;
12635         if (attr->transfer)
12636                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
12637         /* number of actions must be set to 0 in case of dirty stack. */
12638         mhdr_res->actions_num = 0;
12639         if (is_flow_tunnel_match_rule(dev_flow->tof_type)) {
12640                 /*
12641                  * do not add decap action if match rule drops packet
12642                  * HW rejects rules with decap & drop
12643                  *
12644                  * if tunnel match rule was inserted before matching tunnel set
12645                  * rule flow table used in the match rule must be registered.
12646                  * current implementation handles that in the
12647                  * flow_dv_match_register() at the function end.
12648                  */
12649                 bool add_decap = true;
12650                 const struct rte_flow_action *ptr = actions;
12651
12652                 for (; ptr->type != RTE_FLOW_ACTION_TYPE_END; ptr++) {
12653                         if (ptr->type == RTE_FLOW_ACTION_TYPE_DROP) {
12654                                 add_decap = false;
12655                                 break;
12656                         }
12657                 }
12658                 if (add_decap) {
12659                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
12660                                                            attr->transfer,
12661                                                            error))
12662                                 return -rte_errno;
12663                         dev_flow->dv.actions[actions_n++] =
12664                                         dev_flow->dv.encap_decap->action;
12665                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12666                 }
12667         }
12668         for (; !actions_end ; actions++) {
12669                 const struct rte_flow_action_queue *queue;
12670                 const struct rte_flow_action_rss *rss;
12671                 const struct rte_flow_action *action = actions;
12672                 const uint8_t *rss_key;
12673                 struct mlx5_flow_tbl_resource *tbl;
12674                 struct mlx5_aso_age_action *age_act;
12675                 struct mlx5_flow_counter *cnt_act;
12676                 uint32_t port_id = 0;
12677                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
12678                 int action_type = actions->type;
12679                 const struct rte_flow_action *found_action = NULL;
12680                 uint32_t jump_group = 0;
12681                 uint32_t owner_idx;
12682                 struct mlx5_aso_ct_action *ct;
12683
12684                 if (!mlx5_flow_os_action_supported(action_type))
12685                         return rte_flow_error_set(error, ENOTSUP,
12686                                                   RTE_FLOW_ERROR_TYPE_ACTION,
12687                                                   actions,
12688                                                   "action not supported");
12689                 switch (action_type) {
12690                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
12691                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
12692                         break;
12693                 case RTE_FLOW_ACTION_TYPE_VOID:
12694                         break;
12695                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
12696                         if (flow_dv_translate_action_port_id(dev, action,
12697                                                              &port_id, error))
12698                                 return -rte_errno;
12699                         port_id_resource.port_id = port_id;
12700                         MLX5_ASSERT(!handle->rix_port_id_action);
12701                         if (flow_dv_port_id_action_resource_register
12702                             (dev, &port_id_resource, dev_flow, error))
12703                                 return -rte_errno;
12704                         dev_flow->dv.actions[actions_n++] =
12705                                         dev_flow->dv.port_id_action->action;
12706                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
12707                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
12708                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
12709                         num_of_dest++;
12710                         break;
12711                 case RTE_FLOW_ACTION_TYPE_FLAG:
12712                         action_flags |= MLX5_FLOW_ACTION_FLAG;
12713                         dev_flow->handle->mark = 1;
12714                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
12715                                 struct rte_flow_action_mark mark = {
12716                                         .id = MLX5_FLOW_MARK_DEFAULT,
12717                                 };
12718
12719                                 if (flow_dv_convert_action_mark(dev, &mark,
12720                                                                 mhdr_res,
12721                                                                 error))
12722                                         return -rte_errno;
12723                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
12724                                 break;
12725                         }
12726                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
12727                         /*
12728                          * Only one FLAG or MARK is supported per device flow
12729                          * right now. So the pointer to the tag resource must be
12730                          * zero before the register process.
12731                          */
12732                         MLX5_ASSERT(!handle->dvh.rix_tag);
12733                         if (flow_dv_tag_resource_register(dev, tag_be,
12734                                                           dev_flow, error))
12735                                 return -rte_errno;
12736                         MLX5_ASSERT(dev_flow->dv.tag_resource);
12737                         dev_flow->dv.actions[actions_n++] =
12738                                         dev_flow->dv.tag_resource->action;
12739                         break;
12740                 case RTE_FLOW_ACTION_TYPE_MARK:
12741                         action_flags |= MLX5_FLOW_ACTION_MARK;
12742                         dev_flow->handle->mark = 1;
12743                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
12744                                 const struct rte_flow_action_mark *mark =
12745                                         (const struct rte_flow_action_mark *)
12746                                                 actions->conf;
12747
12748                                 if (flow_dv_convert_action_mark(dev, mark,
12749                                                                 mhdr_res,
12750                                                                 error))
12751                                         return -rte_errno;
12752                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
12753                                 break;
12754                         }
12755                         /* Fall-through */
12756                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
12757                         /* Legacy (non-extensive) MARK action. */
12758                         tag_be = mlx5_flow_mark_set
12759                               (((const struct rte_flow_action_mark *)
12760                                (actions->conf))->id);
12761                         MLX5_ASSERT(!handle->dvh.rix_tag);
12762                         if (flow_dv_tag_resource_register(dev, tag_be,
12763                                                           dev_flow, error))
12764                                 return -rte_errno;
12765                         MLX5_ASSERT(dev_flow->dv.tag_resource);
12766                         dev_flow->dv.actions[actions_n++] =
12767                                         dev_flow->dv.tag_resource->action;
12768                         break;
12769                 case RTE_FLOW_ACTION_TYPE_SET_META:
12770                         if (flow_dv_convert_action_set_meta
12771                                 (dev, mhdr_res, attr,
12772                                  (const struct rte_flow_action_set_meta *)
12773                                   actions->conf, error))
12774                                 return -rte_errno;
12775                         action_flags |= MLX5_FLOW_ACTION_SET_META;
12776                         break;
12777                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
12778                         if (flow_dv_convert_action_set_tag
12779                                 (dev, mhdr_res,
12780                                  (const struct rte_flow_action_set_tag *)
12781                                   actions->conf, error))
12782                                 return -rte_errno;
12783                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
12784                         break;
12785                 case RTE_FLOW_ACTION_TYPE_DROP:
12786                         action_flags |= MLX5_FLOW_ACTION_DROP;
12787                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
12788                         break;
12789                 case RTE_FLOW_ACTION_TYPE_QUEUE:
12790                         queue = actions->conf;
12791                         rss_desc->queue_num = 1;
12792                         rss_desc->queue[0] = queue->index;
12793                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
12794                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
12795                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
12796                         num_of_dest++;
12797                         break;
12798                 case RTE_FLOW_ACTION_TYPE_RSS:
12799                         rss = actions->conf;
12800                         memcpy(rss_desc->queue, rss->queue,
12801                                rss->queue_num * sizeof(uint16_t));
12802                         rss_desc->queue_num = rss->queue_num;
12803                         /* NULL RSS key indicates default RSS key. */
12804                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
12805                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
12806                         /*
12807                          * rss->level and rss.types should be set in advance
12808                          * when expanding items for RSS.
12809                          */
12810                         action_flags |= MLX5_FLOW_ACTION_RSS;
12811                         dev_flow->handle->fate_action = rss_desc->shared_rss ?
12812                                 MLX5_FLOW_FATE_SHARED_RSS :
12813                                 MLX5_FLOW_FATE_QUEUE;
12814                         break;
12815                 case MLX5_RTE_FLOW_ACTION_TYPE_AGE:
12816                         flow->age = (uint32_t)(uintptr_t)(action->conf);
12817                         age_act = flow_aso_age_get_by_idx(dev, flow->age);
12818                         __atomic_fetch_add(&age_act->refcnt, 1,
12819                                            __ATOMIC_RELAXED);
12820                         age_act_pos = actions_n++;
12821                         action_flags |= MLX5_FLOW_ACTION_AGE;
12822                         break;
12823                 case RTE_FLOW_ACTION_TYPE_AGE:
12824                         non_shared_age = action->conf;
12825                         age_act_pos = actions_n++;
12826                         action_flags |= MLX5_FLOW_ACTION_AGE;
12827                         break;
12828                 case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
12829                         flow->counter = (uint32_t)(uintptr_t)(action->conf);
12830                         cnt_act = flow_dv_counter_get_by_idx(dev, flow->counter,
12831                                                              NULL);
12832                         __atomic_fetch_add(&cnt_act->shared_info.refcnt, 1,
12833                                            __ATOMIC_RELAXED);
12834                         /* Save information first, will apply later. */
12835                         action_flags |= MLX5_FLOW_ACTION_COUNT;
12836                         break;
12837                 case RTE_FLOW_ACTION_TYPE_COUNT:
12838                         if (!dev_conf->devx) {
12839                                 return rte_flow_error_set
12840                                               (error, ENOTSUP,
12841                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12842                                                NULL,
12843                                                "count action not supported");
12844                         }
12845                         /* Save information first, will apply later. */
12846                         count = action->conf;
12847                         action_flags |= MLX5_FLOW_ACTION_COUNT;
12848                         break;
12849                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
12850                         dev_flow->dv.actions[actions_n++] =
12851                                                 priv->sh->pop_vlan_action;
12852                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
12853                         break;
12854                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
12855                         if (!(action_flags &
12856                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
12857                                 flow_dev_get_vlan_info_from_items(items, &vlan);
12858                         vlan.eth_proto = rte_be_to_cpu_16
12859                              ((((const struct rte_flow_action_of_push_vlan *)
12860                                                    actions->conf)->ethertype));
12861                         found_action = mlx5_flow_find_action
12862                                         (actions + 1,
12863                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
12864                         if (found_action)
12865                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
12866                         found_action = mlx5_flow_find_action
12867                                         (actions + 1,
12868                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
12869                         if (found_action)
12870                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
12871                         if (flow_dv_create_action_push_vlan
12872                                             (dev, attr, &vlan, dev_flow, error))
12873                                 return -rte_errno;
12874                         dev_flow->dv.actions[actions_n++] =
12875                                         dev_flow->dv.push_vlan_res->action;
12876                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
12877                         break;
12878                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
12879                         /* of_vlan_push action handled this action */
12880                         MLX5_ASSERT(action_flags &
12881                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
12882                         break;
12883                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
12884                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
12885                                 break;
12886                         flow_dev_get_vlan_info_from_items(items, &vlan);
12887                         mlx5_update_vlan_vid_pcp(actions, &vlan);
12888                         /* If no VLAN push - this is a modify header action */
12889                         if (flow_dv_convert_action_modify_vlan_vid
12890                                                 (mhdr_res, actions, error))
12891                                 return -rte_errno;
12892                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
12893                         break;
12894                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
12895                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
12896                         if (flow_dv_create_action_l2_encap(dev, actions,
12897                                                            dev_flow,
12898                                                            attr->transfer,
12899                                                            error))
12900                                 return -rte_errno;
12901                         dev_flow->dv.actions[actions_n++] =
12902                                         dev_flow->dv.encap_decap->action;
12903                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
12904                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
12905                                 sample_act->action_flags |=
12906                                                         MLX5_FLOW_ACTION_ENCAP;
12907                         break;
12908                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
12909                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
12910                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
12911                                                            attr->transfer,
12912                                                            error))
12913                                 return -rte_errno;
12914                         dev_flow->dv.actions[actions_n++] =
12915                                         dev_flow->dv.encap_decap->action;
12916                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12917                         break;
12918                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
12919                         /* Handle encap with preceding decap. */
12920                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
12921                                 if (flow_dv_create_action_raw_encap
12922                                         (dev, actions, dev_flow, attr, error))
12923                                         return -rte_errno;
12924                                 dev_flow->dv.actions[actions_n++] =
12925                                         dev_flow->dv.encap_decap->action;
12926                         } else {
12927                                 /* Handle encap without preceding decap. */
12928                                 if (flow_dv_create_action_l2_encap
12929                                     (dev, actions, dev_flow, attr->transfer,
12930                                      error))
12931                                         return -rte_errno;
12932                                 dev_flow->dv.actions[actions_n++] =
12933                                         dev_flow->dv.encap_decap->action;
12934                         }
12935                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
12936                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
12937                                 sample_act->action_flags |=
12938                                                         MLX5_FLOW_ACTION_ENCAP;
12939                         break;
12940                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
12941                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
12942                                 ;
12943                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
12944                                 if (flow_dv_create_action_l2_decap
12945                                     (dev, dev_flow, attr->transfer, error))
12946                                         return -rte_errno;
12947                                 dev_flow->dv.actions[actions_n++] =
12948                                         dev_flow->dv.encap_decap->action;
12949                         }
12950                         /* If decap is followed by encap, handle it at encap. */
12951                         action_flags |= MLX5_FLOW_ACTION_DECAP;
12952                         break;
12953                 case MLX5_RTE_FLOW_ACTION_TYPE_JUMP:
12954                         dev_flow->dv.actions[actions_n++] =
12955                                 (void *)(uintptr_t)action->conf;
12956                         action_flags |= MLX5_FLOW_ACTION_JUMP;
12957                         break;
12958                 case RTE_FLOW_ACTION_TYPE_JUMP:
12959                         jump_group = ((const struct rte_flow_action_jump *)
12960                                                         action->conf)->group;
12961                         grp_info.std_tbl_fix = 0;
12962                         if (dev_flow->skip_scale &
12963                                 (1 << MLX5_SCALE_JUMP_FLOW_GROUP_BIT))
12964                                 grp_info.skip_scale = 1;
12965                         else
12966                                 grp_info.skip_scale = 0;
12967                         ret = mlx5_flow_group_to_table(dev, tunnel,
12968                                                        jump_group,
12969                                                        &table,
12970                                                        &grp_info, error);
12971                         if (ret)
12972                                 return ret;
12973                         tbl = flow_dv_tbl_resource_get(dev, table, attr->egress,
12974                                                        attr->transfer,
12975                                                        !!dev_flow->external,
12976                                                        tunnel, jump_group, 0,
12977                                                        0, error);
12978                         if (!tbl)
12979                                 return rte_flow_error_set
12980                                                 (error, errno,
12981                                                  RTE_FLOW_ERROR_TYPE_ACTION,
12982                                                  NULL,
12983                                                  "cannot create jump action.");
12984                         if (flow_dv_jump_tbl_resource_register
12985                             (dev, tbl, dev_flow, error)) {
12986                                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
12987                                 return rte_flow_error_set
12988                                                 (error, errno,
12989                                                  RTE_FLOW_ERROR_TYPE_ACTION,
12990                                                  NULL,
12991                                                  "cannot create jump action.");
12992                         }
12993                         dev_flow->dv.actions[actions_n++] =
12994                                         dev_flow->dv.jump->action;
12995                         action_flags |= MLX5_FLOW_ACTION_JUMP;
12996                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
12997                         sample_act->action_flags |= MLX5_FLOW_ACTION_JUMP;
12998                         num_of_dest++;
12999                         break;
13000                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
13001                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
13002                         if (flow_dv_convert_action_modify_mac
13003                                         (mhdr_res, actions, error))
13004                                 return -rte_errno;
13005                         action_flags |= actions->type ==
13006                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
13007                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
13008                                         MLX5_FLOW_ACTION_SET_MAC_DST;
13009                         break;
13010                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
13011                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
13012                         if (flow_dv_convert_action_modify_ipv4
13013                                         (mhdr_res, actions, error))
13014                                 return -rte_errno;
13015                         action_flags |= actions->type ==
13016                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
13017                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
13018                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
13019                         break;
13020                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
13021                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
13022                         if (flow_dv_convert_action_modify_ipv6
13023                                         (mhdr_res, actions, error))
13024                                 return -rte_errno;
13025                         action_flags |= actions->type ==
13026                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
13027                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
13028                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
13029                         break;
13030                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
13031                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
13032                         if (flow_dv_convert_action_modify_tp
13033                                         (mhdr_res, actions, items,
13034                                          &flow_attr, dev_flow, !!(action_flags &
13035                                          MLX5_FLOW_ACTION_DECAP), error))
13036                                 return -rte_errno;
13037                         action_flags |= actions->type ==
13038                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
13039                                         MLX5_FLOW_ACTION_SET_TP_SRC :
13040                                         MLX5_FLOW_ACTION_SET_TP_DST;
13041                         break;
13042                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
13043                         if (flow_dv_convert_action_modify_dec_ttl
13044                                         (mhdr_res, items, &flow_attr, dev_flow,
13045                                          !!(action_flags &
13046                                          MLX5_FLOW_ACTION_DECAP), error))
13047                                 return -rte_errno;
13048                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
13049                         break;
13050                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
13051                         if (flow_dv_convert_action_modify_ttl
13052                                         (mhdr_res, actions, items, &flow_attr,
13053                                          dev_flow, !!(action_flags &
13054                                          MLX5_FLOW_ACTION_DECAP), error))
13055                                 return -rte_errno;
13056                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
13057                         break;
13058                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
13059                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
13060                         if (flow_dv_convert_action_modify_tcp_seq
13061                                         (mhdr_res, actions, error))
13062                                 return -rte_errno;
13063                         action_flags |= actions->type ==
13064                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
13065                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
13066                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
13067                         break;
13068
13069                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
13070                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
13071                         if (flow_dv_convert_action_modify_tcp_ack
13072                                         (mhdr_res, actions, error))
13073                                 return -rte_errno;
13074                         action_flags |= actions->type ==
13075                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
13076                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
13077                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
13078                         break;
13079                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
13080                         if (flow_dv_convert_action_set_reg
13081                                         (mhdr_res, actions, error))
13082                                 return -rte_errno;
13083                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
13084                         break;
13085                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
13086                         if (flow_dv_convert_action_copy_mreg
13087                                         (dev, mhdr_res, actions, error))
13088                                 return -rte_errno;
13089                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
13090                         break;
13091                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
13092                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
13093                         dev_flow->handle->fate_action =
13094                                         MLX5_FLOW_FATE_DEFAULT_MISS;
13095                         break;
13096                 case RTE_FLOW_ACTION_TYPE_METER:
13097                         if (!wks->fm)
13098                                 return rte_flow_error_set(error, rte_errno,
13099                                         RTE_FLOW_ERROR_TYPE_ACTION,
13100                                         NULL, "Failed to get meter in flow.");
13101                         /* Set the meter action. */
13102                         dev_flow->dv.actions[actions_n++] =
13103                                 wks->fm->meter_action;
13104                         action_flags |= MLX5_FLOW_ACTION_METER;
13105                         break;
13106                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
13107                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
13108                                                               actions, error))
13109                                 return -rte_errno;
13110                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
13111                         break;
13112                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
13113                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
13114                                                               actions, error))
13115                                 return -rte_errno;
13116                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
13117                         break;
13118                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
13119                         sample_act_pos = actions_n;
13120                         sample = (const struct rte_flow_action_sample *)
13121                                  action->conf;
13122                         actions_n++;
13123                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
13124                         /* put encap action into group if work with port id */
13125                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
13126                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
13127                                 sample_act->action_flags |=
13128                                                         MLX5_FLOW_ACTION_ENCAP;
13129                         break;
13130                 case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
13131                         if (flow_dv_convert_action_modify_field
13132                                         (dev, mhdr_res, actions, attr, error))
13133                                 return -rte_errno;
13134                         action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
13135                         break;
13136                 case RTE_FLOW_ACTION_TYPE_CONNTRACK:
13137                         owner_idx = (uint32_t)(uintptr_t)action->conf;
13138                         ct = flow_aso_ct_get_by_idx(dev, owner_idx);
13139                         if (!ct)
13140                                 return rte_flow_error_set(error, EINVAL,
13141                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13142                                                 NULL,
13143                                                 "Failed to get CT object.");
13144                         if (mlx5_aso_ct_available(priv->sh, ct))
13145                                 return rte_flow_error_set(error, rte_errno,
13146                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13147                                                 NULL,
13148                                                 "CT is unavailable.");
13149                         if (ct->is_original)
13150                                 dev_flow->dv.actions[actions_n] =
13151                                                         ct->dr_action_orig;
13152                         else
13153                                 dev_flow->dv.actions[actions_n] =
13154                                                         ct->dr_action_rply;
13155                         flow->indirect_type = MLX5_INDIRECT_ACTION_TYPE_CT;
13156                         flow->ct = owner_idx;
13157                         __atomic_fetch_add(&ct->refcnt, 1, __ATOMIC_RELAXED);
13158                         actions_n++;
13159                         action_flags |= MLX5_FLOW_ACTION_CT;
13160                         break;
13161                 case RTE_FLOW_ACTION_TYPE_END:
13162                         actions_end = true;
13163                         if (mhdr_res->actions_num) {
13164                                 /* create modify action if needed. */
13165                                 if (flow_dv_modify_hdr_resource_register
13166                                         (dev, mhdr_res, dev_flow, error))
13167                                         return -rte_errno;
13168                                 dev_flow->dv.actions[modify_action_position] =
13169                                         handle->dvh.modify_hdr->action;
13170                         }
13171                         /*
13172                          * Handle AGE and COUNT action by single HW counter
13173                          * when they are not shared.
13174                          */
13175                         if (action_flags & MLX5_FLOW_ACTION_AGE) {
13176                                 if ((non_shared_age &&
13177                                      count && !count->shared) ||
13178                                     !(priv->sh->flow_hit_aso_en &&
13179                                       (attr->group || attr->transfer))) {
13180                                         /* Creates age by counters. */
13181                                         cnt_act = flow_dv_prepare_counter
13182                                                                 (dev, dev_flow,
13183                                                                  flow, count,
13184                                                                  non_shared_age,
13185                                                                  error);
13186                                         if (!cnt_act)
13187                                                 return -rte_errno;
13188                                         dev_flow->dv.actions[age_act_pos] =
13189                                                                 cnt_act->action;
13190                                         break;
13191                                 }
13192                                 if (!flow->age && non_shared_age) {
13193                                         flow->age = flow_dv_aso_age_alloc
13194                                                                 (dev, error);
13195                                         if (!flow->age)
13196                                                 return -rte_errno;
13197                                         flow_dv_aso_age_params_init
13198                                                     (dev, flow->age,
13199                                                      non_shared_age->context ?
13200                                                      non_shared_age->context :
13201                                                      (void *)(uintptr_t)
13202                                                      (dev_flow->flow_idx),
13203                                                      non_shared_age->timeout);
13204                                 }
13205                                 age_act = flow_aso_age_get_by_idx(dev,
13206                                                                   flow->age);
13207                                 dev_flow->dv.actions[age_act_pos] =
13208                                                              age_act->dr_action;
13209                         }
13210                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
13211                                 /*
13212                                  * Create one count action, to be used
13213                                  * by all sub-flows.
13214                                  */
13215                                 cnt_act = flow_dv_prepare_counter(dev, dev_flow,
13216                                                                   flow, count,
13217                                                                   NULL, error);
13218                                 if (!cnt_act)
13219                                         return -rte_errno;
13220                                 dev_flow->dv.actions[actions_n++] =
13221                                                                 cnt_act->action;
13222                         }
13223                 default:
13224                         break;
13225                 }
13226                 if (mhdr_res->actions_num &&
13227                     modify_action_position == UINT32_MAX)
13228                         modify_action_position = actions_n++;
13229         }
13230         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
13231                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
13232                 int item_type = items->type;
13233
13234                 if (!mlx5_flow_os_item_supported(item_type))
13235                         return rte_flow_error_set(error, ENOTSUP,
13236                                                   RTE_FLOW_ERROR_TYPE_ITEM,
13237                                                   NULL, "item not supported");
13238                 switch (item_type) {
13239                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
13240                         flow_dv_translate_item_port_id
13241                                 (dev, match_mask, match_value, items, attr);
13242                         last_item = MLX5_FLOW_ITEM_PORT_ID;
13243                         break;
13244                 case RTE_FLOW_ITEM_TYPE_ETH:
13245                         flow_dv_translate_item_eth(match_mask, match_value,
13246                                                    items, tunnel,
13247                                                    dev_flow->dv.group);
13248                         matcher.priority = action_flags &
13249                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
13250                                         !dev_flow->external ?
13251                                         MLX5_PRIORITY_MAP_L3 :
13252                                         MLX5_PRIORITY_MAP_L2;
13253                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
13254                                              MLX5_FLOW_LAYER_OUTER_L2;
13255                         break;
13256                 case RTE_FLOW_ITEM_TYPE_VLAN:
13257                         flow_dv_translate_item_vlan(dev_flow,
13258                                                     match_mask, match_value,
13259                                                     items, tunnel,
13260                                                     dev_flow->dv.group);
13261                         matcher.priority = MLX5_PRIORITY_MAP_L2;
13262                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
13263                                               MLX5_FLOW_LAYER_INNER_VLAN) :
13264                                              (MLX5_FLOW_LAYER_OUTER_L2 |
13265                                               MLX5_FLOW_LAYER_OUTER_VLAN);
13266                         break;
13267                 case RTE_FLOW_ITEM_TYPE_IPV4:
13268                         mlx5_flow_tunnel_ip_check(items, next_protocol,
13269                                                   &item_flags, &tunnel);
13270                         flow_dv_translate_item_ipv4(match_mask, match_value,
13271                                                     items, tunnel,
13272                                                     dev_flow->dv.group);
13273                         matcher.priority = MLX5_PRIORITY_MAP_L3;
13274                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
13275                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
13276                         if (items->mask != NULL &&
13277                             ((const struct rte_flow_item_ipv4 *)
13278                              items->mask)->hdr.next_proto_id) {
13279                                 next_protocol =
13280                                         ((const struct rte_flow_item_ipv4 *)
13281                                          (items->spec))->hdr.next_proto_id;
13282                                 next_protocol &=
13283                                         ((const struct rte_flow_item_ipv4 *)
13284                                          (items->mask))->hdr.next_proto_id;
13285                         } else {
13286                                 /* Reset for inner layer. */
13287                                 next_protocol = 0xff;
13288                         }
13289                         break;
13290                 case RTE_FLOW_ITEM_TYPE_IPV6:
13291                         mlx5_flow_tunnel_ip_check(items, next_protocol,
13292                                                   &item_flags, &tunnel);
13293                         flow_dv_translate_item_ipv6(match_mask, match_value,
13294                                                     items, tunnel,
13295                                                     dev_flow->dv.group);
13296                         matcher.priority = MLX5_PRIORITY_MAP_L3;
13297                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
13298                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
13299                         if (items->mask != NULL &&
13300                             ((const struct rte_flow_item_ipv6 *)
13301                              items->mask)->hdr.proto) {
13302                                 next_protocol =
13303                                         ((const struct rte_flow_item_ipv6 *)
13304                                          items->spec)->hdr.proto;
13305                                 next_protocol &=
13306                                         ((const struct rte_flow_item_ipv6 *)
13307                                          items->mask)->hdr.proto;
13308                         } else {
13309                                 /* Reset for inner layer. */
13310                                 next_protocol = 0xff;
13311                         }
13312                         break;
13313                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
13314                         flow_dv_translate_item_ipv6_frag_ext(match_mask,
13315                                                              match_value,
13316                                                              items, tunnel);
13317                         last_item = tunnel ?
13318                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
13319                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
13320                         if (items->mask != NULL &&
13321                             ((const struct rte_flow_item_ipv6_frag_ext *)
13322                              items->mask)->hdr.next_header) {
13323                                 next_protocol =
13324                                 ((const struct rte_flow_item_ipv6_frag_ext *)
13325                                  items->spec)->hdr.next_header;
13326                                 next_protocol &=
13327                                 ((const struct rte_flow_item_ipv6_frag_ext *)
13328                                  items->mask)->hdr.next_header;
13329                         } else {
13330                                 /* Reset for inner layer. */
13331                                 next_protocol = 0xff;
13332                         }
13333                         break;
13334                 case RTE_FLOW_ITEM_TYPE_TCP:
13335                         flow_dv_translate_item_tcp(match_mask, match_value,
13336                                                    items, tunnel);
13337                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13338                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
13339                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
13340                         break;
13341                 case RTE_FLOW_ITEM_TYPE_UDP:
13342                         flow_dv_translate_item_udp(match_mask, match_value,
13343                                                    items, tunnel);
13344                         matcher.priority = MLX5_PRIORITY_MAP_L4;
13345                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
13346                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
13347                         break;
13348                 case RTE_FLOW_ITEM_TYPE_GRE:
13349                         flow_dv_translate_item_gre(match_mask, match_value,
13350                                                    items, tunnel);
13351                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13352                         last_item = MLX5_FLOW_LAYER_GRE;
13353                         break;
13354                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
13355                         flow_dv_translate_item_gre_key(match_mask,
13356                                                        match_value, items);
13357                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
13358                         break;
13359                 case RTE_FLOW_ITEM_TYPE_NVGRE:
13360                         flow_dv_translate_item_nvgre(match_mask, match_value,
13361                                                      items, tunnel);
13362                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13363                         last_item = MLX5_FLOW_LAYER_GRE;
13364                         break;
13365                 case RTE_FLOW_ITEM_TYPE_VXLAN:
13366                         flow_dv_translate_item_vxlan(dev, attr,
13367                                                      match_mask, match_value,
13368                                                      items, tunnel);
13369                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13370                         last_item = MLX5_FLOW_LAYER_VXLAN;
13371                         break;
13372                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
13373                         flow_dv_translate_item_vxlan_gpe(match_mask,
13374                                                          match_value, items,
13375                                                          tunnel);
13376                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13377                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
13378                         break;
13379                 case RTE_FLOW_ITEM_TYPE_GENEVE:
13380                         flow_dv_translate_item_geneve(match_mask, match_value,
13381                                                       items, tunnel);
13382                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13383                         last_item = MLX5_FLOW_LAYER_GENEVE;
13384                         break;
13385                 case RTE_FLOW_ITEM_TYPE_GENEVE_OPT:
13386                         ret = flow_dv_translate_item_geneve_opt(dev, match_mask,
13387                                                           match_value,
13388                                                           items, error);
13389                         if (ret)
13390                                 return rte_flow_error_set(error, -ret,
13391                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
13392                                         "cannot create GENEVE TLV option");
13393                         flow->geneve_tlv_option = 1;
13394                         last_item = MLX5_FLOW_LAYER_GENEVE_OPT;
13395                         break;
13396                 case RTE_FLOW_ITEM_TYPE_MPLS:
13397                         flow_dv_translate_item_mpls(match_mask, match_value,
13398                                                     items, last_item, tunnel);
13399                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13400                         last_item = MLX5_FLOW_LAYER_MPLS;
13401                         break;
13402                 case RTE_FLOW_ITEM_TYPE_MARK:
13403                         flow_dv_translate_item_mark(dev, match_mask,
13404                                                     match_value, items);
13405                         last_item = MLX5_FLOW_ITEM_MARK;
13406                         break;
13407                 case RTE_FLOW_ITEM_TYPE_META:
13408                         flow_dv_translate_item_meta(dev, match_mask,
13409                                                     match_value, attr, items);
13410                         last_item = MLX5_FLOW_ITEM_METADATA;
13411                         break;
13412                 case RTE_FLOW_ITEM_TYPE_ICMP:
13413                         flow_dv_translate_item_icmp(match_mask, match_value,
13414                                                     items, tunnel);
13415                         last_item = MLX5_FLOW_LAYER_ICMP;
13416                         break;
13417                 case RTE_FLOW_ITEM_TYPE_ICMP6:
13418                         flow_dv_translate_item_icmp6(match_mask, match_value,
13419                                                       items, tunnel);
13420                         last_item = MLX5_FLOW_LAYER_ICMP6;
13421                         break;
13422                 case RTE_FLOW_ITEM_TYPE_TAG:
13423                         flow_dv_translate_item_tag(dev, match_mask,
13424                                                    match_value, items);
13425                         last_item = MLX5_FLOW_ITEM_TAG;
13426                         break;
13427                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
13428                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
13429                                                         match_value, items);
13430                         last_item = MLX5_FLOW_ITEM_TAG;
13431                         break;
13432                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
13433                         flow_dv_translate_item_tx_queue(dev, match_mask,
13434                                                         match_value,
13435                                                         items);
13436                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
13437                         break;
13438                 case RTE_FLOW_ITEM_TYPE_GTP:
13439                         flow_dv_translate_item_gtp(match_mask, match_value,
13440                                                    items, tunnel);
13441                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
13442                         last_item = MLX5_FLOW_LAYER_GTP;
13443                         break;
13444                 case RTE_FLOW_ITEM_TYPE_GTP_PSC:
13445                         ret = flow_dv_translate_item_gtp_psc(match_mask,
13446                                                           match_value,
13447                                                           items);
13448                         if (ret)
13449                                 return rte_flow_error_set(error, -ret,
13450                                         RTE_FLOW_ERROR_TYPE_ITEM, NULL,
13451                                         "cannot create GTP PSC item");
13452                         last_item = MLX5_FLOW_LAYER_GTP_PSC;
13453                         break;
13454                 case RTE_FLOW_ITEM_TYPE_ECPRI:
13455                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
13456                                 /* Create it only the first time to be used. */
13457                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
13458                                 if (ret)
13459                                         return rte_flow_error_set
13460                                                 (error, -ret,
13461                                                 RTE_FLOW_ERROR_TYPE_ITEM,
13462                                                 NULL,
13463                                                 "cannot create eCPRI parser");
13464                         }
13465                         flow_dv_translate_item_ecpri(dev, match_mask,
13466                                                      match_value, items);
13467                         /* No other protocol should follow eCPRI layer. */
13468                         last_item = MLX5_FLOW_LAYER_ECPRI;
13469                         break;
13470                 case RTE_FLOW_ITEM_TYPE_INTEGRITY:
13471                         flow_dv_translate_item_integrity(match_mask,
13472                                                          match_value,
13473                                                          head_item, items);
13474                         break;
13475                 case RTE_FLOW_ITEM_TYPE_CONNTRACK:
13476                         flow_dv_translate_item_aso_ct(dev, match_mask,
13477                                                       match_value, items);
13478                         break;
13479                 default:
13480                         break;
13481                 }
13482                 item_flags |= last_item;
13483         }
13484         /*
13485          * When E-Switch mode is enabled, we have two cases where we need to
13486          * set the source port manually.
13487          * The first one, is in case of Nic steering rule, and the second is
13488          * E-Switch rule where no port_id item was found. In both cases
13489          * the source port is set according the current port in use.
13490          */
13491         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
13492             (priv->representor || priv->master)) {
13493                 if (flow_dv_translate_item_port_id(dev, match_mask,
13494                                                    match_value, NULL, attr))
13495                         return -rte_errno;
13496         }
13497 #ifdef RTE_LIBRTE_MLX5_DEBUG
13498         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
13499                                               dev_flow->dv.value.buf));
13500 #endif
13501         /*
13502          * Layers may be already initialized from prefix flow if this dev_flow
13503          * is the suffix flow.
13504          */
13505         handle->layers |= item_flags;
13506         if (action_flags & MLX5_FLOW_ACTION_RSS)
13507                 flow_dv_hashfields_set(dev_flow, rss_desc);
13508         /* If has RSS action in the sample action, the Sample/Mirror resource
13509          * should be registered after the hash filed be update.
13510          */
13511         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
13512                 ret = flow_dv_translate_action_sample(dev,
13513                                                       sample,
13514                                                       dev_flow, attr,
13515                                                       &num_of_dest,
13516                                                       sample_actions,
13517                                                       &sample_res,
13518                                                       error);
13519                 if (ret < 0)
13520                         return ret;
13521                 ret = flow_dv_create_action_sample(dev,
13522                                                    dev_flow,
13523                                                    num_of_dest,
13524                                                    &sample_res,
13525                                                    &mdest_res,
13526                                                    sample_actions,
13527                                                    action_flags,
13528                                                    error);
13529                 if (ret < 0)
13530                         return rte_flow_error_set
13531                                                 (error, rte_errno,
13532                                                 RTE_FLOW_ERROR_TYPE_ACTION,
13533                                                 NULL,
13534                                                 "cannot create sample action");
13535                 if (num_of_dest > 1) {
13536                         dev_flow->dv.actions[sample_act_pos] =
13537                         dev_flow->dv.dest_array_res->action;
13538                 } else {
13539                         dev_flow->dv.actions[sample_act_pos] =
13540                         dev_flow->dv.sample_res->verbs_action;
13541                 }
13542         }
13543         /*
13544          * For multiple destination (sample action with ratio=1), the encap
13545          * action and port id action will be combined into group action.
13546          * So need remove the original these actions in the flow and only
13547          * use the sample action instead of.
13548          */
13549         if (num_of_dest > 1 &&
13550             (sample_act->dr_port_id_action || sample_act->dr_jump_action)) {
13551                 int i;
13552                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
13553
13554                 for (i = 0; i < actions_n; i++) {
13555                         if ((sample_act->dr_encap_action &&
13556                                 sample_act->dr_encap_action ==
13557                                 dev_flow->dv.actions[i]) ||
13558                                 (sample_act->dr_port_id_action &&
13559                                 sample_act->dr_port_id_action ==
13560                                 dev_flow->dv.actions[i]) ||
13561                                 (sample_act->dr_jump_action &&
13562                                 sample_act->dr_jump_action ==
13563                                 dev_flow->dv.actions[i]))
13564                                 continue;
13565                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
13566                 }
13567                 memcpy((void *)dev_flow->dv.actions,
13568                                 (void *)temp_actions,
13569                                 tmp_actions_n * sizeof(void *));
13570                 actions_n = tmp_actions_n;
13571         }
13572         dev_flow->dv.actions_n = actions_n;
13573         dev_flow->act_flags = action_flags;
13574         if (wks->skip_matcher_reg)
13575                 return 0;
13576         /* Register matcher. */
13577         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
13578                                     matcher.mask.size);
13579         matcher.priority = mlx5_get_matcher_priority(dev, attr,
13580                                         matcher.priority);
13581         /**
13582          * When creating meter drop flow in drop table, using original
13583          * 5-tuple match, the matcher priority should be lower than
13584          * mtr_id matcher.
13585          */
13586         if (attr->group == MLX5_FLOW_TABLE_LEVEL_METER &&
13587             dev_flow->dv.table_id == MLX5_MTR_TABLE_ID_DROP &&
13588             matcher.priority <= MLX5_REG_BITS)
13589                 matcher.priority += MLX5_REG_BITS;
13590         /* reserved field no needs to be set to 0 here. */
13591         tbl_key.is_fdb = attr->transfer;
13592         tbl_key.is_egress = attr->egress;
13593         tbl_key.level = dev_flow->dv.group;
13594         tbl_key.id = dev_flow->dv.table_id;
13595         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow,
13596                                      tunnel, attr->group, error))
13597                 return -rte_errno;
13598         return 0;
13599 }
13600
13601 /**
13602  * Set hash RX queue by hash fields (see enum ibv_rx_hash_fields)
13603  * and tunnel.
13604  *
13605  * @param[in, out] action
13606  *   Shred RSS action holding hash RX queue objects.
13607  * @param[in] hash_fields
13608  *   Defines combination of packet fields to participate in RX hash.
13609  * @param[in] tunnel
13610  *   Tunnel type
13611  * @param[in] hrxq_idx
13612  *   Hash RX queue index to set.
13613  *
13614  * @return
13615  *   0 on success, otherwise negative errno value.
13616  */
13617 static int
13618 __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action,
13619                               const uint64_t hash_fields,
13620                               uint32_t hrxq_idx)
13621 {
13622         uint32_t *hrxqs = action->hrxq;
13623
13624         switch (hash_fields & ~IBV_RX_HASH_INNER) {
13625         case MLX5_RSS_HASH_IPV4:
13626                 /* fall-through. */
13627         case MLX5_RSS_HASH_IPV4_DST_ONLY:
13628                 /* fall-through. */
13629         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
13630                 hrxqs[0] = hrxq_idx;
13631                 return 0;
13632         case MLX5_RSS_HASH_IPV4_TCP:
13633                 /* fall-through. */
13634         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
13635                 /* fall-through. */
13636         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
13637                 hrxqs[1] = hrxq_idx;
13638                 return 0;
13639         case MLX5_RSS_HASH_IPV4_UDP:
13640                 /* fall-through. */
13641         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
13642                 /* fall-through. */
13643         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
13644                 hrxqs[2] = hrxq_idx;
13645                 return 0;
13646         case MLX5_RSS_HASH_IPV6:
13647                 /* fall-through. */
13648         case MLX5_RSS_HASH_IPV6_DST_ONLY:
13649                 /* fall-through. */
13650         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
13651                 hrxqs[3] = hrxq_idx;
13652                 return 0;
13653         case MLX5_RSS_HASH_IPV6_TCP:
13654                 /* fall-through. */
13655         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
13656                 /* fall-through. */
13657         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
13658                 hrxqs[4] = hrxq_idx;
13659                 return 0;
13660         case MLX5_RSS_HASH_IPV6_UDP:
13661                 /* fall-through. */
13662         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
13663                 /* fall-through. */
13664         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
13665                 hrxqs[5] = hrxq_idx;
13666                 return 0;
13667         case MLX5_RSS_HASH_NONE:
13668                 hrxqs[6] = hrxq_idx;
13669                 return 0;
13670         default:
13671                 return -1;
13672         }
13673 }
13674
13675 /**
13676  * Look up for hash RX queue by hash fields (see enum ibv_rx_hash_fields)
13677  * and tunnel.
13678  *
13679  * @param[in] dev
13680  *   Pointer to the Ethernet device structure.
13681  * @param[in] idx
13682  *   Shared RSS action ID holding hash RX queue objects.
13683  * @param[in] hash_fields
13684  *   Defines combination of packet fields to participate in RX hash.
13685  * @param[in] tunnel
13686  *   Tunnel type
13687  *
13688  * @return
13689  *   Valid hash RX queue index, otherwise 0.
13690  */
13691 static uint32_t
13692 __flow_dv_action_rss_hrxq_lookup(struct rte_eth_dev *dev, uint32_t idx,
13693                                  const uint64_t hash_fields)
13694 {
13695         struct mlx5_priv *priv = dev->data->dev_private;
13696         struct mlx5_shared_action_rss *shared_rss =
13697             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
13698         const uint32_t *hrxqs = shared_rss->hrxq;
13699
13700         switch (hash_fields & ~IBV_RX_HASH_INNER) {
13701         case MLX5_RSS_HASH_IPV4:
13702                 /* fall-through. */
13703         case MLX5_RSS_HASH_IPV4_DST_ONLY:
13704                 /* fall-through. */
13705         case MLX5_RSS_HASH_IPV4_SRC_ONLY:
13706                 return hrxqs[0];
13707         case MLX5_RSS_HASH_IPV4_TCP:
13708                 /* fall-through. */
13709         case MLX5_RSS_HASH_IPV4_TCP_DST_ONLY:
13710                 /* fall-through. */
13711         case MLX5_RSS_HASH_IPV4_TCP_SRC_ONLY:
13712                 return hrxqs[1];
13713         case MLX5_RSS_HASH_IPV4_UDP:
13714                 /* fall-through. */
13715         case MLX5_RSS_HASH_IPV4_UDP_DST_ONLY:
13716                 /* fall-through. */
13717         case MLX5_RSS_HASH_IPV4_UDP_SRC_ONLY:
13718                 return hrxqs[2];
13719         case MLX5_RSS_HASH_IPV6:
13720                 /* fall-through. */
13721         case MLX5_RSS_HASH_IPV6_DST_ONLY:
13722                 /* fall-through. */
13723         case MLX5_RSS_HASH_IPV6_SRC_ONLY:
13724                 return hrxqs[3];
13725         case MLX5_RSS_HASH_IPV6_TCP:
13726                 /* fall-through. */
13727         case MLX5_RSS_HASH_IPV6_TCP_DST_ONLY:
13728                 /* fall-through. */
13729         case MLX5_RSS_HASH_IPV6_TCP_SRC_ONLY:
13730                 return hrxqs[4];
13731         case MLX5_RSS_HASH_IPV6_UDP:
13732                 /* fall-through. */
13733         case MLX5_RSS_HASH_IPV6_UDP_DST_ONLY:
13734                 /* fall-through. */
13735         case MLX5_RSS_HASH_IPV6_UDP_SRC_ONLY:
13736                 return hrxqs[5];
13737         case MLX5_RSS_HASH_NONE:
13738                 return hrxqs[6];
13739         default:
13740                 return 0;
13741         }
13742
13743 }
13744
13745 /**
13746  * Apply the flow to the NIC, lock free,
13747  * (mutex should be acquired by caller).
13748  *
13749  * @param[in] dev
13750  *   Pointer to the Ethernet device structure.
13751  * @param[in, out] flow
13752  *   Pointer to flow structure.
13753  * @param[out] error
13754  *   Pointer to error structure.
13755  *
13756  * @return
13757  *   0 on success, a negative errno value otherwise and rte_errno is set.
13758  */
13759 static int
13760 flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
13761               struct rte_flow_error *error)
13762 {
13763         struct mlx5_flow_dv_workspace *dv;
13764         struct mlx5_flow_handle *dh;
13765         struct mlx5_flow_handle_dv *dv_h;
13766         struct mlx5_flow *dev_flow;
13767         struct mlx5_priv *priv = dev->data->dev_private;
13768         uint32_t handle_idx;
13769         int n;
13770         int err;
13771         int idx;
13772         struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
13773         struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc;
13774         uint8_t misc_mask;
13775
13776         MLX5_ASSERT(wks);
13777         for (idx = wks->flow_idx - 1; idx >= 0; idx--) {
13778                 dev_flow = &wks->flows[idx];
13779                 dv = &dev_flow->dv;
13780                 dh = dev_flow->handle;
13781                 dv_h = &dh->dvh;
13782                 n = dv->actions_n;
13783                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
13784                         if (dv->transfer) {
13785                                 MLX5_ASSERT(priv->sh->dr_drop_action);
13786                                 dv->actions[n++] = priv->sh->dr_drop_action;
13787                         } else {
13788 #ifdef HAVE_MLX5DV_DR
13789                                 /* DR supports drop action placeholder. */
13790                                 MLX5_ASSERT(priv->sh->dr_drop_action);
13791                                 dv->actions[n++] = priv->sh->dr_drop_action;
13792 #else
13793                                 /* For DV we use the explicit drop queue. */
13794                                 MLX5_ASSERT(priv->drop_queue.hrxq);
13795                                 dv->actions[n++] =
13796                                                 priv->drop_queue.hrxq->action;
13797 #endif
13798                         }
13799                 } else if ((dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
13800                            !dv_h->rix_sample && !dv_h->rix_dest_array)) {
13801                         struct mlx5_hrxq *hrxq;
13802                         uint32_t hrxq_idx;
13803
13804                         hrxq = flow_dv_hrxq_prepare(dev, dev_flow, rss_desc,
13805                                                     &hrxq_idx);
13806                         if (!hrxq) {
13807                                 rte_flow_error_set
13808                                         (error, rte_errno,
13809                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13810                                          "cannot get hash queue");
13811                                 goto error;
13812                         }
13813                         dh->rix_hrxq = hrxq_idx;
13814                         dv->actions[n++] = hrxq->action;
13815                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
13816                         struct mlx5_hrxq *hrxq = NULL;
13817                         uint32_t hrxq_idx;
13818
13819                         hrxq_idx = __flow_dv_action_rss_hrxq_lookup(dev,
13820                                                 rss_desc->shared_rss,
13821                                                 dev_flow->hash_fields);
13822                         if (hrxq_idx)
13823                                 hrxq = mlx5_ipool_get
13824                                         (priv->sh->ipool[MLX5_IPOOL_HRXQ],
13825                                          hrxq_idx);
13826                         if (!hrxq) {
13827                                 rte_flow_error_set
13828                                         (error, rte_errno,
13829                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13830                                          "cannot get hash queue");
13831                                 goto error;
13832                         }
13833                         dh->rix_srss = rss_desc->shared_rss;
13834                         dv->actions[n++] = hrxq->action;
13835                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
13836                         if (!priv->sh->default_miss_action) {
13837                                 rte_flow_error_set
13838                                         (error, rte_errno,
13839                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
13840                                          "default miss action not be created.");
13841                                 goto error;
13842                         }
13843                         dv->actions[n++] = priv->sh->default_miss_action;
13844                 }
13845                 misc_mask = flow_dv_matcher_enable(dv->value.buf);
13846                 __flow_dv_adjust_buf_size(&dv->value.size, misc_mask);
13847                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
13848                                                (void *)&dv->value, n,
13849                                                dv->actions, &dh->drv_flow);
13850                 if (err) {
13851                         rte_flow_error_set
13852                                 (error, errno,
13853                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
13854                                 NULL,
13855                                 (!priv->config.allow_duplicate_pattern &&
13856                                 errno == EEXIST) ?
13857                                 "duplicating pattern is not allowed" :
13858                                 "hardware refuses to create flow");
13859                         goto error;
13860                 }
13861                 if (priv->vmwa_context &&
13862                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
13863                         /*
13864                          * The rule contains the VLAN pattern.
13865                          * For VF we are going to create VLAN
13866                          * interface to make hypervisor set correct
13867                          * e-Switch vport context.
13868                          */
13869                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
13870                 }
13871         }
13872         return 0;
13873 error:
13874         err = rte_errno; /* Save rte_errno before cleanup. */
13875         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
13876                        handle_idx, dh, next) {
13877                 /* hrxq is union, don't clear it if the flag is not set. */
13878                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && dh->rix_hrxq) {
13879                         mlx5_hrxq_release(dev, dh->rix_hrxq);
13880                         dh->rix_hrxq = 0;
13881                 } else if (dh->fate_action == MLX5_FLOW_FATE_SHARED_RSS) {
13882                         dh->rix_srss = 0;
13883                 }
13884                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
13885                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
13886         }
13887         rte_errno = err; /* Restore rte_errno. */
13888         return -rte_errno;
13889 }
13890
13891 void
13892 flow_dv_matcher_remove_cb(void *tool_ctx __rte_unused,
13893                           struct mlx5_list_entry *entry)
13894 {
13895         struct mlx5_flow_dv_matcher *resource = container_of(entry,
13896                                                              typeof(*resource),
13897                                                              entry);
13898
13899         claim_zero(mlx5_flow_os_destroy_flow_matcher(resource->matcher_object));
13900         mlx5_free(resource);
13901 }
13902
13903 /**
13904  * Release the flow matcher.
13905  *
13906  * @param dev
13907  *   Pointer to Ethernet device.
13908  * @param port_id
13909  *   Index to port ID action resource.
13910  *
13911  * @return
13912  *   1 while a reference on it exists, 0 when freed.
13913  */
13914 static int
13915 flow_dv_matcher_release(struct rte_eth_dev *dev,
13916                         struct mlx5_flow_handle *handle)
13917 {
13918         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
13919         struct mlx5_flow_tbl_data_entry *tbl = container_of(matcher->tbl,
13920                                                             typeof(*tbl), tbl);
13921         int ret;
13922
13923         MLX5_ASSERT(matcher->matcher_object);
13924         ret = mlx5_list_unregister(tbl->matchers, &matcher->entry);
13925         flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
13926         return ret;
13927 }
13928
13929 void
13930 flow_dv_encap_decap_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
13931 {
13932         struct mlx5_dev_ctx_shared *sh = tool_ctx;
13933         struct mlx5_flow_dv_encap_decap_resource *res =
13934                                        container_of(entry, typeof(*res), entry);
13935
13936         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
13937         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
13938 }
13939
13940 /**
13941  * Release an encap/decap resource.
13942  *
13943  * @param dev
13944  *   Pointer to Ethernet device.
13945  * @param encap_decap_idx
13946  *   Index of encap decap resource.
13947  *
13948  * @return
13949  *   1 while a reference on it exists, 0 when freed.
13950  */
13951 static int
13952 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
13953                                      uint32_t encap_decap_idx)
13954 {
13955         struct mlx5_priv *priv = dev->data->dev_private;
13956         struct mlx5_flow_dv_encap_decap_resource *resource;
13957
13958         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
13959                                   encap_decap_idx);
13960         if (!resource)
13961                 return 0;
13962         MLX5_ASSERT(resource->action);
13963         return mlx5_hlist_unregister(priv->sh->encaps_decaps, &resource->entry);
13964 }
13965
13966 /**
13967  * Release an jump to table action resource.
13968  *
13969  * @param dev
13970  *   Pointer to Ethernet device.
13971  * @param rix_jump
13972  *   Index to the jump action resource.
13973  *
13974  * @return
13975  *   1 while a reference on it exists, 0 when freed.
13976  */
13977 static int
13978 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
13979                                   uint32_t rix_jump)
13980 {
13981         struct mlx5_priv *priv = dev->data->dev_private;
13982         struct mlx5_flow_tbl_data_entry *tbl_data;
13983
13984         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
13985                                   rix_jump);
13986         if (!tbl_data)
13987                 return 0;
13988         return flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl_data->tbl);
13989 }
13990
13991 void
13992 flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
13993 {
13994         struct mlx5_flow_dv_modify_hdr_resource *res =
13995                 container_of(entry, typeof(*res), entry);
13996         struct mlx5_dev_ctx_shared *sh = tool_ctx;
13997
13998         claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
13999         mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
14000 }
14001
14002 /**
14003  * Release a modify-header resource.
14004  *
14005  * @param dev
14006  *   Pointer to Ethernet device.
14007  * @param handle
14008  *   Pointer to mlx5_flow_handle.
14009  *
14010  * @return
14011  *   1 while a reference on it exists, 0 when freed.
14012  */
14013 static int
14014 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
14015                                     struct mlx5_flow_handle *handle)
14016 {
14017         struct mlx5_priv *priv = dev->data->dev_private;
14018         struct mlx5_flow_dv_modify_hdr_resource *entry = handle->dvh.modify_hdr;
14019
14020         MLX5_ASSERT(entry->action);
14021         return mlx5_hlist_unregister(priv->sh->modify_cmds, &entry->entry);
14022 }
14023
14024 void
14025 flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
14026 {
14027         struct mlx5_dev_ctx_shared *sh = tool_ctx;
14028         struct mlx5_flow_dv_port_id_action_resource *resource =
14029                                   container_of(entry, typeof(*resource), entry);
14030
14031         claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
14032         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
14033 }
14034
14035 /**
14036  * Release port ID action resource.
14037  *
14038  * @param dev
14039  *   Pointer to Ethernet device.
14040  * @param handle
14041  *   Pointer to mlx5_flow_handle.
14042  *
14043  * @return
14044  *   1 while a reference on it exists, 0 when freed.
14045  */
14046 static int
14047 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
14048                                         uint32_t port_id)
14049 {
14050         struct mlx5_priv *priv = dev->data->dev_private;
14051         struct mlx5_flow_dv_port_id_action_resource *resource;
14052
14053         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
14054         if (!resource)
14055                 return 0;
14056         MLX5_ASSERT(resource->action);
14057         return mlx5_list_unregister(priv->sh->port_id_action_list,
14058                                     &resource->entry);
14059 }
14060
14061 /**
14062  * Release shared RSS action resource.
14063  *
14064  * @param dev
14065  *   Pointer to Ethernet device.
14066  * @param srss
14067  *   Shared RSS action index.
14068  */
14069 static void
14070 flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
14071 {
14072         struct mlx5_priv *priv = dev->data->dev_private;
14073         struct mlx5_shared_action_rss *shared_rss;
14074
14075         shared_rss = mlx5_ipool_get
14076                         (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], srss);
14077         __atomic_sub_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
14078 }
14079
14080 void
14081 flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
14082 {
14083         struct mlx5_dev_ctx_shared *sh = tool_ctx;
14084         struct mlx5_flow_dv_push_vlan_action_resource *resource =
14085                         container_of(entry, typeof(*resource), entry);
14086
14087         claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
14088         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
14089 }
14090
14091 /**
14092  * Release push vlan action resource.
14093  *
14094  * @param dev
14095  *   Pointer to Ethernet device.
14096  * @param handle
14097  *   Pointer to mlx5_flow_handle.
14098  *
14099  * @return
14100  *   1 while a reference on it exists, 0 when freed.
14101  */
14102 static int
14103 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
14104                                           struct mlx5_flow_handle *handle)
14105 {
14106         struct mlx5_priv *priv = dev->data->dev_private;
14107         struct mlx5_flow_dv_push_vlan_action_resource *resource;
14108         uint32_t idx = handle->dvh.rix_push_vlan;
14109
14110         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
14111         if (!resource)
14112                 return 0;
14113         MLX5_ASSERT(resource->action);
14114         return mlx5_list_unregister(priv->sh->push_vlan_action_list,
14115                                     &resource->entry);
14116 }
14117
14118 /**
14119  * Release the fate resource.
14120  *
14121  * @param dev
14122  *   Pointer to Ethernet device.
14123  * @param handle
14124  *   Pointer to mlx5_flow_handle.
14125  */
14126 static void
14127 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
14128                                struct mlx5_flow_handle *handle)
14129 {
14130         if (!handle->rix_fate)
14131                 return;
14132         switch (handle->fate_action) {
14133         case MLX5_FLOW_FATE_QUEUE:
14134                 if (!handle->dvh.rix_sample && !handle->dvh.rix_dest_array)
14135                         mlx5_hrxq_release(dev, handle->rix_hrxq);
14136                 break;
14137         case MLX5_FLOW_FATE_JUMP:
14138                 flow_dv_jump_tbl_resource_release(dev, handle->rix_jump);
14139                 break;
14140         case MLX5_FLOW_FATE_PORT_ID:
14141                 flow_dv_port_id_action_resource_release(dev,
14142                                 handle->rix_port_id_action);
14143                 break;
14144         default:
14145                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
14146                 break;
14147         }
14148         handle->rix_fate = 0;
14149 }
14150
14151 void
14152 flow_dv_sample_remove_cb(void *tool_ctx __rte_unused,
14153                          struct mlx5_list_entry *entry)
14154 {
14155         struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
14156                                                               typeof(*resource),
14157                                                               entry);
14158         struct rte_eth_dev *dev = resource->dev;
14159         struct mlx5_priv *priv = dev->data->dev_private;
14160
14161         if (resource->verbs_action)
14162                 claim_zero(mlx5_flow_os_destroy_flow_action
14163                                                       (resource->verbs_action));
14164         if (resource->normal_path_tbl)
14165                 flow_dv_tbl_resource_release(MLX5_SH(dev),
14166                                              resource->normal_path_tbl);
14167         flow_dv_sample_sub_actions_release(dev, &resource->sample_idx);
14168         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
14169         DRV_LOG(DEBUG, "sample resource %p: removed", (void *)resource);
14170 }
14171
14172 /**
14173  * Release an sample resource.
14174  *
14175  * @param dev
14176  *   Pointer to Ethernet device.
14177  * @param handle
14178  *   Pointer to mlx5_flow_handle.
14179  *
14180  * @return
14181  *   1 while a reference on it exists, 0 when freed.
14182  */
14183 static int
14184 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
14185                                      struct mlx5_flow_handle *handle)
14186 {
14187         struct mlx5_priv *priv = dev->data->dev_private;
14188         struct mlx5_flow_dv_sample_resource *resource;
14189
14190         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
14191                                   handle->dvh.rix_sample);
14192         if (!resource)
14193                 return 0;
14194         MLX5_ASSERT(resource->verbs_action);
14195         return mlx5_list_unregister(priv->sh->sample_action_list,
14196                                     &resource->entry);
14197 }
14198
14199 void
14200 flow_dv_dest_array_remove_cb(void *tool_ctx __rte_unused,
14201                              struct mlx5_list_entry *entry)
14202 {
14203         struct mlx5_flow_dv_dest_array_resource *resource =
14204                         container_of(entry, typeof(*resource), entry);
14205         struct rte_eth_dev *dev = resource->dev;
14206         struct mlx5_priv *priv = dev->data->dev_private;
14207         uint32_t i = 0;
14208
14209         MLX5_ASSERT(resource->action);
14210         if (resource->action)
14211                 claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
14212         for (; i < resource->num_of_dest; i++)
14213                 flow_dv_sample_sub_actions_release(dev,
14214                                                    &resource->sample_idx[i]);
14215         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
14216         DRV_LOG(DEBUG, "destination array resource %p: removed",
14217                 (void *)resource);
14218 }
14219
14220 /**
14221  * Release an destination array resource.
14222  *
14223  * @param dev
14224  *   Pointer to Ethernet device.
14225  * @param handle
14226  *   Pointer to mlx5_flow_handle.
14227  *
14228  * @return
14229  *   1 while a reference on it exists, 0 when freed.
14230  */
14231 static int
14232 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
14233                                     struct mlx5_flow_handle *handle)
14234 {
14235         struct mlx5_priv *priv = dev->data->dev_private;
14236         struct mlx5_flow_dv_dest_array_resource *resource;
14237
14238         resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
14239                                   handle->dvh.rix_dest_array);
14240         if (!resource)
14241                 return 0;
14242         MLX5_ASSERT(resource->action);
14243         return mlx5_list_unregister(priv->sh->dest_array_list,
14244                                     &resource->entry);
14245 }
14246
14247 static void
14248 flow_dv_geneve_tlv_option_resource_release(struct rte_eth_dev *dev)
14249 {
14250         struct mlx5_priv *priv = dev->data->dev_private;
14251         struct mlx5_dev_ctx_shared *sh = priv->sh;
14252         struct mlx5_geneve_tlv_option_resource *geneve_opt_resource =
14253                                 sh->geneve_tlv_option_resource;
14254         rte_spinlock_lock(&sh->geneve_tlv_opt_sl);
14255         if (geneve_opt_resource) {
14256                 if (!(__atomic_sub_fetch(&geneve_opt_resource->refcnt, 1,
14257                                          __ATOMIC_RELAXED))) {
14258                         claim_zero(mlx5_devx_cmd_destroy
14259                                         (geneve_opt_resource->obj));
14260                         mlx5_free(sh->geneve_tlv_option_resource);
14261                         sh->geneve_tlv_option_resource = NULL;
14262                 }
14263         }
14264         rte_spinlock_unlock(&sh->geneve_tlv_opt_sl);
14265 }
14266
14267 /**
14268  * Remove the flow from the NIC but keeps it in memory.
14269  * Lock free, (mutex should be acquired by caller).
14270  *
14271  * @param[in] dev
14272  *   Pointer to Ethernet device.
14273  * @param[in, out] flow
14274  *   Pointer to flow structure.
14275  */
14276 static void
14277 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
14278 {
14279         struct mlx5_flow_handle *dh;
14280         uint32_t handle_idx;
14281         struct mlx5_priv *priv = dev->data->dev_private;
14282
14283         if (!flow)
14284                 return;
14285         handle_idx = flow->dev_handles;
14286         while (handle_idx) {
14287                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
14288                                     handle_idx);
14289                 if (!dh)
14290                         return;
14291                 if (dh->drv_flow) {
14292                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
14293                         dh->drv_flow = NULL;
14294                 }
14295                 if (dh->fate_action == MLX5_FLOW_FATE_QUEUE)
14296                         flow_dv_fate_resource_release(dev, dh);
14297                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
14298                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
14299                 handle_idx = dh->next.next;
14300         }
14301 }
14302
14303 /**
14304  * Remove the flow from the NIC and the memory.
14305  * Lock free, (mutex should be acquired by caller).
14306  *
14307  * @param[in] dev
14308  *   Pointer to the Ethernet device structure.
14309  * @param[in, out] flow
14310  *   Pointer to flow structure.
14311  */
14312 static void
14313 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
14314 {
14315         struct mlx5_flow_handle *dev_handle;
14316         struct mlx5_priv *priv = dev->data->dev_private;
14317         struct mlx5_flow_meter_info *fm = NULL;
14318         uint32_t srss = 0;
14319
14320         if (!flow)
14321                 return;
14322         flow_dv_remove(dev, flow);
14323         if (flow->counter) {
14324                 flow_dv_counter_free(dev, flow->counter);
14325                 flow->counter = 0;
14326         }
14327         if (flow->meter) {
14328                 fm = flow_dv_meter_find_by_idx(priv, flow->meter);
14329                 if (fm)
14330                         mlx5_flow_meter_detach(priv, fm);
14331                 flow->meter = 0;
14332         }
14333         /* Keep the current age handling by default. */
14334         if (flow->indirect_type == MLX5_INDIRECT_ACTION_TYPE_CT && flow->ct)
14335                 flow_dv_aso_ct_release(dev, flow->ct);
14336         else if (flow->age)
14337                 flow_dv_aso_age_release(dev, flow->age);
14338         if (flow->geneve_tlv_option) {
14339                 flow_dv_geneve_tlv_option_resource_release(dev);
14340                 flow->geneve_tlv_option = 0;
14341         }
14342         while (flow->dev_handles) {
14343                 uint32_t tmp_idx = flow->dev_handles;
14344
14345                 dev_handle = mlx5_ipool_get(priv->sh->ipool
14346                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
14347                 if (!dev_handle)
14348                         return;
14349                 flow->dev_handles = dev_handle->next.next;
14350                 if (dev_handle->dvh.matcher)
14351                         flow_dv_matcher_release(dev, dev_handle);
14352                 if (dev_handle->dvh.rix_sample)
14353                         flow_dv_sample_resource_release(dev, dev_handle);
14354                 if (dev_handle->dvh.rix_dest_array)
14355                         flow_dv_dest_array_resource_release(dev, dev_handle);
14356                 if (dev_handle->dvh.rix_encap_decap)
14357                         flow_dv_encap_decap_resource_release(dev,
14358                                 dev_handle->dvh.rix_encap_decap);
14359                 if (dev_handle->dvh.modify_hdr)
14360                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
14361                 if (dev_handle->dvh.rix_push_vlan)
14362                         flow_dv_push_vlan_action_resource_release(dev,
14363                                                                   dev_handle);
14364                 if (dev_handle->dvh.rix_tag)
14365                         flow_dv_tag_release(dev,
14366                                             dev_handle->dvh.rix_tag);
14367                 if (dev_handle->fate_action != MLX5_FLOW_FATE_SHARED_RSS)
14368                         flow_dv_fate_resource_release(dev, dev_handle);
14369                 else if (!srss)
14370                         srss = dev_handle->rix_srss;
14371                 if (fm && dev_handle->is_meter_flow_id &&
14372                     dev_handle->split_flow_id)
14373                         mlx5_ipool_free(fm->flow_ipool,
14374                                         dev_handle->split_flow_id);
14375                 else if (dev_handle->split_flow_id &&
14376                     !dev_handle->is_meter_flow_id)
14377                         mlx5_ipool_free(priv->sh->ipool
14378                                         [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
14379                                         dev_handle->split_flow_id);
14380                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
14381                            tmp_idx);
14382         }
14383         if (srss)
14384                 flow_dv_shared_rss_action_release(dev, srss);
14385 }
14386
14387 /**
14388  * Release array of hash RX queue objects.
14389  * Helper function.
14390  *
14391  * @param[in] dev
14392  *   Pointer to the Ethernet device structure.
14393  * @param[in, out] hrxqs
14394  *   Array of hash RX queue objects.
14395  *
14396  * @return
14397  *   Total number of references to hash RX queue objects in *hrxqs* array
14398  *   after this operation.
14399  */
14400 static int
14401 __flow_dv_hrxqs_release(struct rte_eth_dev *dev,
14402                         uint32_t (*hrxqs)[MLX5_RSS_HASH_FIELDS_LEN])
14403 {
14404         size_t i;
14405         int remaining = 0;
14406
14407         for (i = 0; i < RTE_DIM(*hrxqs); i++) {
14408                 int ret = mlx5_hrxq_release(dev, (*hrxqs)[i]);
14409
14410                 if (!ret)
14411                         (*hrxqs)[i] = 0;
14412                 remaining += ret;
14413         }
14414         return remaining;
14415 }
14416
14417 /**
14418  * Release all hash RX queue objects representing shared RSS action.
14419  *
14420  * @param[in] dev
14421  *   Pointer to the Ethernet device structure.
14422  * @param[in, out] action
14423  *   Shared RSS action to remove hash RX queue objects from.
14424  *
14425  * @return
14426  *   Total number of references to hash RX queue objects stored in *action*
14427  *   after this operation.
14428  *   Expected to be 0 if no external references held.
14429  */
14430 static int
14431 __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev,
14432                                  struct mlx5_shared_action_rss *shared_rss)
14433 {
14434         return __flow_dv_hrxqs_release(dev, &shared_rss->hrxq);
14435 }
14436
14437 /**
14438  * Adjust L3/L4 hash value of pre-created shared RSS hrxq according to
14439  * user input.
14440  *
14441  * Only one hash value is available for one L3+L4 combination:
14442  * for example:
14443  * MLX5_RSS_HASH_IPV4, MLX5_RSS_HASH_IPV4_SRC_ONLY, and
14444  * MLX5_RSS_HASH_IPV4_DST_ONLY are mutually exclusive so they can share
14445  * same slot in mlx5_rss_hash_fields.
14446  *
14447  * @param[in] rss
14448  *   Pointer to the shared action RSS conf.
14449  * @param[in, out] hash_field
14450  *   hash_field variable needed to be adjusted.
14451  *
14452  * @return
14453  *   void
14454  */
14455 static void
14456 __flow_dv_action_rss_l34_hash_adjust(struct mlx5_shared_action_rss *rss,
14457                                      uint64_t *hash_field)
14458 {
14459         uint64_t rss_types = rss->origin.types;
14460
14461         switch (*hash_field & ~IBV_RX_HASH_INNER) {
14462         case MLX5_RSS_HASH_IPV4:
14463                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
14464                         *hash_field &= ~MLX5_RSS_HASH_IPV4;
14465                         if (rss_types & ETH_RSS_L3_DST_ONLY)
14466                                 *hash_field |= IBV_RX_HASH_DST_IPV4;
14467                         else if (rss_types & ETH_RSS_L3_SRC_ONLY)
14468                                 *hash_field |= IBV_RX_HASH_SRC_IPV4;
14469                         else
14470                                 *hash_field |= MLX5_RSS_HASH_IPV4;
14471                 }
14472                 return;
14473         case MLX5_RSS_HASH_IPV6:
14474                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
14475                         *hash_field &= ~MLX5_RSS_HASH_IPV6;
14476                         if (rss_types & ETH_RSS_L3_DST_ONLY)
14477                                 *hash_field |= IBV_RX_HASH_DST_IPV6;
14478                         else if (rss_types & ETH_RSS_L3_SRC_ONLY)
14479                                 *hash_field |= IBV_RX_HASH_SRC_IPV6;
14480                         else
14481                                 *hash_field |= MLX5_RSS_HASH_IPV6;
14482                 }
14483                 return;
14484         case MLX5_RSS_HASH_IPV4_UDP:
14485                 /* fall-through. */
14486         case MLX5_RSS_HASH_IPV6_UDP:
14487                 if (rss_types & ETH_RSS_UDP) {
14488                         *hash_field &= ~MLX5_UDP_IBV_RX_HASH;
14489                         if (rss_types & ETH_RSS_L4_DST_ONLY)
14490                                 *hash_field |= IBV_RX_HASH_DST_PORT_UDP;
14491                         else if (rss_types & ETH_RSS_L4_SRC_ONLY)
14492                                 *hash_field |= IBV_RX_HASH_SRC_PORT_UDP;
14493                         else
14494                                 *hash_field |= MLX5_UDP_IBV_RX_HASH;
14495                 }
14496                 return;
14497         case MLX5_RSS_HASH_IPV4_TCP:
14498                 /* fall-through. */
14499         case MLX5_RSS_HASH_IPV6_TCP:
14500                 if (rss_types & ETH_RSS_TCP) {
14501                         *hash_field &= ~MLX5_TCP_IBV_RX_HASH;
14502                         if (rss_types & ETH_RSS_L4_DST_ONLY)
14503                                 *hash_field |= IBV_RX_HASH_DST_PORT_TCP;
14504                         else if (rss_types & ETH_RSS_L4_SRC_ONLY)
14505                                 *hash_field |= IBV_RX_HASH_SRC_PORT_TCP;
14506                         else
14507                                 *hash_field |= MLX5_TCP_IBV_RX_HASH;
14508                 }
14509                 return;
14510         default:
14511                 return;
14512         }
14513 }
14514
14515 /**
14516  * Setup shared RSS action.
14517  * Prepare set of hash RX queue objects sufficient to handle all valid
14518  * hash_fields combinations (see enum ibv_rx_hash_fields).
14519  *
14520  * @param[in] dev
14521  *   Pointer to the Ethernet device structure.
14522  * @param[in] action_idx
14523  *   Shared RSS action ipool index.
14524  * @param[in, out] action
14525  *   Partially initialized shared RSS action.
14526  * @param[out] error
14527  *   Perform verbose error reporting if not NULL. Initialized in case of
14528  *   error only.
14529  *
14530  * @return
14531  *   0 on success, otherwise negative errno value.
14532  */
14533 static int
14534 __flow_dv_action_rss_setup(struct rte_eth_dev *dev,
14535                            uint32_t action_idx,
14536                            struct mlx5_shared_action_rss *shared_rss,
14537                            struct rte_flow_error *error)
14538 {
14539         struct mlx5_flow_rss_desc rss_desc = { 0 };
14540         size_t i;
14541         int err;
14542
14543         if (mlx5_ind_table_obj_setup(dev, shared_rss->ind_tbl)) {
14544                 return rte_flow_error_set(error, rte_errno,
14545                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14546                                           "cannot setup indirection table");
14547         }
14548         memcpy(rss_desc.key, shared_rss->origin.key, MLX5_RSS_HASH_KEY_LEN);
14549         rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN;
14550         rss_desc.const_q = shared_rss->origin.queue;
14551         rss_desc.queue_num = shared_rss->origin.queue_num;
14552         /* Set non-zero value to indicate a shared RSS. */
14553         rss_desc.shared_rss = action_idx;
14554         rss_desc.ind_tbl = shared_rss->ind_tbl;
14555         for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) {
14556                 uint32_t hrxq_idx;
14557                 uint64_t hash_fields = mlx5_rss_hash_fields[i];
14558                 int tunnel = 0;
14559
14560                 __flow_dv_action_rss_l34_hash_adjust(shared_rss, &hash_fields);
14561                 if (shared_rss->origin.level > 1) {
14562                         hash_fields |= IBV_RX_HASH_INNER;
14563                         tunnel = 1;
14564                 }
14565                 rss_desc.tunnel = tunnel;
14566                 rss_desc.hash_fields = hash_fields;
14567                 hrxq_idx = mlx5_hrxq_get(dev, &rss_desc);
14568                 if (!hrxq_idx) {
14569                         rte_flow_error_set
14570                                 (error, rte_errno,
14571                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14572                                  "cannot get hash queue");
14573                         goto error_hrxq_new;
14574                 }
14575                 err = __flow_dv_action_rss_hrxq_set
14576                         (shared_rss, hash_fields, hrxq_idx);
14577                 MLX5_ASSERT(!err);
14578         }
14579         return 0;
14580 error_hrxq_new:
14581         err = rte_errno;
14582         __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
14583         if (!mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true))
14584                 shared_rss->ind_tbl = NULL;
14585         rte_errno = err;
14586         return -rte_errno;
14587 }
14588
14589 /**
14590  * Create shared RSS action.
14591  *
14592  * @param[in] dev
14593  *   Pointer to the Ethernet device structure.
14594  * @param[in] conf
14595  *   Shared action configuration.
14596  * @param[in] rss
14597  *   RSS action specification used to create shared action.
14598  * @param[out] error
14599  *   Perform verbose error reporting if not NULL. Initialized in case of
14600  *   error only.
14601  *
14602  * @return
14603  *   A valid shared action ID in case of success, 0 otherwise and
14604  *   rte_errno is set.
14605  */
14606 static uint32_t
14607 __flow_dv_action_rss_create(struct rte_eth_dev *dev,
14608                             const struct rte_flow_indir_action_conf *conf,
14609                             const struct rte_flow_action_rss *rss,
14610                             struct rte_flow_error *error)
14611 {
14612         struct mlx5_priv *priv = dev->data->dev_private;
14613         struct mlx5_shared_action_rss *shared_rss = NULL;
14614         void *queue = NULL;
14615         struct rte_flow_action_rss *origin;
14616         const uint8_t *rss_key;
14617         uint32_t queue_size = rss->queue_num * sizeof(uint16_t);
14618         uint32_t idx;
14619
14620         RTE_SET_USED(conf);
14621         queue = mlx5_malloc(0, RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
14622                             0, SOCKET_ID_ANY);
14623         shared_rss = mlx5_ipool_zmalloc
14624                          (priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], &idx);
14625         if (!shared_rss || !queue) {
14626                 rte_flow_error_set(error, ENOMEM,
14627                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14628                                    "cannot allocate resource memory");
14629                 goto error_rss_init;
14630         }
14631         if (idx > (1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET)) {
14632                 rte_flow_error_set(error, E2BIG,
14633                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14634                                    "rss action number out of range");
14635                 goto error_rss_init;
14636         }
14637         shared_rss->ind_tbl = mlx5_malloc(MLX5_MEM_ZERO,
14638                                           sizeof(*shared_rss->ind_tbl),
14639                                           0, SOCKET_ID_ANY);
14640         if (!shared_rss->ind_tbl) {
14641                 rte_flow_error_set(error, ENOMEM,
14642                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
14643                                    "cannot allocate resource memory");
14644                 goto error_rss_init;
14645         }
14646         memcpy(queue, rss->queue, queue_size);
14647         shared_rss->ind_tbl->queues = queue;
14648         shared_rss->ind_tbl->queues_n = rss->queue_num;
14649         origin = &shared_rss->origin;
14650         origin->func = rss->func;
14651         origin->level = rss->level;
14652         /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
14653         origin->types = !rss->types ? ETH_RSS_IP : rss->types;
14654         /* NULL RSS key indicates default RSS key. */
14655         rss_key = !rss->key ? rss_hash_default_key : rss->key;
14656         memcpy(shared_rss->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
14657         origin->key = &shared_rss->key[0];
14658         origin->key_len = MLX5_RSS_HASH_KEY_LEN;
14659         origin->queue = queue;
14660         origin->queue_num = rss->queue_num;
14661         if (__flow_dv_action_rss_setup(dev, idx, shared_rss, error))
14662                 goto error_rss_init;
14663         rte_spinlock_init(&shared_rss->action_rss_sl);
14664         __atomic_add_fetch(&shared_rss->refcnt, 1, __ATOMIC_RELAXED);
14665         rte_spinlock_lock(&priv->shared_act_sl);
14666         ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14667                      &priv->rss_shared_actions, idx, shared_rss, next);
14668         rte_spinlock_unlock(&priv->shared_act_sl);
14669         return idx;
14670 error_rss_init:
14671         if (shared_rss) {
14672                 if (shared_rss->ind_tbl)
14673                         mlx5_free(shared_rss->ind_tbl);
14674                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14675                                 idx);
14676         }
14677         if (queue)
14678                 mlx5_free(queue);
14679         return 0;
14680 }
14681
14682 /**
14683  * Destroy the shared RSS action.
14684  * Release related hash RX queue objects.
14685  *
14686  * @param[in] dev
14687  *   Pointer to the Ethernet device structure.
14688  * @param[in] idx
14689  *   The shared RSS action object ID to be removed.
14690  * @param[out] error
14691  *   Perform verbose error reporting if not NULL. Initialized in case of
14692  *   error only.
14693  *
14694  * @return
14695  *   0 on success, otherwise negative errno value.
14696  */
14697 static int
14698 __flow_dv_action_rss_release(struct rte_eth_dev *dev, uint32_t idx,
14699                              struct rte_flow_error *error)
14700 {
14701         struct mlx5_priv *priv = dev->data->dev_private;
14702         struct mlx5_shared_action_rss *shared_rss =
14703             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
14704         uint32_t old_refcnt = 1;
14705         int remaining;
14706         uint16_t *queue = NULL;
14707
14708         if (!shared_rss)
14709                 return rte_flow_error_set(error, EINVAL,
14710                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14711                                           "invalid shared action");
14712         remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
14713         if (remaining)
14714                 return rte_flow_error_set(error, EBUSY,
14715                                           RTE_FLOW_ERROR_TYPE_ACTION,
14716                                           NULL,
14717                                           "shared rss hrxq has references");
14718         if (!__atomic_compare_exchange_n(&shared_rss->refcnt, &old_refcnt,
14719                                          0, 0, __ATOMIC_ACQUIRE,
14720                                          __ATOMIC_RELAXED))
14721                 return rte_flow_error_set(error, EBUSY,
14722                                           RTE_FLOW_ERROR_TYPE_ACTION,
14723                                           NULL,
14724                                           "shared rss has references");
14725         queue = shared_rss->ind_tbl->queues;
14726         remaining = mlx5_ind_table_obj_release(dev, shared_rss->ind_tbl, true);
14727         if (remaining)
14728                 return rte_flow_error_set(error, EBUSY,
14729                                           RTE_FLOW_ERROR_TYPE_ACTION,
14730                                           NULL,
14731                                           "shared rss indirection table has"
14732                                           " references");
14733         mlx5_free(queue);
14734         rte_spinlock_lock(&priv->shared_act_sl);
14735         ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14736                      &priv->rss_shared_actions, idx, shared_rss, next);
14737         rte_spinlock_unlock(&priv->shared_act_sl);
14738         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
14739                         idx);
14740         return 0;
14741 }
14742
14743 /**
14744  * Create indirect action, lock free,
14745  * (mutex should be acquired by caller).
14746  * Dispatcher for action type specific call.
14747  *
14748  * @param[in] dev
14749  *   Pointer to the Ethernet device structure.
14750  * @param[in] conf
14751  *   Shared action configuration.
14752  * @param[in] action
14753  *   Action specification used to create indirect action.
14754  * @param[out] error
14755  *   Perform verbose error reporting if not NULL. Initialized in case of
14756  *   error only.
14757  *
14758  * @return
14759  *   A valid shared action handle in case of success, NULL otherwise and
14760  *   rte_errno is set.
14761  */
14762 static struct rte_flow_action_handle *
14763 flow_dv_action_create(struct rte_eth_dev *dev,
14764                       const struct rte_flow_indir_action_conf *conf,
14765                       const struct rte_flow_action *action,
14766                       struct rte_flow_error *err)
14767 {
14768         struct mlx5_priv *priv = dev->data->dev_private;
14769         uint32_t age_idx = 0;
14770         uint32_t idx = 0;
14771         uint32_t ret = 0;
14772
14773         switch (action->type) {
14774         case RTE_FLOW_ACTION_TYPE_RSS:
14775                 ret = __flow_dv_action_rss_create(dev, conf, action->conf, err);
14776                 idx = (MLX5_INDIRECT_ACTION_TYPE_RSS <<
14777                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
14778                 break;
14779         case RTE_FLOW_ACTION_TYPE_AGE:
14780                 age_idx = flow_dv_aso_age_alloc(dev, err);
14781                 if (!age_idx) {
14782                         ret = -rte_errno;
14783                         break;
14784                 }
14785                 idx = (MLX5_INDIRECT_ACTION_TYPE_AGE <<
14786                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | age_idx;
14787                 flow_dv_aso_age_params_init(dev, age_idx,
14788                                         ((const struct rte_flow_action_age *)
14789                                                 action->conf)->context ?
14790                                         ((const struct rte_flow_action_age *)
14791                                                 action->conf)->context :
14792                                         (void *)(uintptr_t)idx,
14793                                         ((const struct rte_flow_action_age *)
14794                                                 action->conf)->timeout);
14795                 ret = age_idx;
14796                 break;
14797         case RTE_FLOW_ACTION_TYPE_COUNT:
14798                 ret = flow_dv_translate_create_counter(dev, NULL, NULL, NULL);
14799                 idx = (MLX5_INDIRECT_ACTION_TYPE_COUNT <<
14800                        MLX5_INDIRECT_ACTION_TYPE_OFFSET) | ret;
14801                 break;
14802         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
14803                 ret = flow_dv_translate_create_conntrack(dev, action->conf,
14804                                                          err);
14805                 idx = MLX5_INDIRECT_ACT_CT_GEN_IDX(PORT_ID(priv), ret);
14806                 break;
14807         default:
14808                 rte_flow_error_set(err, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
14809                                    NULL, "action type not supported");
14810                 break;
14811         }
14812         return ret ? (struct rte_flow_action_handle *)(uintptr_t)idx : NULL;
14813 }
14814
14815 /**
14816  * Destroy the indirect action.
14817  * Release action related resources on the NIC and the memory.
14818  * Lock free, (mutex should be acquired by caller).
14819  * Dispatcher for action type specific call.
14820  *
14821  * @param[in] dev
14822  *   Pointer to the Ethernet device structure.
14823  * @param[in] handle
14824  *   The indirect action object handle to be removed.
14825  * @param[out] error
14826  *   Perform verbose error reporting if not NULL. Initialized in case of
14827  *   error only.
14828  *
14829  * @return
14830  *   0 on success, otherwise negative errno value.
14831  */
14832 static int
14833 flow_dv_action_destroy(struct rte_eth_dev *dev,
14834                        struct rte_flow_action_handle *handle,
14835                        struct rte_flow_error *error)
14836 {
14837         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
14838         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
14839         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
14840         struct mlx5_flow_counter *cnt;
14841         uint32_t no_flow_refcnt = 1;
14842         int ret;
14843
14844         switch (type) {
14845         case MLX5_INDIRECT_ACTION_TYPE_RSS:
14846                 return __flow_dv_action_rss_release(dev, idx, error);
14847         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
14848                 cnt = flow_dv_counter_get_by_idx(dev, idx, NULL);
14849                 if (!__atomic_compare_exchange_n(&cnt->shared_info.refcnt,
14850                                                  &no_flow_refcnt, 1, false,
14851                                                  __ATOMIC_ACQUIRE,
14852                                                  __ATOMIC_RELAXED))
14853                         return rte_flow_error_set(error, EBUSY,
14854                                                   RTE_FLOW_ERROR_TYPE_ACTION,
14855                                                   NULL,
14856                                                   "Indirect count action has references");
14857                 flow_dv_counter_free(dev, idx);
14858                 return 0;
14859         case MLX5_INDIRECT_ACTION_TYPE_AGE:
14860                 ret = flow_dv_aso_age_release(dev, idx);
14861                 if (ret)
14862                         /*
14863                          * In this case, the last flow has a reference will
14864                          * actually release the age action.
14865                          */
14866                         DRV_LOG(DEBUG, "Indirect age action %" PRIu32 " was"
14867                                 " released with references %d.", idx, ret);
14868                 return 0;
14869         case MLX5_INDIRECT_ACTION_TYPE_CT:
14870                 ret = flow_dv_aso_ct_release(dev, idx);
14871                 if (ret < 0)
14872                         return ret;
14873                 if (ret > 0)
14874                         DRV_LOG(DEBUG, "Connection tracking object %u still "
14875                                 "has references %d.", idx, ret);
14876                 return 0;
14877         default:
14878                 return rte_flow_error_set(error, ENOTSUP,
14879                                           RTE_FLOW_ERROR_TYPE_ACTION,
14880                                           NULL,
14881                                           "action type not supported");
14882         }
14883 }
14884
14885 /**
14886  * Updates in place shared RSS action configuration.
14887  *
14888  * @param[in] dev
14889  *   Pointer to the Ethernet device structure.
14890  * @param[in] idx
14891  *   The shared RSS action object ID to be updated.
14892  * @param[in] action_conf
14893  *   RSS action specification used to modify *shared_rss*.
14894  * @param[out] error
14895  *   Perform verbose error reporting if not NULL. Initialized in case of
14896  *   error only.
14897  *
14898  * @return
14899  *   0 on success, otherwise negative errno value.
14900  * @note: currently only support update of RSS queues.
14901  */
14902 static int
14903 __flow_dv_action_rss_update(struct rte_eth_dev *dev, uint32_t idx,
14904                             const struct rte_flow_action_rss *action_conf,
14905                             struct rte_flow_error *error)
14906 {
14907         struct mlx5_priv *priv = dev->data->dev_private;
14908         struct mlx5_shared_action_rss *shared_rss =
14909             mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
14910         int ret = 0;
14911         void *queue = NULL;
14912         uint16_t *queue_old = NULL;
14913         uint32_t queue_size = action_conf->queue_num * sizeof(uint16_t);
14914
14915         if (!shared_rss)
14916                 return rte_flow_error_set(error, EINVAL,
14917                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14918                                           "invalid shared action to update");
14919         if (priv->obj_ops.ind_table_modify == NULL)
14920                 return rte_flow_error_set(error, ENOTSUP,
14921                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14922                                           "cannot modify indirection table");
14923         queue = mlx5_malloc(MLX5_MEM_ZERO,
14924                             RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
14925                             0, SOCKET_ID_ANY);
14926         if (!queue)
14927                 return rte_flow_error_set(error, ENOMEM,
14928                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14929                                           NULL,
14930                                           "cannot allocate resource memory");
14931         memcpy(queue, action_conf->queue, queue_size);
14932         MLX5_ASSERT(shared_rss->ind_tbl);
14933         rte_spinlock_lock(&shared_rss->action_rss_sl);
14934         queue_old = shared_rss->ind_tbl->queues;
14935         ret = mlx5_ind_table_obj_modify(dev, shared_rss->ind_tbl,
14936                                         queue, action_conf->queue_num, true);
14937         if (ret) {
14938                 mlx5_free(queue);
14939                 ret = rte_flow_error_set(error, rte_errno,
14940                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
14941                                           "cannot update indirection table");
14942         } else {
14943                 mlx5_free(queue_old);
14944                 shared_rss->origin.queue = queue;
14945                 shared_rss->origin.queue_num = action_conf->queue_num;
14946         }
14947         rte_spinlock_unlock(&shared_rss->action_rss_sl);
14948         return ret;
14949 }
14950
14951 /*
14952  * Updates in place conntrack context or direction.
14953  * Context update should be synchronized.
14954  *
14955  * @param[in] dev
14956  *   Pointer to the Ethernet device structure.
14957  * @param[in] idx
14958  *   The conntrack object ID to be updated.
14959  * @param[in] update
14960  *   Pointer to the structure of information to update.
14961  * @param[out] error
14962  *   Perform verbose error reporting if not NULL. Initialized in case of
14963  *   error only.
14964  *
14965  * @return
14966  *   0 on success, otherwise negative errno value.
14967  */
14968 static int
14969 __flow_dv_action_ct_update(struct rte_eth_dev *dev, uint32_t idx,
14970                            const struct rte_flow_modify_conntrack *update,
14971                            struct rte_flow_error *error)
14972 {
14973         struct mlx5_priv *priv = dev->data->dev_private;
14974         struct mlx5_aso_ct_action *ct;
14975         const struct rte_flow_action_conntrack *new_prf;
14976         int ret = 0;
14977         uint16_t owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
14978         uint32_t dev_idx;
14979
14980         if (PORT_ID(priv) != owner)
14981                 return rte_flow_error_set(error, EACCES,
14982                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14983                                           NULL,
14984                                           "CT object owned by another port");
14985         dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
14986         ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
14987         if (!ct->refcnt)
14988                 return rte_flow_error_set(error, ENOMEM,
14989                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
14990                                           NULL,
14991                                           "CT object is inactive");
14992         new_prf = &update->new_ct;
14993         if (update->direction)
14994                 ct->is_original = !!new_prf->is_original_dir;
14995         if (update->state) {
14996                 /* Only validate the profile when it needs to be updated. */
14997                 ret = mlx5_validate_action_ct(dev, new_prf, error);
14998                 if (ret)
14999                         return ret;
15000                 ret = mlx5_aso_ct_update_by_wqe(priv->sh, ct, new_prf);
15001                 if (ret)
15002                         return rte_flow_error_set(error, EIO,
15003                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15004                                         NULL,
15005                                         "Failed to send CT context update WQE");
15006                 /* Block until ready or a failure. */
15007                 ret = mlx5_aso_ct_available(priv->sh, ct);
15008                 if (ret)
15009                         rte_flow_error_set(error, rte_errno,
15010                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15011                                            NULL,
15012                                            "Timeout to get the CT update");
15013         }
15014         return ret;
15015 }
15016
15017 /**
15018  * Updates in place shared action configuration, lock free,
15019  * (mutex should be acquired by caller).
15020  *
15021  * @param[in] dev
15022  *   Pointer to the Ethernet device structure.
15023  * @param[in] handle
15024  *   The indirect action object handle to be updated.
15025  * @param[in] update
15026  *   Action specification used to modify the action pointed by *handle*.
15027  *   *update* could be of same type with the action pointed by the *handle*
15028  *   handle argument, or some other structures like a wrapper, depending on
15029  *   the indirect action type.
15030  * @param[out] error
15031  *   Perform verbose error reporting if not NULL. Initialized in case of
15032  *   error only.
15033  *
15034  * @return
15035  *   0 on success, otherwise negative errno value.
15036  */
15037 static int
15038 flow_dv_action_update(struct rte_eth_dev *dev,
15039                         struct rte_flow_action_handle *handle,
15040                         const void *update,
15041                         struct rte_flow_error *err)
15042 {
15043         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
15044         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
15045         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
15046         const void *action_conf;
15047
15048         switch (type) {
15049         case MLX5_INDIRECT_ACTION_TYPE_RSS:
15050                 action_conf = ((const struct rte_flow_action *)update)->conf;
15051                 return __flow_dv_action_rss_update(dev, idx, action_conf, err);
15052         case MLX5_INDIRECT_ACTION_TYPE_CT:
15053                 return __flow_dv_action_ct_update(dev, idx, update, err);
15054         default:
15055                 return rte_flow_error_set(err, ENOTSUP,
15056                                           RTE_FLOW_ERROR_TYPE_ACTION,
15057                                           NULL,
15058                                           "action type update not supported");
15059         }
15060 }
15061
15062 /**
15063  * Destroy the meter sub policy table rules.
15064  * Lock free, (mutex should be acquired by caller).
15065  *
15066  * @param[in] dev
15067  *   Pointer to Ethernet device.
15068  * @param[in] sub_policy
15069  *   Pointer to meter sub policy table.
15070  */
15071 static void
15072 __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
15073                              struct mlx5_flow_meter_sub_policy *sub_policy)
15074 {
15075         struct mlx5_priv *priv = dev->data->dev_private;
15076         struct mlx5_flow_tbl_data_entry *tbl;
15077         struct mlx5_flow_meter_policy *policy = sub_policy->main_policy;
15078         struct mlx5_flow_meter_info *next_fm;
15079         struct mlx5_sub_policy_color_rule *color_rule;
15080         void *tmp;
15081         uint32_t i;
15082
15083         for (i = 0; i < RTE_COLORS; i++) {
15084                 next_fm = NULL;
15085                 if (i == RTE_COLOR_GREEN && policy &&
15086                     policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR)
15087                         next_fm = mlx5_flow_meter_find(priv,
15088                                         policy->act_cnt[i].next_mtr_id, NULL);
15089                 TAILQ_FOREACH_SAFE(color_rule, &sub_policy->color_rules[i],
15090                                    next_port, tmp) {
15091                         claim_zero(mlx5_flow_os_destroy_flow(color_rule->rule));
15092                         tbl = container_of(color_rule->matcher->tbl,
15093                                            typeof(*tbl), tbl);
15094                         mlx5_list_unregister(tbl->matchers,
15095                                              &color_rule->matcher->entry);
15096                         TAILQ_REMOVE(&sub_policy->color_rules[i],
15097                                      color_rule, next_port);
15098                         mlx5_free(color_rule);
15099                         if (next_fm)
15100                                 mlx5_flow_meter_detach(priv, next_fm);
15101                 }
15102         }
15103         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
15104                 if (sub_policy->rix_hrxq[i]) {
15105                         if (policy && !policy->is_hierarchy)
15106                                 mlx5_hrxq_release(dev, sub_policy->rix_hrxq[i]);
15107                         sub_policy->rix_hrxq[i] = 0;
15108                 }
15109                 if (sub_policy->jump_tbl[i]) {
15110                         flow_dv_tbl_resource_release(MLX5_SH(dev),
15111                                                      sub_policy->jump_tbl[i]);
15112                         sub_policy->jump_tbl[i] = NULL;
15113                 }
15114         }
15115         if (sub_policy->tbl_rsc) {
15116                 flow_dv_tbl_resource_release(MLX5_SH(dev),
15117                                              sub_policy->tbl_rsc);
15118                 sub_policy->tbl_rsc = NULL;
15119         }
15120 }
15121
15122 /**
15123  * Destroy policy rules, lock free,
15124  * (mutex should be acquired by caller).
15125  * Dispatcher for action type specific call.
15126  *
15127  * @param[in] dev
15128  *   Pointer to the Ethernet device structure.
15129  * @param[in] mtr_policy
15130  *   Meter policy struct.
15131  */
15132 static void
15133 flow_dv_destroy_policy_rules(struct rte_eth_dev *dev,
15134                              struct mlx5_flow_meter_policy *mtr_policy)
15135 {
15136         uint32_t i, j;
15137         struct mlx5_flow_meter_sub_policy *sub_policy;
15138         uint16_t sub_policy_num;
15139
15140         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15141                 sub_policy_num = (mtr_policy->sub_policy_num >>
15142                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
15143                         MLX5_MTR_SUB_POLICY_NUM_MASK;
15144                 for (j = 0; j < sub_policy_num; j++) {
15145                         sub_policy = mtr_policy->sub_policys[i][j];
15146                         if (sub_policy)
15147                                 __flow_dv_destroy_sub_policy_rules(dev,
15148                                                                    sub_policy);
15149                 }
15150         }
15151 }
15152
15153 /**
15154  * Destroy policy action, lock free,
15155  * (mutex should be acquired by caller).
15156  * Dispatcher for action type specific call.
15157  *
15158  * @param[in] dev
15159  *   Pointer to the Ethernet device structure.
15160  * @param[in] mtr_policy
15161  *   Meter policy struct.
15162  */
15163 static void
15164 flow_dv_destroy_mtr_policy_acts(struct rte_eth_dev *dev,
15165                       struct mlx5_flow_meter_policy *mtr_policy)
15166 {
15167         struct rte_flow_action *rss_action;
15168         struct mlx5_flow_handle dev_handle;
15169         uint32_t i, j;
15170
15171         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
15172                 if (mtr_policy->act_cnt[i].rix_mark) {
15173                         flow_dv_tag_release(dev,
15174                                 mtr_policy->act_cnt[i].rix_mark);
15175                         mtr_policy->act_cnt[i].rix_mark = 0;
15176                 }
15177                 if (mtr_policy->act_cnt[i].modify_hdr) {
15178                         dev_handle.dvh.modify_hdr =
15179                                 mtr_policy->act_cnt[i].modify_hdr;
15180                         flow_dv_modify_hdr_resource_release(dev, &dev_handle);
15181                 }
15182                 switch (mtr_policy->act_cnt[i].fate_action) {
15183                 case MLX5_FLOW_FATE_SHARED_RSS:
15184                         rss_action = mtr_policy->act_cnt[i].rss;
15185                         mlx5_free(rss_action);
15186                         break;
15187                 case MLX5_FLOW_FATE_PORT_ID:
15188                         if (mtr_policy->act_cnt[i].rix_port_id_action) {
15189                                 flow_dv_port_id_action_resource_release(dev,
15190                                 mtr_policy->act_cnt[i].rix_port_id_action);
15191                                 mtr_policy->act_cnt[i].rix_port_id_action = 0;
15192                         }
15193                         break;
15194                 case MLX5_FLOW_FATE_DROP:
15195                 case MLX5_FLOW_FATE_JUMP:
15196                         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
15197                                 mtr_policy->act_cnt[i].dr_jump_action[j] =
15198                                                 NULL;
15199                         break;
15200                 default:
15201                         /*Queue action do nothing*/
15202                         break;
15203                 }
15204         }
15205         for (j = 0; j < MLX5_MTR_DOMAIN_MAX; j++)
15206                 mtr_policy->dr_drop_action[j] = NULL;
15207 }
15208
15209 /**
15210  * Create policy action per domain, lock free,
15211  * (mutex should be acquired by caller).
15212  * Dispatcher for action type specific call.
15213  *
15214  * @param[in] dev
15215  *   Pointer to the Ethernet device structure.
15216  * @param[in] mtr_policy
15217  *   Meter policy struct.
15218  * @param[in] action
15219  *   Action specification used to create meter actions.
15220  * @param[out] error
15221  *   Perform verbose error reporting if not NULL. Initialized in case of
15222  *   error only.
15223  *
15224  * @return
15225  *   0 on success, otherwise negative errno value.
15226  */
15227 static int
15228 __flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev,
15229                         struct mlx5_flow_meter_policy *mtr_policy,
15230                         const struct rte_flow_action *actions[RTE_COLORS],
15231                         enum mlx5_meter_domain domain,
15232                         struct rte_mtr_error *error)
15233 {
15234         struct mlx5_priv *priv = dev->data->dev_private;
15235         struct rte_flow_error flow_err;
15236         const struct rte_flow_action *act;
15237         uint64_t action_flags;
15238         struct mlx5_flow_handle dh;
15239         struct mlx5_flow dev_flow;
15240         struct mlx5_flow_dv_port_id_action_resource port_id_action;
15241         int i, ret;
15242         uint8_t egress, transfer;
15243         struct mlx5_meter_policy_action_container *act_cnt = NULL;
15244         union {
15245                 struct mlx5_flow_dv_modify_hdr_resource res;
15246                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
15247                             sizeof(struct mlx5_modification_cmd) *
15248                             (MLX5_MAX_MODIFY_NUM + 1)];
15249         } mhdr_dummy;
15250         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
15251
15252         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
15253         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
15254         memset(&dh, 0, sizeof(struct mlx5_flow_handle));
15255         memset(&dev_flow, 0, sizeof(struct mlx5_flow));
15256         memset(&port_id_action, 0,
15257                sizeof(struct mlx5_flow_dv_port_id_action_resource));
15258         memset(mhdr_res, 0, sizeof(*mhdr_res));
15259         mhdr_res->ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
15260                                        (egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
15261                                         MLX5DV_FLOW_TABLE_TYPE_NIC_RX);
15262         dev_flow.handle = &dh;
15263         dev_flow.dv.port_id_action = &port_id_action;
15264         dev_flow.external = true;
15265         for (i = 0; i < RTE_COLORS; i++) {
15266                 if (i < MLX5_MTR_RTE_COLORS)
15267                         act_cnt = &mtr_policy->act_cnt[i];
15268                 /* Skip the color policy actions creation. */
15269                 if ((i == RTE_COLOR_YELLOW && mtr_policy->skip_y) ||
15270                     (i == RTE_COLOR_GREEN && mtr_policy->skip_g))
15271                         continue;
15272                 action_flags = 0;
15273                 for (act = actions[i];
15274                      act && act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
15275                         switch (act->type) {
15276                         case RTE_FLOW_ACTION_TYPE_MARK:
15277                         {
15278                                 uint32_t tag_be = mlx5_flow_mark_set
15279                                         (((const struct rte_flow_action_mark *)
15280                                         (act->conf))->id);
15281
15282                                 if (i >= MLX5_MTR_RTE_COLORS)
15283                                         return -rte_mtr_error_set(error,
15284                                           ENOTSUP,
15285                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15286                                           NULL,
15287                                           "cannot create policy "
15288                                           "mark action for this color");
15289                                 dev_flow.handle->mark = 1;
15290                                 if (flow_dv_tag_resource_register(dev, tag_be,
15291                                                   &dev_flow, &flow_err))
15292                                         return -rte_mtr_error_set(error,
15293                                         ENOTSUP,
15294                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15295                                         NULL,
15296                                         "cannot setup policy mark action");
15297                                 MLX5_ASSERT(dev_flow.dv.tag_resource);
15298                                 act_cnt->rix_mark =
15299                                         dev_flow.handle->dvh.rix_tag;
15300                                 action_flags |= MLX5_FLOW_ACTION_MARK;
15301                                 break;
15302                         }
15303                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
15304                                 if (i >= MLX5_MTR_RTE_COLORS)
15305                                         return -rte_mtr_error_set(error,
15306                                           ENOTSUP,
15307                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15308                                           NULL,
15309                                           "cannot create policy "
15310                                           "set tag action for this color");
15311                                 if (flow_dv_convert_action_set_tag
15312                                 (dev, mhdr_res,
15313                                 (const struct rte_flow_action_set_tag *)
15314                                 act->conf,  &flow_err))
15315                                         return -rte_mtr_error_set(error,
15316                                         ENOTSUP,
15317                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15318                                         NULL, "cannot convert policy "
15319                                         "set tag action");
15320                                 if (!mhdr_res->actions_num)
15321                                         return -rte_mtr_error_set(error,
15322                                         ENOTSUP,
15323                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15324                                         NULL, "cannot find policy "
15325                                         "set tag action");
15326                                 action_flags |= MLX5_FLOW_ACTION_SET_TAG;
15327                                 break;
15328                         case RTE_FLOW_ACTION_TYPE_DROP:
15329                         {
15330                                 struct mlx5_flow_mtr_mng *mtrmng =
15331                                                 priv->sh->mtrmng;
15332                                 struct mlx5_flow_tbl_data_entry *tbl_data;
15333
15334                                 /*
15335                                  * Create the drop table with
15336                                  * METER DROP level.
15337                                  */
15338                                 if (!mtrmng->drop_tbl[domain]) {
15339                                         mtrmng->drop_tbl[domain] =
15340                                         flow_dv_tbl_resource_get(dev,
15341                                         MLX5_FLOW_TABLE_LEVEL_METER,
15342                                         egress, transfer, false, NULL, 0,
15343                                         0, MLX5_MTR_TABLE_ID_DROP, &flow_err);
15344                                         if (!mtrmng->drop_tbl[domain])
15345                                                 return -rte_mtr_error_set
15346                                         (error, ENOTSUP,
15347                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15348                                         NULL,
15349                                         "Failed to create meter drop table");
15350                                 }
15351                                 tbl_data = container_of
15352                                 (mtrmng->drop_tbl[domain],
15353                                 struct mlx5_flow_tbl_data_entry, tbl);
15354                                 if (i < MLX5_MTR_RTE_COLORS) {
15355                                         act_cnt->dr_jump_action[domain] =
15356                                                 tbl_data->jump.action;
15357                                         act_cnt->fate_action =
15358                                                 MLX5_FLOW_FATE_DROP;
15359                                 }
15360                                 if (i == RTE_COLOR_RED)
15361                                         mtr_policy->dr_drop_action[domain] =
15362                                                 tbl_data->jump.action;
15363                                 action_flags |= MLX5_FLOW_ACTION_DROP;
15364                                 break;
15365                         }
15366                         case RTE_FLOW_ACTION_TYPE_QUEUE:
15367                         {
15368                                 if (i >= MLX5_MTR_RTE_COLORS)
15369                                         return -rte_mtr_error_set(error,
15370                                         ENOTSUP,
15371                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15372                                         NULL, "cannot create policy "
15373                                         "fate queue for this color");
15374                                 act_cnt->queue =
15375                                 ((const struct rte_flow_action_queue *)
15376                                         (act->conf))->index;
15377                                 act_cnt->fate_action =
15378                                         MLX5_FLOW_FATE_QUEUE;
15379                                 dev_flow.handle->fate_action =
15380                                         MLX5_FLOW_FATE_QUEUE;
15381                                 mtr_policy->is_queue = 1;
15382                                 action_flags |= MLX5_FLOW_ACTION_QUEUE;
15383                                 break;
15384                         }
15385                         case RTE_FLOW_ACTION_TYPE_RSS:
15386                         {
15387                                 int rss_size;
15388
15389                                 if (i >= MLX5_MTR_RTE_COLORS)
15390                                         return -rte_mtr_error_set(error,
15391                                           ENOTSUP,
15392                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15393                                           NULL,
15394                                           "cannot create policy "
15395                                           "rss action for this color");
15396                                 /*
15397                                  * Save RSS conf into policy struct
15398                                  * for translate stage.
15399                                  */
15400                                 rss_size = (int)rte_flow_conv
15401                                         (RTE_FLOW_CONV_OP_ACTION,
15402                                         NULL, 0, act, &flow_err);
15403                                 if (rss_size <= 0)
15404                                         return -rte_mtr_error_set(error,
15405                                           ENOTSUP,
15406                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15407                                           NULL, "Get the wrong "
15408                                           "rss action struct size");
15409                                 act_cnt->rss = mlx5_malloc(MLX5_MEM_ZERO,
15410                                                 rss_size, 0, SOCKET_ID_ANY);
15411                                 if (!act_cnt->rss)
15412                                         return -rte_mtr_error_set(error,
15413                                           ENOTSUP,
15414                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15415                                           NULL,
15416                                           "Fail to malloc rss action memory");
15417                                 ret = rte_flow_conv(RTE_FLOW_CONV_OP_ACTION,
15418                                         act_cnt->rss, rss_size,
15419                                         act, &flow_err);
15420                                 if (ret < 0)
15421                                         return -rte_mtr_error_set(error,
15422                                           ENOTSUP,
15423                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15424                                           NULL, "Fail to save "
15425                                           "rss action into policy struct");
15426                                 act_cnt->fate_action =
15427                                         MLX5_FLOW_FATE_SHARED_RSS;
15428                                 action_flags |= MLX5_FLOW_ACTION_RSS;
15429                                 break;
15430                         }
15431                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
15432                         {
15433                                 struct mlx5_flow_dv_port_id_action_resource
15434                                         port_id_resource;
15435                                 uint32_t port_id = 0;
15436
15437                                 if (i >= MLX5_MTR_RTE_COLORS)
15438                                         return -rte_mtr_error_set(error,
15439                                         ENOTSUP,
15440                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15441                                         NULL, "cannot create policy "
15442                                         "port action for this color");
15443                                 memset(&port_id_resource, 0,
15444                                         sizeof(port_id_resource));
15445                                 if (flow_dv_translate_action_port_id(dev, act,
15446                                                 &port_id, &flow_err))
15447                                         return -rte_mtr_error_set(error,
15448                                         ENOTSUP,
15449                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15450                                         NULL, "cannot translate "
15451                                         "policy port action");
15452                                 port_id_resource.port_id = port_id;
15453                                 if (flow_dv_port_id_action_resource_register
15454                                         (dev, &port_id_resource,
15455                                         &dev_flow, &flow_err))
15456                                         return -rte_mtr_error_set(error,
15457                                         ENOTSUP,
15458                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15459                                         NULL, "cannot setup "
15460                                         "policy port action");
15461                                 act_cnt->rix_port_id_action =
15462                                         dev_flow.handle->rix_port_id_action;
15463                                 act_cnt->fate_action =
15464                                         MLX5_FLOW_FATE_PORT_ID;
15465                                 action_flags |= MLX5_FLOW_ACTION_PORT_ID;
15466                                 break;
15467                         }
15468                         case RTE_FLOW_ACTION_TYPE_JUMP:
15469                         {
15470                                 uint32_t jump_group = 0;
15471                                 uint32_t table = 0;
15472                                 struct mlx5_flow_tbl_data_entry *tbl_data;
15473                                 struct flow_grp_info grp_info = {
15474                                         .external = !!dev_flow.external,
15475                                         .transfer = !!transfer,
15476                                         .fdb_def_rule = !!priv->fdb_def_rule,
15477                                         .std_tbl_fix = 0,
15478                                         .skip_scale = dev_flow.skip_scale &
15479                                         (1 << MLX5_SCALE_FLOW_GROUP_BIT),
15480                                 };
15481                                 struct mlx5_flow_meter_sub_policy *sub_policy =
15482                                         mtr_policy->sub_policys[domain][0];
15483
15484                                 if (i >= MLX5_MTR_RTE_COLORS)
15485                                         return -rte_mtr_error_set(error,
15486                                           ENOTSUP,
15487                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15488                                           NULL,
15489                                           "cannot create policy "
15490                                           "jump action for this color");
15491                                 jump_group =
15492                                 ((const struct rte_flow_action_jump *)
15493                                                         act->conf)->group;
15494                                 if (mlx5_flow_group_to_table(dev, NULL,
15495                                                        jump_group,
15496                                                        &table,
15497                                                        &grp_info, &flow_err))
15498                                         return -rte_mtr_error_set(error,
15499                                         ENOTSUP,
15500                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15501                                         NULL, "cannot setup "
15502                                         "policy jump action");
15503                                 sub_policy->jump_tbl[i] =
15504                                 flow_dv_tbl_resource_get(dev,
15505                                         table, egress,
15506                                         transfer,
15507                                         !!dev_flow.external,
15508                                         NULL, jump_group, 0,
15509                                         0, &flow_err);
15510                                 if
15511                                 (!sub_policy->jump_tbl[i])
15512                                         return  -rte_mtr_error_set(error,
15513                                         ENOTSUP,
15514                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
15515                                         NULL, "cannot create jump action.");
15516                                 tbl_data = container_of
15517                                 (sub_policy->jump_tbl[i],
15518                                 struct mlx5_flow_tbl_data_entry, tbl);
15519                                 act_cnt->dr_jump_action[domain] =
15520                                         tbl_data->jump.action;
15521                                 act_cnt->fate_action =
15522                                         MLX5_FLOW_FATE_JUMP;
15523                                 action_flags |= MLX5_FLOW_ACTION_JUMP;
15524                                 break;
15525                         }
15526                         /*
15527                          * No need to check meter hierarchy for Y or R colors
15528                          * here since it is done in the validation stage.
15529                          */
15530                         case RTE_FLOW_ACTION_TYPE_METER:
15531                         {
15532                                 const struct rte_flow_action_meter *mtr;
15533                                 struct mlx5_flow_meter_info *next_fm;
15534                                 struct mlx5_flow_meter_policy *next_policy;
15535                                 struct rte_flow_action tag_action;
15536                                 struct mlx5_rte_flow_action_set_tag set_tag;
15537                                 uint32_t next_mtr_idx = 0;
15538
15539                                 mtr = act->conf;
15540                                 next_fm = mlx5_flow_meter_find(priv,
15541                                                         mtr->mtr_id,
15542                                                         &next_mtr_idx);
15543                                 if (!next_fm)
15544                                         return -rte_mtr_error_set(error, EINVAL,
15545                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
15546                                                 "Fail to find next meter.");
15547                                 if (next_fm->def_policy)
15548                                         return -rte_mtr_error_set(error, EINVAL,
15549                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
15550                                 "Hierarchy only supports termination meter.");
15551                                 next_policy = mlx5_flow_meter_policy_find(dev,
15552                                                 next_fm->policy_id, NULL);
15553                                 MLX5_ASSERT(next_policy);
15554                                 if (next_fm->drop_cnt) {
15555                                         set_tag.id =
15556                                                 (enum modify_reg)
15557                                                 mlx5_flow_get_reg_id(dev,
15558                                                 MLX5_MTR_ID,
15559                                                 0,
15560                                                 (struct rte_flow_error *)error);
15561                                         set_tag.offset = (priv->mtr_reg_share ?
15562                                                 MLX5_MTR_COLOR_BITS : 0);
15563                                         set_tag.length = (priv->mtr_reg_share ?
15564                                                MLX5_MTR_IDLE_BITS_IN_COLOR_REG :
15565                                                MLX5_REG_BITS);
15566                                         set_tag.data = next_mtr_idx;
15567                                         tag_action.type =
15568                                                 (enum rte_flow_action_type)
15569                                                 MLX5_RTE_FLOW_ACTION_TYPE_TAG;
15570                                         tag_action.conf = &set_tag;
15571                                         if (flow_dv_convert_action_set_reg
15572                                                 (mhdr_res, &tag_action,
15573                                                 (struct rte_flow_error *)error))
15574                                                 return -rte_errno;
15575                                         action_flags |=
15576                                                 MLX5_FLOW_ACTION_SET_TAG;
15577                                 }
15578                                 act_cnt->fate_action = MLX5_FLOW_FATE_MTR;
15579                                 act_cnt->next_mtr_id = next_fm->meter_id;
15580                                 act_cnt->next_sub_policy = NULL;
15581                                 mtr_policy->is_hierarchy = 1;
15582                                 mtr_policy->dev = next_policy->dev;
15583                                 action_flags |=
15584                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
15585                                 break;
15586                         }
15587                         default:
15588                                 return -rte_mtr_error_set(error, ENOTSUP,
15589                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
15590                                           NULL, "action type not supported");
15591                         }
15592                         if (action_flags & MLX5_FLOW_ACTION_SET_TAG) {
15593                                 /* create modify action if needed. */
15594                                 dev_flow.dv.group = 1;
15595                                 if (flow_dv_modify_hdr_resource_register
15596                                         (dev, mhdr_res, &dev_flow, &flow_err))
15597                                         return -rte_mtr_error_set(error,
15598                                                 ENOTSUP,
15599                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
15600                                                 NULL, "cannot register policy "
15601                                                 "set tag action");
15602                                 act_cnt->modify_hdr =
15603                                         dev_flow.handle->dvh.modify_hdr;
15604                         }
15605                 }
15606         }
15607         return 0;
15608 }
15609
15610 /**
15611  * Create policy action per domain, lock free,
15612  * (mutex should be acquired by caller).
15613  * Dispatcher for action type specific call.
15614  *
15615  * @param[in] dev
15616  *   Pointer to the Ethernet device structure.
15617  * @param[in] mtr_policy
15618  *   Meter policy struct.
15619  * @param[in] action
15620  *   Action specification used to create meter actions.
15621  * @param[out] error
15622  *   Perform verbose error reporting if not NULL. Initialized in case of
15623  *   error only.
15624  *
15625  * @return
15626  *   0 on success, otherwise negative errno value.
15627  */
15628 static int
15629 flow_dv_create_mtr_policy_acts(struct rte_eth_dev *dev,
15630                       struct mlx5_flow_meter_policy *mtr_policy,
15631                       const struct rte_flow_action *actions[RTE_COLORS],
15632                       struct rte_mtr_error *error)
15633 {
15634         int ret, i;
15635         uint16_t sub_policy_num;
15636
15637         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15638                 sub_policy_num = (mtr_policy->sub_policy_num >>
15639                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
15640                         MLX5_MTR_SUB_POLICY_NUM_MASK;
15641                 if (sub_policy_num) {
15642                         ret = __flow_dv_create_domain_policy_acts(dev,
15643                                 mtr_policy, actions,
15644                                 (enum mlx5_meter_domain)i, error);
15645                         /* Cleaning resource is done in the caller level. */
15646                         if (ret)
15647                                 return ret;
15648                 }
15649         }
15650         return 0;
15651 }
15652
15653 /**
15654  * Query a DV flow rule for its statistics via DevX.
15655  *
15656  * @param[in] dev
15657  *   Pointer to Ethernet device.
15658  * @param[in] cnt_idx
15659  *   Index to the flow counter.
15660  * @param[out] data
15661  *   Data retrieved by the query.
15662  * @param[out] error
15663  *   Perform verbose error reporting if not NULL.
15664  *
15665  * @return
15666  *   0 on success, a negative errno value otherwise and rte_errno is set.
15667  */
15668 static int
15669 flow_dv_query_count(struct rte_eth_dev *dev, uint32_t cnt_idx, void *data,
15670                     struct rte_flow_error *error)
15671 {
15672         struct mlx5_priv *priv = dev->data->dev_private;
15673         struct rte_flow_query_count *qc = data;
15674
15675         if (!priv->config.devx)
15676                 return rte_flow_error_set(error, ENOTSUP,
15677                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15678                                           NULL,
15679                                           "counters are not supported");
15680         if (cnt_idx) {
15681                 uint64_t pkts, bytes;
15682                 struct mlx5_flow_counter *cnt;
15683                 int err = _flow_dv_query_count(dev, cnt_idx, &pkts, &bytes);
15684
15685                 if (err)
15686                         return rte_flow_error_set(error, -err,
15687                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15688                                         NULL, "cannot read counters");
15689                 cnt = flow_dv_counter_get_by_idx(dev, cnt_idx, NULL);
15690                 qc->hits_set = 1;
15691                 qc->bytes_set = 1;
15692                 qc->hits = pkts - cnt->hits;
15693                 qc->bytes = bytes - cnt->bytes;
15694                 if (qc->reset) {
15695                         cnt->hits = pkts;
15696                         cnt->bytes = bytes;
15697                 }
15698                 return 0;
15699         }
15700         return rte_flow_error_set(error, EINVAL,
15701                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15702                                   NULL,
15703                                   "counters are not available");
15704 }
15705
15706 static int
15707 flow_dv_action_query(struct rte_eth_dev *dev,
15708                      const struct rte_flow_action_handle *handle, void *data,
15709                      struct rte_flow_error *error)
15710 {
15711         struct mlx5_age_param *age_param;
15712         struct rte_flow_query_age *resp;
15713         uint32_t act_idx = (uint32_t)(uintptr_t)handle;
15714         uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
15715         uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
15716         struct mlx5_priv *priv = dev->data->dev_private;
15717         struct mlx5_aso_ct_action *ct;
15718         uint16_t owner;
15719         uint32_t dev_idx;
15720
15721         switch (type) {
15722         case MLX5_INDIRECT_ACTION_TYPE_AGE:
15723                 age_param = &flow_aso_age_get_by_idx(dev, idx)->age_params;
15724                 resp = data;
15725                 resp->aged = __atomic_load_n(&age_param->state,
15726                                               __ATOMIC_RELAXED) == AGE_TMOUT ?
15727                                                                           1 : 0;
15728                 resp->sec_since_last_hit_valid = !resp->aged;
15729                 if (resp->sec_since_last_hit_valid)
15730                         resp->sec_since_last_hit = __atomic_load_n
15731                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
15732                 return 0;
15733         case MLX5_INDIRECT_ACTION_TYPE_COUNT:
15734                 return flow_dv_query_count(dev, idx, data, error);
15735         case MLX5_INDIRECT_ACTION_TYPE_CT:
15736                 owner = (uint16_t)MLX5_INDIRECT_ACT_CT_GET_OWNER(idx);
15737                 if (owner != PORT_ID(priv))
15738                         return rte_flow_error_set(error, EACCES,
15739                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15740                                         NULL,
15741                                         "CT object owned by another port");
15742                 dev_idx = MLX5_INDIRECT_ACT_CT_GET_IDX(idx);
15743                 ct = flow_aso_ct_get_by_dev_idx(dev, dev_idx);
15744                 MLX5_ASSERT(ct);
15745                 if (!ct->refcnt)
15746                         return rte_flow_error_set(error, EFAULT,
15747                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15748                                         NULL,
15749                                         "CT object is inactive");
15750                 ((struct rte_flow_action_conntrack *)data)->peer_port =
15751                                                         ct->peer;
15752                 ((struct rte_flow_action_conntrack *)data)->is_original_dir =
15753                                                         ct->is_original;
15754                 if (mlx5_aso_ct_query_by_wqe(priv->sh, ct, data))
15755                         return rte_flow_error_set(error, EIO,
15756                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15757                                         NULL,
15758                                         "Failed to query CT context");
15759                 return 0;
15760         default:
15761                 return rte_flow_error_set(error, ENOTSUP,
15762                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
15763                                           "action type query not supported");
15764         }
15765 }
15766
15767 /**
15768  * Query a flow rule AGE action for aging information.
15769  *
15770  * @param[in] dev
15771  *   Pointer to Ethernet device.
15772  * @param[in] flow
15773  *   Pointer to the sub flow.
15774  * @param[out] data
15775  *   data retrieved by the query.
15776  * @param[out] error
15777  *   Perform verbose error reporting if not NULL.
15778  *
15779  * @return
15780  *   0 on success, a negative errno value otherwise and rte_errno is set.
15781  */
15782 static int
15783 flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow,
15784                   void *data, struct rte_flow_error *error)
15785 {
15786         struct rte_flow_query_age *resp = data;
15787         struct mlx5_age_param *age_param;
15788
15789         if (flow->age) {
15790                 struct mlx5_aso_age_action *act =
15791                                      flow_aso_age_get_by_idx(dev, flow->age);
15792
15793                 age_param = &act->age_params;
15794         } else if (flow->counter) {
15795                 age_param = flow_dv_counter_idx_get_age(dev, flow->counter);
15796
15797                 if (!age_param || !age_param->timeout)
15798                         return rte_flow_error_set
15799                                         (error, EINVAL,
15800                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15801                                          NULL, "cannot read age data");
15802         } else {
15803                 return rte_flow_error_set(error, EINVAL,
15804                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
15805                                           NULL, "age data not available");
15806         }
15807         resp->aged = __atomic_load_n(&age_param->state, __ATOMIC_RELAXED) ==
15808                                      AGE_TMOUT ? 1 : 0;
15809         resp->sec_since_last_hit_valid = !resp->aged;
15810         if (resp->sec_since_last_hit_valid)
15811                 resp->sec_since_last_hit = __atomic_load_n
15812                              (&age_param->sec_since_last_hit, __ATOMIC_RELAXED);
15813         return 0;
15814 }
15815
15816 /**
15817  * Query a flow.
15818  *
15819  * @see rte_flow_query()
15820  * @see rte_flow_ops
15821  */
15822 static int
15823 flow_dv_query(struct rte_eth_dev *dev,
15824               struct rte_flow *flow __rte_unused,
15825               const struct rte_flow_action *actions __rte_unused,
15826               void *data __rte_unused,
15827               struct rte_flow_error *error __rte_unused)
15828 {
15829         int ret = -EINVAL;
15830
15831         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
15832                 switch (actions->type) {
15833                 case RTE_FLOW_ACTION_TYPE_VOID:
15834                         break;
15835                 case RTE_FLOW_ACTION_TYPE_COUNT:
15836                         ret = flow_dv_query_count(dev, flow->counter, data,
15837                                                   error);
15838                         break;
15839                 case RTE_FLOW_ACTION_TYPE_AGE:
15840                         ret = flow_dv_query_age(dev, flow, data, error);
15841                         break;
15842                 default:
15843                         return rte_flow_error_set(error, ENOTSUP,
15844                                                   RTE_FLOW_ERROR_TYPE_ACTION,
15845                                                   actions,
15846                                                   "action not supported");
15847                 }
15848         }
15849         return ret;
15850 }
15851
15852 /**
15853  * Destroy the meter table set.
15854  * Lock free, (mutex should be acquired by caller).
15855  *
15856  * @param[in] dev
15857  *   Pointer to Ethernet device.
15858  * @param[in] fm
15859  *   Meter information table.
15860  */
15861 static void
15862 flow_dv_destroy_mtr_tbls(struct rte_eth_dev *dev,
15863                         struct mlx5_flow_meter_info *fm)
15864 {
15865         struct mlx5_priv *priv = dev->data->dev_private;
15866         int i;
15867
15868         if (!fm || !priv->config.dv_flow_en)
15869                 return;
15870         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15871                 if (fm->drop_rule[i]) {
15872                         claim_zero(mlx5_flow_os_destroy_flow(fm->drop_rule[i]));
15873                         fm->drop_rule[i] = NULL;
15874                 }
15875         }
15876 }
15877
15878 static void
15879 flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
15880 {
15881         struct mlx5_priv *priv = dev->data->dev_private;
15882         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
15883         struct mlx5_flow_tbl_data_entry *tbl;
15884         int i, j;
15885
15886         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
15887                 if (mtrmng->def_rule[i]) {
15888                         claim_zero(mlx5_flow_os_destroy_flow
15889                                         (mtrmng->def_rule[i]));
15890                         mtrmng->def_rule[i] = NULL;
15891                 }
15892                 if (mtrmng->def_matcher[i]) {
15893                         tbl = container_of(mtrmng->def_matcher[i]->tbl,
15894                                 struct mlx5_flow_tbl_data_entry, tbl);
15895                         mlx5_list_unregister(tbl->matchers,
15896                                              &mtrmng->def_matcher[i]->entry);
15897                         mtrmng->def_matcher[i] = NULL;
15898                 }
15899                 for (j = 0; j < MLX5_REG_BITS; j++) {
15900                         if (mtrmng->drop_matcher[i][j]) {
15901                                 tbl =
15902                                 container_of(mtrmng->drop_matcher[i][j]->tbl,
15903                                              struct mlx5_flow_tbl_data_entry,
15904                                              tbl);
15905                                 mlx5_list_unregister(tbl->matchers,
15906                                             &mtrmng->drop_matcher[i][j]->entry);
15907                                 mtrmng->drop_matcher[i][j] = NULL;
15908                         }
15909                 }
15910                 if (mtrmng->drop_tbl[i]) {
15911                         flow_dv_tbl_resource_release(MLX5_SH(dev),
15912                                 mtrmng->drop_tbl[i]);
15913                         mtrmng->drop_tbl[i] = NULL;
15914                 }
15915         }
15916 }
15917
15918 /* Number of meter flow actions, count and jump or count and drop. */
15919 #define METER_ACTIONS 2
15920
15921 static void
15922 __flow_dv_destroy_domain_def_policy(struct rte_eth_dev *dev,
15923                                     enum mlx5_meter_domain domain)
15924 {
15925         struct mlx5_priv *priv = dev->data->dev_private;
15926         struct mlx5_flow_meter_def_policy *def_policy =
15927                         priv->sh->mtrmng->def_policy[domain];
15928
15929         __flow_dv_destroy_sub_policy_rules(dev, &def_policy->sub_policy);
15930         mlx5_free(def_policy);
15931         priv->sh->mtrmng->def_policy[domain] = NULL;
15932 }
15933
15934 /**
15935  * Destroy the default policy table set.
15936  *
15937  * @param[in] dev
15938  *   Pointer to Ethernet device.
15939  */
15940 static void
15941 flow_dv_destroy_def_policy(struct rte_eth_dev *dev)
15942 {
15943         struct mlx5_priv *priv = dev->data->dev_private;
15944         int i;
15945
15946         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++)
15947                 if (priv->sh->mtrmng->def_policy[i])
15948                         __flow_dv_destroy_domain_def_policy(dev,
15949                                         (enum mlx5_meter_domain)i);
15950         priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
15951 }
15952
15953 static int
15954 __flow_dv_create_policy_flow(struct rte_eth_dev *dev,
15955                         uint32_t color_reg_c_idx,
15956                         enum rte_color color, void *matcher_object,
15957                         int actions_n, void *actions,
15958                         bool match_src_port, const struct rte_flow_item *item,
15959                         void **rule, const struct rte_flow_attr *attr)
15960 {
15961         int ret;
15962         struct mlx5_flow_dv_match_params value = {
15963                 .size = sizeof(value.buf),
15964         };
15965         struct mlx5_flow_dv_match_params matcher = {
15966                 .size = sizeof(matcher.buf),
15967         };
15968         struct mlx5_priv *priv = dev->data->dev_private;
15969         uint8_t misc_mask;
15970
15971         if (match_src_port && (priv->representor || priv->master)) {
15972                 if (flow_dv_translate_item_port_id(dev, matcher.buf,
15973                                                    value.buf, item, attr)) {
15974                         DRV_LOG(ERR, "Failed to create meter policy%d flow's"
15975                                 " value with port.", color);
15976                         return -1;
15977                 }
15978         }
15979         flow_dv_match_meta_reg(matcher.buf, value.buf,
15980                                (enum modify_reg)color_reg_c_idx,
15981                                rte_col_2_mlx5_col(color), UINT32_MAX);
15982         misc_mask = flow_dv_matcher_enable(value.buf);
15983         __flow_dv_adjust_buf_size(&value.size, misc_mask);
15984         ret = mlx5_flow_os_create_flow(matcher_object, (void *)&value,
15985                                        actions_n, actions, rule);
15986         if (ret) {
15987                 DRV_LOG(ERR, "Failed to create meter policy%d flow.", color);
15988                 return -1;
15989         }
15990         return 0;
15991 }
15992
15993 static int
15994 __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
15995                         uint32_t color_reg_c_idx,
15996                         uint16_t priority,
15997                         struct mlx5_flow_meter_sub_policy *sub_policy,
15998                         const struct rte_flow_attr *attr,
15999                         bool match_src_port,
16000                         const struct rte_flow_item *item,
16001                         struct mlx5_flow_dv_matcher **policy_matcher,
16002                         struct rte_flow_error *error)
16003 {
16004         struct mlx5_list_entry *entry;
16005         struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc;
16006         struct mlx5_flow_dv_matcher matcher = {
16007                 .mask = {
16008                         .size = sizeof(matcher.mask.buf),
16009                 },
16010                 .tbl = tbl_rsc,
16011         };
16012         struct mlx5_flow_dv_match_params value = {
16013                 .size = sizeof(value.buf),
16014         };
16015         struct mlx5_flow_cb_ctx ctx = {
16016                 .error = error,
16017                 .data = &matcher,
16018         };
16019         struct mlx5_flow_tbl_data_entry *tbl_data;
16020         struct mlx5_priv *priv = dev->data->dev_private;
16021         const uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1;
16022
16023         if (match_src_port && (priv->representor || priv->master)) {
16024                 if (flow_dv_translate_item_port_id(dev, matcher.mask.buf,
16025                                                    value.buf, item, attr)) {
16026                         DRV_LOG(ERR, "Failed to register meter policy%d matcher"
16027                                 " with port.", priority);
16028                         return -1;
16029                 }
16030         }
16031         tbl_data = container_of(tbl_rsc, struct mlx5_flow_tbl_data_entry, tbl);
16032         if (priority < RTE_COLOR_RED)
16033                 flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16034                         (enum modify_reg)color_reg_c_idx, 0, color_mask);
16035         matcher.priority = priority;
16036         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
16037                                     matcher.mask.size);
16038         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16039         if (!entry) {
16040                 DRV_LOG(ERR, "Failed to register meter drop matcher.");
16041                 return -1;
16042         }
16043         *policy_matcher =
16044                 container_of(entry, struct mlx5_flow_dv_matcher, entry);
16045         return 0;
16046 }
16047
16048 /**
16049  * Create the policy rules per domain.
16050  *
16051  * @param[in] dev
16052  *   Pointer to Ethernet device.
16053  * @param[in] sub_policy
16054  *    Pointer to sub policy table..
16055  * @param[in] egress
16056  *   Direction of the table.
16057  * @param[in] transfer
16058  *   E-Switch or NIC flow.
16059  * @param[in] acts
16060  *   Pointer to policy action list per color.
16061  *
16062  * @return
16063  *   0 on success, -1 otherwise.
16064  */
16065 static int
16066 __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev,
16067                 struct mlx5_flow_meter_sub_policy *sub_policy,
16068                 uint8_t egress, uint8_t transfer, bool match_src_port,
16069                 struct mlx5_meter_policy_acts acts[RTE_COLORS])
16070 {
16071         struct mlx5_priv *priv = dev->data->dev_private;
16072         struct rte_flow_error flow_err;
16073         uint32_t color_reg_c_idx;
16074         struct rte_flow_attr attr = {
16075                 .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
16076                 .priority = 0,
16077                 .ingress = 0,
16078                 .egress = !!egress,
16079                 .transfer = !!transfer,
16080                 .reserved = 0,
16081         };
16082         int i;
16083         int ret = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &flow_err);
16084         struct mlx5_sub_policy_color_rule *color_rule;
16085         bool svport_match;
16086         struct mlx5_sub_policy_color_rule *tmp_rules[RTE_COLORS] = {NULL};
16087
16088         if (ret < 0)
16089                 return -1;
16090         /* Create policy table with POLICY level. */
16091         if (!sub_policy->tbl_rsc)
16092                 sub_policy->tbl_rsc = flow_dv_tbl_resource_get(dev,
16093                                 MLX5_FLOW_TABLE_LEVEL_POLICY,
16094                                 egress, transfer, false, NULL, 0, 0,
16095                                 sub_policy->idx, &flow_err);
16096         if (!sub_policy->tbl_rsc) {
16097                 DRV_LOG(ERR,
16098                         "Failed to create meter sub policy table.");
16099                 return -1;
16100         }
16101         /* Prepare matchers. */
16102         color_reg_c_idx = ret;
16103         for (i = 0; i < RTE_COLORS; i++) {
16104                 TAILQ_INIT(&sub_policy->color_rules[i]);
16105                 if (!acts[i].actions_n)
16106                         continue;
16107                 color_rule = mlx5_malloc(MLX5_MEM_ZERO,
16108                                 sizeof(struct mlx5_sub_policy_color_rule),
16109                                 0, SOCKET_ID_ANY);
16110                 if (!color_rule) {
16111                         DRV_LOG(ERR, "No memory to create color rule.");
16112                         goto err_exit;
16113                 }
16114                 tmp_rules[i] = color_rule;
16115                 TAILQ_INSERT_TAIL(&sub_policy->color_rules[i],
16116                                   color_rule, next_port);
16117                 color_rule->src_port = priv->representor_id;
16118                 /* No use. */
16119                 attr.priority = i;
16120                 /* Create matchers for colors. */
16121                 svport_match = (i != RTE_COLOR_RED) ? match_src_port : false;
16122                 if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx,
16123                                 MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy,
16124                                 &attr, svport_match, NULL,
16125                                 &color_rule->matcher, &flow_err)) {
16126                         DRV_LOG(ERR, "Failed to create color%u matcher.", i);
16127                         goto err_exit;
16128                 }
16129                 /* Create flow, matching color. */
16130                 if (__flow_dv_create_policy_flow(dev,
16131                                 color_reg_c_idx, (enum rte_color)i,
16132                                 color_rule->matcher->matcher_object,
16133                                 acts[i].actions_n, acts[i].dv_actions,
16134                                 svport_match, NULL, &color_rule->rule,
16135                                 &attr)) {
16136                         DRV_LOG(ERR, "Failed to create color%u rule.", i);
16137                         goto err_exit;
16138                 }
16139         }
16140         return 0;
16141 err_exit:
16142         /* All the policy rules will be cleared. */
16143         do {
16144                 color_rule = tmp_rules[i];
16145                 if (color_rule) {
16146                         if (color_rule->rule)
16147                                 mlx5_flow_os_destroy_flow(color_rule->rule);
16148                         if (color_rule->matcher) {
16149                                 struct mlx5_flow_tbl_data_entry *tbl =
16150                                         container_of(color_rule->matcher->tbl,
16151                                                      typeof(*tbl), tbl);
16152                                 mlx5_list_unregister(tbl->matchers,
16153                                                 &color_rule->matcher->entry);
16154                         }
16155                         TAILQ_REMOVE(&sub_policy->color_rules[i],
16156                                      color_rule, next_port);
16157                         mlx5_free(color_rule);
16158                 }
16159         } while (i--);
16160         return -1;
16161 }
16162
16163 static int
16164 __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev,
16165                         struct mlx5_flow_meter_policy *mtr_policy,
16166                         struct mlx5_flow_meter_sub_policy *sub_policy,
16167                         uint32_t domain)
16168 {
16169         struct mlx5_priv *priv = dev->data->dev_private;
16170         struct mlx5_meter_policy_acts acts[RTE_COLORS];
16171         struct mlx5_flow_dv_tag_resource *tag;
16172         struct mlx5_flow_dv_port_id_action_resource *port_action;
16173         struct mlx5_hrxq *hrxq;
16174         struct mlx5_flow_meter_info *next_fm = NULL;
16175         struct mlx5_flow_meter_policy *next_policy;
16176         struct mlx5_flow_meter_sub_policy *next_sub_policy;
16177         struct mlx5_flow_tbl_data_entry *tbl_data;
16178         struct rte_flow_error error;
16179         uint8_t egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16180         uint8_t transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16181         bool mtr_first = egress || (transfer && priv->representor_id != UINT16_MAX);
16182         bool match_src_port = false;
16183         int i;
16184
16185         /* If RSS or Queue, no previous actions / rules is created. */
16186         for (i = 0; i < RTE_COLORS; i++) {
16187                 acts[i].actions_n = 0;
16188                 if (i == RTE_COLOR_RED) {
16189                         /* Only support drop on red. */
16190                         acts[i].dv_actions[0] =
16191                                 mtr_policy->dr_drop_action[domain];
16192                         acts[i].actions_n = 1;
16193                         continue;
16194                 }
16195                 if (i == RTE_COLOR_GREEN &&
16196                     mtr_policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) {
16197                         struct rte_flow_attr attr = {
16198                                 .transfer = transfer
16199                         };
16200
16201                         next_fm = mlx5_flow_meter_find(priv,
16202                                         mtr_policy->act_cnt[i].next_mtr_id,
16203                                         NULL);
16204                         if (!next_fm) {
16205                                 DRV_LOG(ERR,
16206                                         "Failed to get next hierarchy meter.");
16207                                 goto err_exit;
16208                         }
16209                         if (mlx5_flow_meter_attach(priv, next_fm,
16210                                                    &attr, &error)) {
16211                                 DRV_LOG(ERR, "%s", error.message);
16212                                 next_fm = NULL;
16213                                 goto err_exit;
16214                         }
16215                         /* Meter action must be the first for TX. */
16216                         if (mtr_first) {
16217                                 acts[i].dv_actions[acts[i].actions_n] =
16218                                         next_fm->meter_action;
16219                                 acts[i].actions_n++;
16220                         }
16221                 }
16222                 if (mtr_policy->act_cnt[i].rix_mark) {
16223                         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG],
16224                                         mtr_policy->act_cnt[i].rix_mark);
16225                         if (!tag) {
16226                                 DRV_LOG(ERR, "Failed to find "
16227                                 "mark action for policy.");
16228                                 goto err_exit;
16229                         }
16230                         acts[i].dv_actions[acts[i].actions_n] = tag->action;
16231                         acts[i].actions_n++;
16232                 }
16233                 if (mtr_policy->act_cnt[i].modify_hdr) {
16234                         acts[i].dv_actions[acts[i].actions_n] =
16235                                 mtr_policy->act_cnt[i].modify_hdr->action;
16236                         acts[i].actions_n++;
16237                 }
16238                 if (mtr_policy->act_cnt[i].fate_action) {
16239                         switch (mtr_policy->act_cnt[i].fate_action) {
16240                         case MLX5_FLOW_FATE_PORT_ID:
16241                                 port_action = mlx5_ipool_get
16242                                         (priv->sh->ipool[MLX5_IPOOL_PORT_ID],
16243                                 mtr_policy->act_cnt[i].rix_port_id_action);
16244                                 if (!port_action) {
16245                                         DRV_LOG(ERR, "Failed to find "
16246                                                 "port action for policy.");
16247                                         goto err_exit;
16248                                 }
16249                                 acts[i].dv_actions[acts[i].actions_n] =
16250                                         port_action->action;
16251                                 acts[i].actions_n++;
16252                                 mtr_policy->dev = dev;
16253                                 match_src_port = true;
16254                                 break;
16255                         case MLX5_FLOW_FATE_DROP:
16256                         case MLX5_FLOW_FATE_JUMP:
16257                                 acts[i].dv_actions[acts[i].actions_n] =
16258                                 mtr_policy->act_cnt[i].dr_jump_action[domain];
16259                                 acts[i].actions_n++;
16260                                 break;
16261                         case MLX5_FLOW_FATE_SHARED_RSS:
16262                         case MLX5_FLOW_FATE_QUEUE:
16263                                 hrxq = mlx5_ipool_get
16264                                         (priv->sh->ipool[MLX5_IPOOL_HRXQ],
16265                                          sub_policy->rix_hrxq[i]);
16266                                 if (!hrxq) {
16267                                         DRV_LOG(ERR, "Failed to find "
16268                                                 "queue action for policy.");
16269                                         goto err_exit;
16270                                 }
16271                                 acts[i].dv_actions[acts[i].actions_n] =
16272                                         hrxq->action;
16273                                 acts[i].actions_n++;
16274                                 break;
16275                         case MLX5_FLOW_FATE_MTR:
16276                                 if (!next_fm) {
16277                                         DRV_LOG(ERR,
16278                                                 "No next hierarchy meter.");
16279                                         goto err_exit;
16280                                 }
16281                                 if (!mtr_first) {
16282                                         acts[i].dv_actions[acts[i].actions_n] =
16283                                                         next_fm->meter_action;
16284                                         acts[i].actions_n++;
16285                                 }
16286                                 if (mtr_policy->act_cnt[i].next_sub_policy) {
16287                                         next_sub_policy =
16288                                         mtr_policy->act_cnt[i].next_sub_policy;
16289                                 } else {
16290                                         next_policy =
16291                                                 mlx5_flow_meter_policy_find(dev,
16292                                                 next_fm->policy_id, NULL);
16293                                         MLX5_ASSERT(next_policy);
16294                                         next_sub_policy =
16295                                         next_policy->sub_policys[domain][0];
16296                                 }
16297                                 tbl_data =
16298                                         container_of(next_sub_policy->tbl_rsc,
16299                                         struct mlx5_flow_tbl_data_entry, tbl);
16300                                 acts[i].dv_actions[acts[i].actions_n++] =
16301                                                         tbl_data->jump.action;
16302                                 if (mtr_policy->act_cnt[i].modify_hdr)
16303                                         match_src_port = !!transfer;
16304                                 break;
16305                         default:
16306                                 /*Queue action do nothing*/
16307                                 break;
16308                         }
16309                 }
16310         }
16311         if (__flow_dv_create_domain_policy_rules(dev, sub_policy,
16312                                 egress, transfer, match_src_port, acts)) {
16313                 DRV_LOG(ERR,
16314                         "Failed to create policy rules per domain.");
16315                 goto err_exit;
16316         }
16317         return 0;
16318 err_exit:
16319         if (next_fm)
16320                 mlx5_flow_meter_detach(priv, next_fm);
16321         return -1;
16322 }
16323
16324 /**
16325  * Create the policy rules.
16326  *
16327  * @param[in] dev
16328  *   Pointer to Ethernet device.
16329  * @param[in,out] mtr_policy
16330  *   Pointer to meter policy table.
16331  *
16332  * @return
16333  *   0 on success, -1 otherwise.
16334  */
16335 static int
16336 flow_dv_create_policy_rules(struct rte_eth_dev *dev,
16337                              struct mlx5_flow_meter_policy *mtr_policy)
16338 {
16339         int i;
16340         uint16_t sub_policy_num;
16341
16342         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16343                 sub_policy_num = (mtr_policy->sub_policy_num >>
16344                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
16345                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16346                 if (!sub_policy_num)
16347                         continue;
16348                 /* Prepare actions list and create policy rules. */
16349                 if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
16350                         mtr_policy->sub_policys[i][0], i)) {
16351                         DRV_LOG(ERR, "Failed to create policy action "
16352                                 "list per domain.");
16353                         return -1;
16354                 }
16355         }
16356         return 0;
16357 }
16358
16359 static int
16360 __flow_dv_create_domain_def_policy(struct rte_eth_dev *dev, uint32_t domain)
16361 {
16362         struct mlx5_priv *priv = dev->data->dev_private;
16363         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
16364         struct mlx5_flow_meter_def_policy *def_policy;
16365         struct mlx5_flow_tbl_resource *jump_tbl;
16366         struct mlx5_flow_tbl_data_entry *tbl_data;
16367         uint8_t egress, transfer;
16368         struct rte_flow_error error;
16369         struct mlx5_meter_policy_acts acts[RTE_COLORS];
16370         int ret;
16371
16372         egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16373         transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16374         def_policy = mtrmng->def_policy[domain];
16375         if (!def_policy) {
16376                 def_policy = mlx5_malloc(MLX5_MEM_ZERO,
16377                         sizeof(struct mlx5_flow_meter_def_policy),
16378                         RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
16379                 if (!def_policy) {
16380                         DRV_LOG(ERR, "Failed to alloc default policy table.");
16381                         goto def_policy_error;
16382                 }
16383                 mtrmng->def_policy[domain] = def_policy;
16384                 /* Create the meter suffix table with SUFFIX level. */
16385                 jump_tbl = flow_dv_tbl_resource_get(dev,
16386                                 MLX5_FLOW_TABLE_LEVEL_METER,
16387                                 egress, transfer, false, NULL, 0,
16388                                 0, MLX5_MTR_TABLE_ID_SUFFIX, &error);
16389                 if (!jump_tbl) {
16390                         DRV_LOG(ERR,
16391                                 "Failed to create meter suffix table.");
16392                         goto def_policy_error;
16393                 }
16394                 def_policy->sub_policy.jump_tbl[RTE_COLOR_GREEN] = jump_tbl;
16395                 tbl_data = container_of(jump_tbl,
16396                                         struct mlx5_flow_tbl_data_entry, tbl);
16397                 def_policy->dr_jump_action[RTE_COLOR_GREEN] =
16398                                                 tbl_data->jump.action;
16399                 acts[RTE_COLOR_GREEN].dv_actions[0] = tbl_data->jump.action;
16400                 acts[RTE_COLOR_GREEN].actions_n = 1;
16401                 /*
16402                  * YELLOW has the same default policy as GREEN does.
16403                  * G & Y share the same table and action. The 2nd time of table
16404                  * resource getting is just to update the reference count for
16405                  * the releasing stage.
16406                  */
16407                 jump_tbl = flow_dv_tbl_resource_get(dev,
16408                                 MLX5_FLOW_TABLE_LEVEL_METER,
16409                                 egress, transfer, false, NULL, 0,
16410                                 0, MLX5_MTR_TABLE_ID_SUFFIX, &error);
16411                 if (!jump_tbl) {
16412                         DRV_LOG(ERR,
16413                                 "Failed to get meter suffix table.");
16414                         goto def_policy_error;
16415                 }
16416                 def_policy->sub_policy.jump_tbl[RTE_COLOR_YELLOW] = jump_tbl;
16417                 tbl_data = container_of(jump_tbl,
16418                                         struct mlx5_flow_tbl_data_entry, tbl);
16419                 def_policy->dr_jump_action[RTE_COLOR_YELLOW] =
16420                                                 tbl_data->jump.action;
16421                 acts[RTE_COLOR_YELLOW].dv_actions[0] = tbl_data->jump.action;
16422                 acts[RTE_COLOR_YELLOW].actions_n = 1;
16423                 /* Create jump action to the drop table. */
16424                 if (!mtrmng->drop_tbl[domain]) {
16425                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get
16426                                 (dev, MLX5_FLOW_TABLE_LEVEL_METER,
16427                                  egress, transfer, false, NULL, 0,
16428                                  0, MLX5_MTR_TABLE_ID_DROP, &error);
16429                         if (!mtrmng->drop_tbl[domain]) {
16430                                 DRV_LOG(ERR, "Failed to create meter "
16431                                         "drop table for default policy.");
16432                                 goto def_policy_error;
16433                         }
16434                 }
16435                 /* all RED: unique Drop table for jump action. */
16436                 tbl_data = container_of(mtrmng->drop_tbl[domain],
16437                                         struct mlx5_flow_tbl_data_entry, tbl);
16438                 def_policy->dr_jump_action[RTE_COLOR_RED] =
16439                                                 tbl_data->jump.action;
16440                 acts[RTE_COLOR_RED].dv_actions[0] = tbl_data->jump.action;
16441                 acts[RTE_COLOR_RED].actions_n = 1;
16442                 /* Create default policy rules. */
16443                 ret = __flow_dv_create_domain_policy_rules(dev,
16444                                         &def_policy->sub_policy,
16445                                         egress, transfer, false, acts);
16446                 if (ret) {
16447                         DRV_LOG(ERR, "Failed to create default policy rules.");
16448                         goto def_policy_error;
16449                 }
16450         }
16451         return 0;
16452 def_policy_error:
16453         __flow_dv_destroy_domain_def_policy(dev,
16454                                             (enum mlx5_meter_domain)domain);
16455         return -1;
16456 }
16457
16458 /**
16459  * Create the default policy table set.
16460  *
16461  * @param[in] dev
16462  *   Pointer to Ethernet device.
16463  * @return
16464  *   0 on success, -1 otherwise.
16465  */
16466 static int
16467 flow_dv_create_def_policy(struct rte_eth_dev *dev)
16468 {
16469         struct mlx5_priv *priv = dev->data->dev_private;
16470         int i;
16471
16472         /* Non-termination policy table. */
16473         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16474                 if (!priv->config.dv_esw_en && i == MLX5_MTR_DOMAIN_TRANSFER)
16475                         continue;
16476                 if (__flow_dv_create_domain_def_policy(dev, i)) {
16477                         DRV_LOG(ERR, "Failed to create default policy");
16478                         /* Rollback the created default policies for others. */
16479                         flow_dv_destroy_def_policy(dev);
16480                         return -1;
16481                 }
16482         }
16483         return 0;
16484 }
16485
16486 /**
16487  * Create the needed meter tables.
16488  * Lock free, (mutex should be acquired by caller).
16489  *
16490  * @param[in] dev
16491  *   Pointer to Ethernet device.
16492  * @param[in] fm
16493  *   Meter information table.
16494  * @param[in] mtr_idx
16495  *   Meter index.
16496  * @param[in] domain_bitmap
16497  *   Domain bitmap.
16498  * @return
16499  *   0 on success, -1 otherwise.
16500  */
16501 static int
16502 flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
16503                         struct mlx5_flow_meter_info *fm,
16504                         uint32_t mtr_idx,
16505                         uint8_t domain_bitmap)
16506 {
16507         struct mlx5_priv *priv = dev->data->dev_private;
16508         struct mlx5_flow_mtr_mng *mtrmng = priv->sh->mtrmng;
16509         struct rte_flow_error error;
16510         struct mlx5_flow_tbl_data_entry *tbl_data;
16511         uint8_t egress, transfer;
16512         void *actions[METER_ACTIONS];
16513         int domain, ret, i;
16514         struct mlx5_flow_counter *cnt;
16515         struct mlx5_flow_dv_match_params value = {
16516                 .size = sizeof(value.buf),
16517         };
16518         struct mlx5_flow_dv_match_params matcher_para = {
16519                 .size = sizeof(matcher_para.buf),
16520         };
16521         int mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
16522                                                      0, &error);
16523         uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1;
16524         uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
16525         struct mlx5_list_entry *entry;
16526         struct mlx5_flow_dv_matcher matcher = {
16527                 .mask = {
16528                         .size = sizeof(matcher.mask.buf),
16529                 },
16530         };
16531         struct mlx5_flow_dv_matcher *drop_matcher;
16532         struct mlx5_flow_cb_ctx ctx = {
16533                 .error = &error,
16534                 .data = &matcher,
16535         };
16536         uint8_t misc_mask;
16537
16538         if (!priv->mtr_en || mtr_id_reg_c < 0) {
16539                 rte_errno = ENOTSUP;
16540                 return -1;
16541         }
16542         for (domain = 0; domain < MLX5_MTR_DOMAIN_MAX; domain++) {
16543                 if (!(domain_bitmap & (1 << domain)) ||
16544                         (mtrmng->def_rule[domain] && !fm->drop_cnt))
16545                         continue;
16546                 egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
16547                 transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
16548                 /* Create the drop table with METER DROP level. */
16549                 if (!mtrmng->drop_tbl[domain]) {
16550                         mtrmng->drop_tbl[domain] = flow_dv_tbl_resource_get(dev,
16551                                         MLX5_FLOW_TABLE_LEVEL_METER,
16552                                         egress, transfer, false, NULL, 0,
16553                                         0, MLX5_MTR_TABLE_ID_DROP, &error);
16554                         if (!mtrmng->drop_tbl[domain]) {
16555                                 DRV_LOG(ERR, "Failed to create meter drop table.");
16556                                 goto policy_error;
16557                         }
16558                 }
16559                 /* Create default matcher in drop table. */
16560                 matcher.tbl = mtrmng->drop_tbl[domain],
16561                 tbl_data = container_of(mtrmng->drop_tbl[domain],
16562                                 struct mlx5_flow_tbl_data_entry, tbl);
16563                 if (!mtrmng->def_matcher[domain]) {
16564                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16565                                        (enum modify_reg)mtr_id_reg_c,
16566                                        0, 0);
16567                         matcher.priority = MLX5_MTRS_DEFAULT_RULE_PRIORITY;
16568                         matcher.crc = rte_raw_cksum
16569                                         ((const void *)matcher.mask.buf,
16570                                         matcher.mask.size);
16571                         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16572                         if (!entry) {
16573                                 DRV_LOG(ERR, "Failed to register meter "
16574                                 "drop default matcher.");
16575                                 goto policy_error;
16576                         }
16577                         mtrmng->def_matcher[domain] = container_of(entry,
16578                         struct mlx5_flow_dv_matcher, entry);
16579                 }
16580                 /* Create default rule in drop table. */
16581                 if (!mtrmng->def_rule[domain]) {
16582                         i = 0;
16583                         actions[i++] = priv->sh->dr_drop_action;
16584                         flow_dv_match_meta_reg(matcher_para.buf, value.buf,
16585                                 (enum modify_reg)mtr_id_reg_c, 0, 0);
16586                         misc_mask = flow_dv_matcher_enable(value.buf);
16587                         __flow_dv_adjust_buf_size(&value.size, misc_mask);
16588                         ret = mlx5_flow_os_create_flow
16589                                 (mtrmng->def_matcher[domain]->matcher_object,
16590                                 (void *)&value, i, actions,
16591                                 &mtrmng->def_rule[domain]);
16592                         if (ret) {
16593                                 DRV_LOG(ERR, "Failed to create meter "
16594                                 "default drop rule for drop table.");
16595                                 goto policy_error;
16596                         }
16597                 }
16598                 if (!fm->drop_cnt)
16599                         continue;
16600                 MLX5_ASSERT(mtrmng->max_mtr_bits);
16601                 if (!mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1]) {
16602                         /* Create matchers for Drop. */
16603                         flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
16604                                         (enum modify_reg)mtr_id_reg_c, 0,
16605                                         (mtr_id_mask << mtr_id_offset));
16606                         matcher.priority = MLX5_REG_BITS - mtrmng->max_mtr_bits;
16607                         matcher.crc = rte_raw_cksum
16608                                         ((const void *)matcher.mask.buf,
16609                                         matcher.mask.size);
16610                         entry = mlx5_list_register(tbl_data->matchers, &ctx);
16611                         if (!entry) {
16612                                 DRV_LOG(ERR,
16613                                 "Failed to register meter drop matcher.");
16614                                 goto policy_error;
16615                         }
16616                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1] =
16617                                 container_of(entry, struct mlx5_flow_dv_matcher,
16618                                              entry);
16619                 }
16620                 drop_matcher =
16621                         mtrmng->drop_matcher[domain][mtrmng->max_mtr_bits - 1];
16622                 /* Create drop rule, matching meter_id only. */
16623                 flow_dv_match_meta_reg(matcher_para.buf, value.buf,
16624                                 (enum modify_reg)mtr_id_reg_c,
16625                                 (mtr_idx << mtr_id_offset), UINT32_MAX);
16626                 i = 0;
16627                 cnt = flow_dv_counter_get_by_idx(dev,
16628                                         fm->drop_cnt, NULL);
16629                 actions[i++] = cnt->action;
16630                 actions[i++] = priv->sh->dr_drop_action;
16631                 misc_mask = flow_dv_matcher_enable(value.buf);
16632                 __flow_dv_adjust_buf_size(&value.size, misc_mask);
16633                 ret = mlx5_flow_os_create_flow(drop_matcher->matcher_object,
16634                                                (void *)&value, i, actions,
16635                                                &fm->drop_rule[domain]);
16636                 if (ret) {
16637                         DRV_LOG(ERR, "Failed to create meter "
16638                                 "drop rule for drop table.");
16639                                 goto policy_error;
16640                 }
16641         }
16642         return 0;
16643 policy_error:
16644         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
16645                 if (fm->drop_rule[i]) {
16646                         claim_zero(mlx5_flow_os_destroy_flow
16647                                 (fm->drop_rule[i]));
16648                         fm->drop_rule[i] = NULL;
16649                 }
16650         }
16651         return -1;
16652 }
16653
16654 static struct mlx5_flow_meter_sub_policy *
16655 __flow_dv_meter_get_rss_sub_policy(struct rte_eth_dev *dev,
16656                 struct mlx5_flow_meter_policy *mtr_policy,
16657                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS],
16658                 struct mlx5_flow_meter_sub_policy *next_sub_policy,
16659                 bool *is_reuse)
16660 {
16661         struct mlx5_priv *priv = dev->data->dev_private;
16662         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
16663         uint32_t sub_policy_idx = 0;
16664         uint32_t hrxq_idx[MLX5_MTR_RTE_COLORS] = {0};
16665         uint32_t i, j;
16666         struct mlx5_hrxq *hrxq;
16667         struct mlx5_flow_handle dh;
16668         struct mlx5_meter_policy_action_container *act_cnt;
16669         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
16670         uint16_t sub_policy_num;
16671
16672         rte_spinlock_lock(&mtr_policy->sl);
16673         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16674                 if (!rss_desc[i])
16675                         continue;
16676                 hrxq_idx[i] = mlx5_hrxq_get(dev, rss_desc[i]);
16677                 if (!hrxq_idx[i]) {
16678                         rte_spinlock_unlock(&mtr_policy->sl);
16679                         return NULL;
16680                 }
16681         }
16682         sub_policy_num = (mtr_policy->sub_policy_num >>
16683                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16684                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16685         for (j = 0; j < sub_policy_num; j++) {
16686                 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16687                         if (rss_desc[i] &&
16688                             hrxq_idx[i] !=
16689                             mtr_policy->sub_policys[domain][j]->rix_hrxq[i])
16690                                 break;
16691                 }
16692                 if (i >= MLX5_MTR_RTE_COLORS) {
16693                         /*
16694                          * Found the sub policy table with
16695                          * the same queue per color.
16696                          */
16697                         rte_spinlock_unlock(&mtr_policy->sl);
16698                         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++)
16699                                 mlx5_hrxq_release(dev, hrxq_idx[i]);
16700                         *is_reuse = true;
16701                         return mtr_policy->sub_policys[domain][j];
16702                 }
16703         }
16704         /* Create sub policy. */
16705         if (!mtr_policy->sub_policys[domain][0]->rix_hrxq[0]) {
16706                 /* Reuse the first pre-allocated sub_policy. */
16707                 sub_policy = mtr_policy->sub_policys[domain][0];
16708                 sub_policy_idx = sub_policy->idx;
16709         } else {
16710                 sub_policy = mlx5_ipool_zmalloc
16711                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16712                                  &sub_policy_idx);
16713                 if (!sub_policy ||
16714                     sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM) {
16715                         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++)
16716                                 mlx5_hrxq_release(dev, hrxq_idx[i]);
16717                         goto rss_sub_policy_error;
16718                 }
16719                 sub_policy->idx = sub_policy_idx;
16720                 sub_policy->main_policy = mtr_policy;
16721         }
16722         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
16723                 if (!rss_desc[i])
16724                         continue;
16725                 sub_policy->rix_hrxq[i] = hrxq_idx[i];
16726                 if (mtr_policy->is_hierarchy) {
16727                         act_cnt = &mtr_policy->act_cnt[i];
16728                         act_cnt->next_sub_policy = next_sub_policy;
16729                         mlx5_hrxq_release(dev, hrxq_idx[i]);
16730                 } else {
16731                         /*
16732                          * Overwrite the last action from
16733                          * RSS action to Queue action.
16734                          */
16735                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
16736                                               hrxq_idx[i]);
16737                         if (!hrxq) {
16738                                 DRV_LOG(ERR, "Failed to get policy hrxq");
16739                                 goto rss_sub_policy_error;
16740                         }
16741                         act_cnt = &mtr_policy->act_cnt[i];
16742                         if (act_cnt->rix_mark || act_cnt->modify_hdr) {
16743                                 memset(&dh, 0, sizeof(struct mlx5_flow_handle));
16744                                 if (act_cnt->rix_mark)
16745                                         dh.mark = 1;
16746                                 dh.fate_action = MLX5_FLOW_FATE_QUEUE;
16747                                 dh.rix_hrxq = hrxq_idx[i];
16748                                 flow_drv_rxq_flags_set(dev, &dh);
16749                         }
16750                 }
16751         }
16752         if (__flow_dv_create_policy_acts_rules(dev, mtr_policy,
16753                                                sub_policy, domain)) {
16754                 DRV_LOG(ERR, "Failed to create policy "
16755                         "rules for ingress domain.");
16756                 goto rss_sub_policy_error;
16757         }
16758         if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16759                 i = (mtr_policy->sub_policy_num >>
16760                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16761                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16762                 if (i >= MLX5_MTR_RSS_MAX_SUB_POLICY) {
16763                         DRV_LOG(ERR, "No free sub-policy slot.");
16764                         goto rss_sub_policy_error;
16765                 }
16766                 mtr_policy->sub_policys[domain][i] = sub_policy;
16767                 i++;
16768                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16769                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
16770                 mtr_policy->sub_policy_num |=
16771                         (i & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16772                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
16773         }
16774         rte_spinlock_unlock(&mtr_policy->sl);
16775         *is_reuse = false;
16776         return sub_policy;
16777 rss_sub_policy_error:
16778         if (sub_policy) {
16779                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
16780                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16781                         i = (mtr_policy->sub_policy_num >>
16782                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16783                         MLX5_MTR_SUB_POLICY_NUM_MASK;
16784                         mtr_policy->sub_policys[domain][i] = NULL;
16785                         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16786                                         sub_policy->idx);
16787                 }
16788         }
16789         rte_spinlock_unlock(&mtr_policy->sl);
16790         return NULL;
16791 }
16792
16793 /**
16794  * Find the policy table for prefix table with RSS.
16795  *
16796  * @param[in] dev
16797  *   Pointer to Ethernet device.
16798  * @param[in] mtr_policy
16799  *   Pointer to meter policy table.
16800  * @param[in] rss_desc
16801  *   Pointer to rss_desc
16802  * @return
16803  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
16804  */
16805 static struct mlx5_flow_meter_sub_policy *
16806 flow_dv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
16807                 struct mlx5_flow_meter_policy *mtr_policy,
16808                 struct mlx5_flow_rss_desc *rss_desc[MLX5_MTR_RTE_COLORS])
16809 {
16810         struct mlx5_priv *priv = dev->data->dev_private;
16811         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
16812         struct mlx5_flow_meter_info *next_fm;
16813         struct mlx5_flow_meter_policy *next_policy;
16814         struct mlx5_flow_meter_sub_policy *next_sub_policy = NULL;
16815         struct mlx5_flow_meter_policy *policies[MLX5_MTR_CHAIN_MAX_NUM];
16816         struct mlx5_flow_meter_sub_policy *sub_policies[MLX5_MTR_CHAIN_MAX_NUM];
16817         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
16818         bool reuse_sub_policy;
16819         uint32_t i = 0;
16820         uint32_t j = 0;
16821
16822         while (true) {
16823                 /* Iterate hierarchy to get all policies in this hierarchy. */
16824                 policies[i++] = mtr_policy;
16825                 if (!mtr_policy->is_hierarchy)
16826                         break;
16827                 if (i >= MLX5_MTR_CHAIN_MAX_NUM) {
16828                         DRV_LOG(ERR, "Exceed max meter number in hierarchy.");
16829                         return NULL;
16830                 }
16831                 next_fm = mlx5_flow_meter_find(priv,
16832                         mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
16833                 if (!next_fm) {
16834                         DRV_LOG(ERR, "Failed to get next meter in hierarchy.");
16835                         return NULL;
16836                 }
16837                 next_policy =
16838                         mlx5_flow_meter_policy_find(dev, next_fm->policy_id,
16839                                                     NULL);
16840                 MLX5_ASSERT(next_policy);
16841                 mtr_policy = next_policy;
16842         }
16843         while (i) {
16844                 /**
16845                  * From last policy to the first one in hierarchy,
16846                  * create / get the sub policy for each of them.
16847                  */
16848                 sub_policy = __flow_dv_meter_get_rss_sub_policy(dev,
16849                                                         policies[--i],
16850                                                         rss_desc,
16851                                                         next_sub_policy,
16852                                                         &reuse_sub_policy);
16853                 if (!sub_policy) {
16854                         DRV_LOG(ERR, "Failed to get the sub policy.");
16855                         goto err_exit;
16856                 }
16857                 if (!reuse_sub_policy)
16858                         sub_policies[j++] = sub_policy;
16859                 next_sub_policy = sub_policy;
16860         }
16861         return sub_policy;
16862 err_exit:
16863         while (j) {
16864                 uint16_t sub_policy_num;
16865
16866                 sub_policy = sub_policies[--j];
16867                 mtr_policy = sub_policy->main_policy;
16868                 __flow_dv_destroy_sub_policy_rules(dev, sub_policy);
16869                 if (sub_policy != mtr_policy->sub_policys[domain][0]) {
16870                         sub_policy_num = (mtr_policy->sub_policy_num >>
16871                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
16872                                 MLX5_MTR_SUB_POLICY_NUM_MASK;
16873                         mtr_policy->sub_policys[domain][sub_policy_num - 1] =
16874                                                                         NULL;
16875                         sub_policy_num--;
16876                         mtr_policy->sub_policy_num &=
16877                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
16878                                   (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
16879                         mtr_policy->sub_policy_num |=
16880                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
16881                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
16882                         mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
16883                                         sub_policy->idx);
16884                 }
16885         }
16886         return NULL;
16887 }
16888
16889 /**
16890  * Create the sub policy tag rule for all meters in hierarchy.
16891  *
16892  * @param[in] dev
16893  *   Pointer to Ethernet device.
16894  * @param[in] fm
16895  *   Meter information table.
16896  * @param[in] src_port
16897  *   The src port this extra rule should use.
16898  * @param[in] item
16899  *   The src port match item.
16900  * @param[out] error
16901  *   Perform verbose error reporting if not NULL.
16902  * @return
16903  *   0 on success, a negative errno value otherwise and rte_errno is set.
16904  */
16905 static int
16906 flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev,
16907                                 struct mlx5_flow_meter_info *fm,
16908                                 int32_t src_port,
16909                                 const struct rte_flow_item *item,
16910                                 struct rte_flow_error *error)
16911 {
16912         struct mlx5_priv *priv = dev->data->dev_private;
16913         struct mlx5_flow_meter_policy *mtr_policy;
16914         struct mlx5_flow_meter_sub_policy *sub_policy;
16915         struct mlx5_flow_meter_info *next_fm = NULL;
16916         struct mlx5_flow_meter_policy *next_policy;
16917         struct mlx5_flow_meter_sub_policy *next_sub_policy;
16918         struct mlx5_flow_tbl_data_entry *tbl_data;
16919         struct mlx5_sub_policy_color_rule *color_rule;
16920         struct mlx5_meter_policy_acts acts;
16921         uint32_t color_reg_c_idx;
16922         bool mtr_first = (src_port != UINT16_MAX) ? true : false;
16923         struct rte_flow_attr attr = {
16924                 .group = MLX5_FLOW_TABLE_LEVEL_POLICY,
16925                 .priority = 0,
16926                 .ingress = 0,
16927                 .egress = 0,
16928                 .transfer = 1,
16929                 .reserved = 0,
16930         };
16931         uint32_t domain = MLX5_MTR_DOMAIN_TRANSFER;
16932         int i;
16933
16934         mtr_policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
16935         MLX5_ASSERT(mtr_policy);
16936         if (!mtr_policy->is_hierarchy)
16937                 return 0;
16938         next_fm = mlx5_flow_meter_find(priv,
16939                         mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
16940         if (!next_fm) {
16941                 return rte_flow_error_set(error, EINVAL,
16942                                 RTE_FLOW_ERROR_TYPE_ACTION, NULL,
16943                                 "Failed to find next meter in hierarchy.");
16944         }
16945         if (!next_fm->drop_cnt)
16946                 goto exit;
16947         color_reg_c_idx = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, error);
16948         sub_policy = mtr_policy->sub_policys[domain][0];
16949         for (i = 0; i < RTE_COLORS; i++) {
16950                 bool rule_exist = false;
16951                 struct mlx5_meter_policy_action_container *act_cnt;
16952
16953                 if (i >= RTE_COLOR_YELLOW)
16954                         break;
16955                 TAILQ_FOREACH(color_rule,
16956                               &sub_policy->color_rules[i], next_port)
16957                         if (color_rule->src_port == src_port) {
16958                                 rule_exist = true;
16959                                 break;
16960                         }
16961                 if (rule_exist)
16962                         continue;
16963                 color_rule = mlx5_malloc(MLX5_MEM_ZERO,
16964                                 sizeof(struct mlx5_sub_policy_color_rule),
16965                                 0, SOCKET_ID_ANY);
16966                 if (!color_rule)
16967                         return rte_flow_error_set(error, ENOMEM,
16968                                 RTE_FLOW_ERROR_TYPE_ACTION,
16969                                 NULL, "No memory to create tag color rule.");
16970                 color_rule->src_port = src_port;
16971                 attr.priority = i;
16972                 next_policy = mlx5_flow_meter_policy_find(dev,
16973                                                 next_fm->policy_id, NULL);
16974                 MLX5_ASSERT(next_policy);
16975                 next_sub_policy = next_policy->sub_policys[domain][0];
16976                 tbl_data = container_of(next_sub_policy->tbl_rsc,
16977                                         struct mlx5_flow_tbl_data_entry, tbl);
16978                 act_cnt = &mtr_policy->act_cnt[i];
16979                 if (mtr_first) {
16980                         acts.dv_actions[0] = next_fm->meter_action;
16981                         acts.dv_actions[1] = act_cnt->modify_hdr->action;
16982                 } else {
16983                         acts.dv_actions[0] = act_cnt->modify_hdr->action;
16984                         acts.dv_actions[1] = next_fm->meter_action;
16985                 }
16986                 acts.dv_actions[2] = tbl_data->jump.action;
16987                 acts.actions_n = 3;
16988                 if (mlx5_flow_meter_attach(priv, next_fm, &attr, error)) {
16989                         next_fm = NULL;
16990                         goto err_exit;
16991                 }
16992                 if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx,
16993                                 MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy,
16994                                 &attr, true, item,
16995                                 &color_rule->matcher, error)) {
16996                         rte_flow_error_set(error, errno,
16997                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
16998                                 "Failed to create hierarchy meter matcher.");
16999                         goto err_exit;
17000                 }
17001                 if (__flow_dv_create_policy_flow(dev, color_reg_c_idx,
17002                                         (enum rte_color)i,
17003                                         color_rule->matcher->matcher_object,
17004                                         acts.actions_n, acts.dv_actions,
17005                                         true, item,
17006                                         &color_rule->rule, &attr)) {
17007                         rte_flow_error_set(error, errno,
17008                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
17009                                 "Failed to create hierarchy meter rule.");
17010                         goto err_exit;
17011                 }
17012                 TAILQ_INSERT_TAIL(&sub_policy->color_rules[i],
17013                                   color_rule, next_port);
17014         }
17015 exit:
17016         /**
17017          * Recursive call to iterate all meters in hierarchy and
17018          * create needed rules.
17019          */
17020         return flow_dv_meter_hierarchy_rule_create(dev, next_fm,
17021                                                 src_port, item, error);
17022 err_exit:
17023         if (color_rule) {
17024                 if (color_rule->rule)
17025                         mlx5_flow_os_destroy_flow(color_rule->rule);
17026                 if (color_rule->matcher) {
17027                         struct mlx5_flow_tbl_data_entry *tbl =
17028                                 container_of(color_rule->matcher->tbl,
17029                                                 typeof(*tbl), tbl);
17030                         mlx5_list_unregister(tbl->matchers,
17031                                                 &color_rule->matcher->entry);
17032                 }
17033                 mlx5_free(color_rule);
17034         }
17035         if (next_fm)
17036                 mlx5_flow_meter_detach(priv, next_fm);
17037         return -rte_errno;
17038 }
17039
17040 /**
17041  * Destroy the sub policy table with RX queue.
17042  *
17043  * @param[in] dev
17044  *   Pointer to Ethernet device.
17045  * @param[in] mtr_policy
17046  *   Pointer to meter policy table.
17047  */
17048 static void
17049 flow_dv_destroy_sub_policy_with_rxq(struct rte_eth_dev *dev,
17050                                     struct mlx5_flow_meter_policy *mtr_policy)
17051 {
17052         struct mlx5_priv *priv = dev->data->dev_private;
17053         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
17054         uint32_t domain = MLX5_MTR_DOMAIN_INGRESS;
17055         uint32_t i, j;
17056         uint16_t sub_policy_num, new_policy_num;
17057
17058         rte_spinlock_lock(&mtr_policy->sl);
17059         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
17060                 switch (mtr_policy->act_cnt[i].fate_action) {
17061                 case MLX5_FLOW_FATE_SHARED_RSS:
17062                         sub_policy_num = (mtr_policy->sub_policy_num >>
17063                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain)) &
17064                         MLX5_MTR_SUB_POLICY_NUM_MASK;
17065                         new_policy_num = sub_policy_num;
17066                         for (j = 0; j < sub_policy_num; j++) {
17067                                 sub_policy =
17068                                         mtr_policy->sub_policys[domain][j];
17069                                 if (sub_policy) {
17070                                         __flow_dv_destroy_sub_policy_rules(dev,
17071                                                 sub_policy);
17072                                 if (sub_policy !=
17073                                         mtr_policy->sub_policys[domain][0]) {
17074                                         mtr_policy->sub_policys[domain][j] =
17075                                                                 NULL;
17076                                         mlx5_ipool_free
17077                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
17078                                                 sub_policy->idx);
17079                                                 new_policy_num--;
17080                                         }
17081                                 }
17082                         }
17083                         if (new_policy_num != sub_policy_num) {
17084                                 mtr_policy->sub_policy_num &=
17085                                 ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
17086                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain));
17087                                 mtr_policy->sub_policy_num |=
17088                                 (new_policy_num &
17089                                         MLX5_MTR_SUB_POLICY_NUM_MASK) <<
17090                                 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * domain);
17091                         }
17092                         break;
17093                 case MLX5_FLOW_FATE_QUEUE:
17094                         sub_policy = mtr_policy->sub_policys[domain][0];
17095                         __flow_dv_destroy_sub_policy_rules(dev,
17096                                                            sub_policy);
17097                         break;
17098                 default:
17099                         /*Other actions without queue and do nothing*/
17100                         break;
17101                 }
17102         }
17103         rte_spinlock_unlock(&mtr_policy->sl);
17104 }
17105
17106 /**
17107  * Validate the batch counter support in root table.
17108  *
17109  * Create a simple flow with invalid counter and drop action on root table to
17110  * validate if batch counter with offset on root table is supported or not.
17111  *
17112  * @param[in] dev
17113  *   Pointer to rte_eth_dev structure.
17114  *
17115  * @return
17116  *   0 on success, a negative errno value otherwise and rte_errno is set.
17117  */
17118 int
17119 mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev)
17120 {
17121         struct mlx5_priv *priv = dev->data->dev_private;
17122         struct mlx5_dev_ctx_shared *sh = priv->sh;
17123         struct mlx5_flow_dv_match_params mask = {
17124                 .size = sizeof(mask.buf),
17125         };
17126         struct mlx5_flow_dv_match_params value = {
17127                 .size = sizeof(value.buf),
17128         };
17129         struct mlx5dv_flow_matcher_attr dv_attr = {
17130                 .type = IBV_FLOW_ATTR_NORMAL | IBV_FLOW_ATTR_FLAGS_EGRESS,
17131                 .priority = 0,
17132                 .match_criteria_enable = 0,
17133                 .match_mask = (void *)&mask,
17134         };
17135         void *actions[2] = { 0 };
17136         struct mlx5_flow_tbl_resource *tbl = NULL;
17137         struct mlx5_devx_obj *dcs = NULL;
17138         void *matcher = NULL;
17139         void *flow = NULL;
17140         int ret = -1;
17141
17142         tbl = flow_dv_tbl_resource_get(dev, 0, 1, 0, false, NULL,
17143                                         0, 0, 0, NULL);
17144         if (!tbl)
17145                 goto err;
17146         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
17147         if (!dcs)
17148                 goto err;
17149         ret = mlx5_flow_os_create_flow_action_count(dcs->obj, UINT16_MAX,
17150                                                     &actions[0]);
17151         if (ret)
17152                 goto err;
17153         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
17154         __flow_dv_adjust_buf_size(&mask.size, dv_attr.match_criteria_enable);
17155         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
17156                                                &matcher);
17157         if (ret)
17158                 goto err;
17159         __flow_dv_adjust_buf_size(&value.size, dv_attr.match_criteria_enable);
17160         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 1,
17161                                        actions, &flow);
17162 err:
17163         /*
17164          * If batch counter with offset is not supported, the driver will not
17165          * validate the invalid offset value, flow create should success.
17166          * In this case, it means batch counter is not supported in root table.
17167          *
17168          * Otherwise, if flow create is failed, counter offset is supported.
17169          */
17170         if (flow) {
17171                 DRV_LOG(INFO, "Batch counter is not supported in root "
17172                               "table. Switch to fallback mode.");
17173                 rte_errno = ENOTSUP;
17174                 ret = -rte_errno;
17175                 claim_zero(mlx5_flow_os_destroy_flow(flow));
17176         } else {
17177                 /* Check matcher to make sure validate fail at flow create. */
17178                 if (!matcher || (matcher && errno != EINVAL))
17179                         DRV_LOG(ERR, "Unexpected error in counter offset "
17180                                      "support detection");
17181                 ret = 0;
17182         }
17183         if (actions[0])
17184                 claim_zero(mlx5_flow_os_destroy_flow_action(actions[0]));
17185         if (matcher)
17186                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
17187         if (tbl)
17188                 flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
17189         if (dcs)
17190                 claim_zero(mlx5_devx_cmd_destroy(dcs));
17191         return ret;
17192 }
17193
17194 /**
17195  * Query a devx counter.
17196  *
17197  * @param[in] dev
17198  *   Pointer to the Ethernet device structure.
17199  * @param[in] cnt
17200  *   Index to the flow counter.
17201  * @param[in] clear
17202  *   Set to clear the counter statistics.
17203  * @param[out] pkts
17204  *   The statistics value of packets.
17205  * @param[out] bytes
17206  *   The statistics value of bytes.
17207  *
17208  * @return
17209  *   0 on success, otherwise return -1.
17210  */
17211 static int
17212 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
17213                       uint64_t *pkts, uint64_t *bytes)
17214 {
17215         struct mlx5_priv *priv = dev->data->dev_private;
17216         struct mlx5_flow_counter *cnt;
17217         uint64_t inn_pkts, inn_bytes;
17218         int ret;
17219
17220         if (!priv->config.devx)
17221                 return -1;
17222
17223         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
17224         if (ret)
17225                 return -1;
17226         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
17227         *pkts = inn_pkts - cnt->hits;
17228         *bytes = inn_bytes - cnt->bytes;
17229         if (clear) {
17230                 cnt->hits = inn_pkts;
17231                 cnt->bytes = inn_bytes;
17232         }
17233         return 0;
17234 }
17235
17236 /**
17237  * Get aged-out flows.
17238  *
17239  * @param[in] dev
17240  *   Pointer to the Ethernet device structure.
17241  * @param[in] context
17242  *   The address of an array of pointers to the aged-out flows contexts.
17243  * @param[in] nb_contexts
17244  *   The length of context array pointers.
17245  * @param[out] error
17246  *   Perform verbose error reporting if not NULL. Initialized in case of
17247  *   error only.
17248  *
17249  * @return
17250  *   how many contexts get in success, otherwise negative errno value.
17251  *   if nb_contexts is 0, return the amount of all aged contexts.
17252  *   if nb_contexts is not 0 , return the amount of aged flows reported
17253  *   in the context array.
17254  * @note: only stub for now
17255  */
17256 static int
17257 flow_get_aged_flows(struct rte_eth_dev *dev,
17258                     void **context,
17259                     uint32_t nb_contexts,
17260                     struct rte_flow_error *error)
17261 {
17262         struct mlx5_priv *priv = dev->data->dev_private;
17263         struct mlx5_age_info *age_info;
17264         struct mlx5_age_param *age_param;
17265         struct mlx5_flow_counter *counter;
17266         struct mlx5_aso_age_action *act;
17267         int nb_flows = 0;
17268
17269         if (nb_contexts && !context)
17270                 return rte_flow_error_set(error, EINVAL,
17271                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
17272                                           NULL, "empty context");
17273         age_info = GET_PORT_AGE_INFO(priv);
17274         rte_spinlock_lock(&age_info->aged_sl);
17275         LIST_FOREACH(act, &age_info->aged_aso, next) {
17276                 nb_flows++;
17277                 if (nb_contexts) {
17278                         context[nb_flows - 1] =
17279                                                 act->age_params.context;
17280                         if (!(--nb_contexts))
17281                                 break;
17282                 }
17283         }
17284         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
17285                 nb_flows++;
17286                 if (nb_contexts) {
17287                         age_param = MLX5_CNT_TO_AGE(counter);
17288                         context[nb_flows - 1] = age_param->context;
17289                         if (!(--nb_contexts))
17290                                 break;
17291                 }
17292         }
17293         rte_spinlock_unlock(&age_info->aged_sl);
17294         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
17295         return nb_flows;
17296 }
17297
17298 /*
17299  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
17300  */
17301 static uint32_t
17302 flow_dv_counter_allocate(struct rte_eth_dev *dev)
17303 {
17304         return flow_dv_counter_alloc(dev, 0);
17305 }
17306
17307 /**
17308  * Validate indirect action.
17309  * Dispatcher for action type specific validation.
17310  *
17311  * @param[in] dev
17312  *   Pointer to the Ethernet device structure.
17313  * @param[in] conf
17314  *   Indirect action configuration.
17315  * @param[in] action
17316  *   The indirect action object to validate.
17317  * @param[out] error
17318  *   Perform verbose error reporting if not NULL. Initialized in case of
17319  *   error only.
17320  *
17321  * @return
17322  *   0 on success, otherwise negative errno value.
17323  */
17324 static int
17325 flow_dv_action_validate(struct rte_eth_dev *dev,
17326                         const struct rte_flow_indir_action_conf *conf,
17327                         const struct rte_flow_action *action,
17328                         struct rte_flow_error *err)
17329 {
17330         struct mlx5_priv *priv = dev->data->dev_private;
17331
17332         RTE_SET_USED(conf);
17333         switch (action->type) {
17334         case RTE_FLOW_ACTION_TYPE_RSS:
17335                 /*
17336                  * priv->obj_ops is set according to driver capabilities.
17337                  * When DevX capabilities are
17338                  * sufficient, it is set to devx_obj_ops.
17339                  * Otherwise, it is set to ibv_obj_ops.
17340                  * ibv_obj_ops doesn't support ind_table_modify operation.
17341                  * In this case the indirect RSS action can't be used.
17342                  */
17343                 if (priv->obj_ops.ind_table_modify == NULL)
17344                         return rte_flow_error_set
17345                                         (err, ENOTSUP,
17346                                          RTE_FLOW_ERROR_TYPE_ACTION,
17347                                          NULL,
17348                                          "Indirect RSS action not supported");
17349                 return mlx5_validate_action_rss(dev, action, err);
17350         case RTE_FLOW_ACTION_TYPE_AGE:
17351                 if (!priv->sh->aso_age_mng)
17352                         return rte_flow_error_set(err, ENOTSUP,
17353                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
17354                                                 NULL,
17355                                                 "Indirect age action not supported");
17356                 return flow_dv_validate_action_age(0, action, dev, err);
17357         case RTE_FLOW_ACTION_TYPE_COUNT:
17358                 /*
17359                  * There are two mechanisms to share the action count.
17360                  * The old mechanism uses the shared field to share, while the
17361                  * new mechanism uses the indirect action API.
17362                  * This validation comes to make sure that the two mechanisms
17363                  * are not combined.
17364                  */
17365                 if (is_shared_action_count(action))
17366                         return rte_flow_error_set(err, ENOTSUP,
17367                                                   RTE_FLOW_ERROR_TYPE_ACTION,
17368                                                   NULL,
17369                                                   "Mix shared and indirect counter is not supported");
17370                 return flow_dv_validate_action_count(dev, true, 0, err);
17371         case RTE_FLOW_ACTION_TYPE_CONNTRACK:
17372                 if (!priv->sh->ct_aso_en)
17373                         return rte_flow_error_set(err, ENOTSUP,
17374                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
17375                                         "ASO CT is not supported");
17376                 return mlx5_validate_action_ct(dev, action->conf, err);
17377         default:
17378                 return rte_flow_error_set(err, ENOTSUP,
17379                                           RTE_FLOW_ERROR_TYPE_ACTION,
17380                                           NULL,
17381                                           "action type not supported");
17382         }
17383 }
17384
17385 /*
17386  * Check if the RSS configurations for colors of a meter policy match
17387  * each other, except the queues.
17388  *
17389  * @param[in] r1
17390  *   Pointer to the first RSS flow action.
17391  * @param[in] r2
17392  *   Pointer to the second RSS flow action.
17393  *
17394  * @return
17395  *   0 on match, 1 on conflict.
17396  */
17397 static inline int
17398 flow_dv_mtr_policy_rss_compare(const struct rte_flow_action_rss *r1,
17399                                const struct rte_flow_action_rss *r2)
17400 {
17401         if (!r1 || !r2)
17402                 return 0;
17403         if (r1->func != r2->func || r1->level != r2->level ||
17404             r1->types != r2->types || r1->key_len != r2->key_len ||
17405             memcmp(r1->key, r2->key, r1->key_len))
17406                 return 1;
17407         return 0;
17408 }
17409
17410 /**
17411  * Validate the meter hierarchy chain for meter policy.
17412  *
17413  * @param[in] dev
17414  *   Pointer to the Ethernet device structure.
17415  * @param[in] meter_id
17416  *   Meter id.
17417  * @param[in] action_flags
17418  *   Holds the actions detected until now.
17419  * @param[out] is_rss
17420  *   Is RSS or not.
17421  * @param[out] hierarchy_domain
17422  *   The domain bitmap for hierarchy policy.
17423  * @param[out] error
17424  *   Perform verbose error reporting if not NULL. Initialized in case of
17425  *   error only.
17426  *
17427  * @return
17428  *   0 on success, otherwise negative errno value with error set.
17429  */
17430 static int
17431 flow_dv_validate_policy_mtr_hierarchy(struct rte_eth_dev *dev,
17432                                   uint32_t meter_id,
17433                                   uint64_t action_flags,
17434                                   bool *is_rss,
17435                                   uint8_t *hierarchy_domain,
17436                                   struct rte_mtr_error *error)
17437 {
17438         struct mlx5_priv *priv = dev->data->dev_private;
17439         struct mlx5_flow_meter_info *fm;
17440         struct mlx5_flow_meter_policy *policy;
17441         uint8_t cnt = 1;
17442
17443         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
17444                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
17445                 return -rte_mtr_error_set(error, EINVAL,
17446                                         RTE_MTR_ERROR_TYPE_POLICER_ACTION_GREEN,
17447                                         NULL,
17448                                         "Multiple fate actions not supported.");
17449         while (true) {
17450                 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
17451                 if (!fm)
17452                         return -rte_mtr_error_set(error, EINVAL,
17453                                                 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
17454                                         "Meter not found in meter hierarchy.");
17455                 if (fm->def_policy)
17456                         return -rte_mtr_error_set(error, EINVAL,
17457                                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
17458                         "Non termination meter not supported in hierarchy.");
17459                 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
17460                 MLX5_ASSERT(policy);
17461                 if (!policy->is_hierarchy) {
17462                         if (policy->transfer)
17463                                 *hierarchy_domain |=
17464                                                 MLX5_MTR_DOMAIN_TRANSFER_BIT;
17465                         if (policy->ingress)
17466                                 *hierarchy_domain |=
17467                                                 MLX5_MTR_DOMAIN_INGRESS_BIT;
17468                         if (policy->egress)
17469                                 *hierarchy_domain |= MLX5_MTR_DOMAIN_EGRESS_BIT;
17470                         *is_rss = policy->is_rss;
17471                         break;
17472                 }
17473                 meter_id = policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id;
17474                 if (++cnt >= MLX5_MTR_CHAIN_MAX_NUM)
17475                         return -rte_mtr_error_set(error, EINVAL,
17476                                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
17477                                         "Exceed max hierarchy meter number.");
17478         }
17479         return 0;
17480 }
17481
17482 /**
17483  * Validate meter policy actions.
17484  * Dispatcher for action type specific validation.
17485  *
17486  * @param[in] dev
17487  *   Pointer to the Ethernet device structure.
17488  * @param[in] action
17489  *   The meter policy action object to validate.
17490  * @param[in] attr
17491  *   Attributes of flow to determine steering domain.
17492  * @param[out] error
17493  *   Perform verbose error reporting if not NULL. Initialized in case of
17494  *   error only.
17495  *
17496  * @return
17497  *   0 on success, otherwise negative errno value.
17498  */
17499 static int
17500 flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev,
17501                         const struct rte_flow_action *actions[RTE_COLORS],
17502                         struct rte_flow_attr *attr,
17503                         bool *is_rss,
17504                         uint8_t *domain_bitmap,
17505                         uint8_t *policy_mode,
17506                         struct rte_mtr_error *error)
17507 {
17508         struct mlx5_priv *priv = dev->data->dev_private;
17509         struct mlx5_dev_config *dev_conf = &priv->config;
17510         const struct rte_flow_action *act;
17511         uint64_t action_flags[RTE_COLORS] = {0};
17512         int actions_n;
17513         int i, ret;
17514         struct rte_flow_error flow_err;
17515         uint8_t domain_color[RTE_COLORS] = {0};
17516         uint8_t def_domain = MLX5_MTR_ALL_DOMAIN_BIT;
17517         uint8_t hierarchy_domain = 0;
17518         const struct rte_flow_action_meter *mtr;
17519         bool def_green = false;
17520         bool def_yellow = false;
17521         const struct rte_flow_action_rss *rss_color[RTE_COLORS] = {NULL};
17522
17523         if (!priv->config.dv_esw_en)
17524                 def_domain &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
17525         *domain_bitmap = def_domain;
17526         /* Red color could only support DROP action. */
17527         if (!actions[RTE_COLOR_RED] ||
17528             actions[RTE_COLOR_RED]->type != RTE_FLOW_ACTION_TYPE_DROP)
17529                 return -rte_mtr_error_set(error, ENOTSUP,
17530                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17531                                 NULL, "Red color only supports drop action.");
17532         /*
17533          * Check default policy actions:
17534          * Green / Yellow: no action, Red: drop action
17535          * Either G or Y will trigger default policy actions to be created.
17536          */
17537         if (!actions[RTE_COLOR_GREEN] ||
17538             actions[RTE_COLOR_GREEN]->type == RTE_FLOW_ACTION_TYPE_END)
17539                 def_green = true;
17540         if (!actions[RTE_COLOR_YELLOW] ||
17541             actions[RTE_COLOR_YELLOW]->type == RTE_FLOW_ACTION_TYPE_END)
17542                 def_yellow = true;
17543         if (def_green && def_yellow) {
17544                 *policy_mode = MLX5_MTR_POLICY_MODE_DEF;
17545                 return 0;
17546         } else if (!def_green && def_yellow) {
17547                 *policy_mode = MLX5_MTR_POLICY_MODE_OG;
17548         } else if (def_green && !def_yellow) {
17549                 *policy_mode = MLX5_MTR_POLICY_MODE_OY;
17550         }
17551         /* Set to empty string in case of NULL pointer access by user. */
17552         flow_err.message = "";
17553         for (i = 0; i < RTE_COLORS; i++) {
17554                 act = actions[i];
17555                 for (action_flags[i] = 0, actions_n = 0;
17556                      act && act->type != RTE_FLOW_ACTION_TYPE_END;
17557                      act++) {
17558                         if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
17559                                 return -rte_mtr_error_set(error, ENOTSUP,
17560                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17561                                           NULL, "too many actions");
17562                         switch (act->type) {
17563                         case RTE_FLOW_ACTION_TYPE_PORT_ID:
17564                                 if (!priv->config.dv_esw_en)
17565                                         return -rte_mtr_error_set(error,
17566                                         ENOTSUP,
17567                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17568                                         NULL, "PORT action validate check"
17569                                         " fail for ESW disable");
17570                                 ret = flow_dv_validate_action_port_id(dev,
17571                                                 action_flags[i],
17572                                                 act, attr, &flow_err);
17573                                 if (ret)
17574                                         return -rte_mtr_error_set(error,
17575                                         ENOTSUP,
17576                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17577                                         NULL, flow_err.message ?
17578                                         flow_err.message :
17579                                         "PORT action validate check fail");
17580                                 ++actions_n;
17581                                 action_flags[i] |= MLX5_FLOW_ACTION_PORT_ID;
17582                                 break;
17583                         case RTE_FLOW_ACTION_TYPE_MARK:
17584                                 ret = flow_dv_validate_action_mark(dev, act,
17585                                                            action_flags[i],
17586                                                            attr, &flow_err);
17587                                 if (ret < 0)
17588                                         return -rte_mtr_error_set(error,
17589                                         ENOTSUP,
17590                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17591                                         NULL, flow_err.message ?
17592                                         flow_err.message :
17593                                         "Mark action validate check fail");
17594                                 if (dev_conf->dv_xmeta_en !=
17595                                         MLX5_XMETA_MODE_LEGACY)
17596                                         return -rte_mtr_error_set(error,
17597                                         ENOTSUP,
17598                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17599                                         NULL, "Extend MARK action is "
17600                                         "not supported. Please try use "
17601                                         "default policy for meter.");
17602                                 action_flags[i] |= MLX5_FLOW_ACTION_MARK;
17603                                 ++actions_n;
17604                                 break;
17605                         case RTE_FLOW_ACTION_TYPE_SET_TAG:
17606                                 ret = flow_dv_validate_action_set_tag(dev,
17607                                                         act, action_flags[i],
17608                                                         attr, &flow_err);
17609                                 if (ret)
17610                                         return -rte_mtr_error_set(error,
17611                                         ENOTSUP,
17612                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17613                                         NULL, flow_err.message ?
17614                                         flow_err.message :
17615                                         "Set tag action validate check fail");
17616                                 action_flags[i] |= MLX5_FLOW_ACTION_SET_TAG;
17617                                 ++actions_n;
17618                                 break;
17619                         case RTE_FLOW_ACTION_TYPE_DROP:
17620                                 ret = mlx5_flow_validate_action_drop
17621                                         (action_flags[i], attr, &flow_err);
17622                                 if (ret < 0)
17623                                         return -rte_mtr_error_set(error,
17624                                         ENOTSUP,
17625                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17626                                         NULL, flow_err.message ?
17627                                         flow_err.message :
17628                                         "Drop action validate check fail");
17629                                 action_flags[i] |= MLX5_FLOW_ACTION_DROP;
17630                                 ++actions_n;
17631                                 break;
17632                         case RTE_FLOW_ACTION_TYPE_QUEUE:
17633                                 /*
17634                                  * Check whether extensive
17635                                  * metadata feature is engaged.
17636                                  */
17637                                 if (dev_conf->dv_flow_en &&
17638                                     (dev_conf->dv_xmeta_en !=
17639                                      MLX5_XMETA_MODE_LEGACY) &&
17640                                     mlx5_flow_ext_mreg_supported(dev))
17641                                         return -rte_mtr_error_set(error,
17642                                           ENOTSUP,
17643                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17644                                           NULL, "Queue action with meta "
17645                                           "is not supported. Please try use "
17646                                           "default policy for meter.");
17647                                 ret = mlx5_flow_validate_action_queue(act,
17648                                                         action_flags[i], dev,
17649                                                         attr, &flow_err);
17650                                 if (ret < 0)
17651                                         return -rte_mtr_error_set(error,
17652                                           ENOTSUP,
17653                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17654                                           NULL, flow_err.message ?
17655                                           flow_err.message :
17656                                           "Queue action validate check fail");
17657                                 action_flags[i] |= MLX5_FLOW_ACTION_QUEUE;
17658                                 ++actions_n;
17659                                 break;
17660                         case RTE_FLOW_ACTION_TYPE_RSS:
17661                                 if (dev_conf->dv_flow_en &&
17662                                     (dev_conf->dv_xmeta_en !=
17663                                      MLX5_XMETA_MODE_LEGACY) &&
17664                                     mlx5_flow_ext_mreg_supported(dev))
17665                                         return -rte_mtr_error_set(error,
17666                                           ENOTSUP,
17667                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17668                                           NULL, "RSS action with meta "
17669                                           "is not supported. Please try use "
17670                                           "default policy for meter.");
17671                                 ret = mlx5_validate_action_rss(dev, act,
17672                                                                &flow_err);
17673                                 if (ret < 0)
17674                                         return -rte_mtr_error_set(error,
17675                                           ENOTSUP,
17676                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17677                                           NULL, flow_err.message ?
17678                                           flow_err.message :
17679                                           "RSS action validate check fail");
17680                                 action_flags[i] |= MLX5_FLOW_ACTION_RSS;
17681                                 ++actions_n;
17682                                 /* Either G or Y will set the RSS. */
17683                                 rss_color[i] = act->conf;
17684                                 break;
17685                         case RTE_FLOW_ACTION_TYPE_JUMP:
17686                                 ret = flow_dv_validate_action_jump(dev,
17687                                         NULL, act, action_flags[i],
17688                                         attr, true, &flow_err);
17689                                 if (ret)
17690                                         return -rte_mtr_error_set(error,
17691                                           ENOTSUP,
17692                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17693                                           NULL, flow_err.message ?
17694                                           flow_err.message :
17695                                           "Jump action validate check fail");
17696                                 ++actions_n;
17697                                 action_flags[i] |= MLX5_FLOW_ACTION_JUMP;
17698                                 break;
17699                         /*
17700                          * Only the last meter in the hierarchy will support
17701                          * the YELLOW color steering. Then in the meter policy
17702                          * actions list, there should be no other meter inside.
17703                          */
17704                         case RTE_FLOW_ACTION_TYPE_METER:
17705                                 if (i != RTE_COLOR_GREEN)
17706                                         return -rte_mtr_error_set(error,
17707                                                 ENOTSUP,
17708                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17709                                                 NULL, flow_err.message ?
17710                                                 flow_err.message :
17711                                   "Meter hierarchy only supports GREEN color.");
17712                                 mtr = act->conf;
17713                                 ret = flow_dv_validate_policy_mtr_hierarchy(dev,
17714                                                         mtr->mtr_id,
17715                                                         action_flags[i],
17716                                                         is_rss,
17717                                                         &hierarchy_domain,
17718                                                         error);
17719                                 if (ret)
17720                                         return ret;
17721                                 ++actions_n;
17722                                 action_flags[i] |=
17723                                 MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
17724                                 break;
17725                         default:
17726                                 return -rte_mtr_error_set(error, ENOTSUP,
17727                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17728                                         NULL,
17729                                         "Doesn't support optional action");
17730                         }
17731                 }
17732                 if (action_flags[i] & MLX5_FLOW_ACTION_PORT_ID)
17733                         domain_color[i] = MLX5_MTR_DOMAIN_TRANSFER_BIT;
17734                 else if ((action_flags[i] &
17735                           (MLX5_FLOW_ACTION_RSS | MLX5_FLOW_ACTION_QUEUE)) ||
17736                          (action_flags[i] & MLX5_FLOW_ACTION_MARK))
17737                         /*
17738                          * Only support MLX5_XMETA_MODE_LEGACY
17739                          * so MARK action is only in ingress domain.
17740                          */
17741                         domain_color[i] = MLX5_MTR_DOMAIN_INGRESS_BIT;
17742                 else if (action_flags[i] &
17743                          MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
17744                         domain_color[i] = hierarchy_domain;
17745                 else
17746                         domain_color[i] = def_domain;
17747                 /*
17748                  * Non-termination actions only support NIC Tx domain.
17749                  * The adjustion should be skipped when there is no
17750                  * action or only END is provided. The default domains
17751                  * bit-mask is set to find the MIN intersection.
17752                  * The action flags checking should also be skipped.
17753                  */
17754                 if ((def_green && i == RTE_COLOR_GREEN) ||
17755                     (def_yellow && i == RTE_COLOR_YELLOW))
17756                         continue;
17757                 /*
17758                  * Validate the drop action mutual exclusion
17759                  * with other actions. Drop action is mutually-exclusive
17760                  * with any other action, except for Count action.
17761                  */
17762                 if ((action_flags[i] & MLX5_FLOW_ACTION_DROP) &&
17763                     (action_flags[i] & ~MLX5_FLOW_ACTION_DROP)) {
17764                         return -rte_mtr_error_set(error, ENOTSUP,
17765                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17766                                 NULL, "Drop action is mutually-exclusive "
17767                                 "with any other action");
17768                 }
17769                 /* Eswitch has few restrictions on using items and actions */
17770                 if (domain_color[i] & MLX5_MTR_DOMAIN_TRANSFER_BIT) {
17771                         if (!mlx5_flow_ext_mreg_supported(dev) &&
17772                             action_flags[i] & MLX5_FLOW_ACTION_MARK)
17773                                 return -rte_mtr_error_set(error, ENOTSUP,
17774                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17775                                         NULL, "unsupported action MARK");
17776                         if (action_flags[i] & MLX5_FLOW_ACTION_QUEUE)
17777                                 return -rte_mtr_error_set(error, ENOTSUP,
17778                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17779                                         NULL, "unsupported action QUEUE");
17780                         if (action_flags[i] & MLX5_FLOW_ACTION_RSS)
17781                                 return -rte_mtr_error_set(error, ENOTSUP,
17782                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17783                                         NULL, "unsupported action RSS");
17784                         if (!(action_flags[i] & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
17785                                 return -rte_mtr_error_set(error, ENOTSUP,
17786                                         RTE_MTR_ERROR_TYPE_METER_POLICY,
17787                                         NULL, "no fate action is found");
17788                 } else {
17789                         if (!(action_flags[i] & MLX5_FLOW_FATE_ACTIONS) &&
17790                             (domain_color[i] & MLX5_MTR_DOMAIN_INGRESS_BIT)) {
17791                                 if ((domain_color[i] &
17792                                      MLX5_MTR_DOMAIN_EGRESS_BIT))
17793                                         domain_color[i] =
17794                                                 MLX5_MTR_DOMAIN_EGRESS_BIT;
17795                                 else
17796                                         return -rte_mtr_error_set(error,
17797                                                 ENOTSUP,
17798                                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
17799                                                 NULL,
17800                                                 "no fate action is found");
17801                         }
17802                 }
17803         }
17804         /* If both colors have RSS, the attributes should be the same. */
17805         if (flow_dv_mtr_policy_rss_compare(rss_color[RTE_COLOR_GREEN],
17806                                            rss_color[RTE_COLOR_YELLOW]))
17807                 return -rte_mtr_error_set(error, EINVAL,
17808                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17809                                           NULL, "policy RSS attr conflict");
17810         if (rss_color[RTE_COLOR_GREEN] || rss_color[RTE_COLOR_YELLOW])
17811                 *is_rss = true;
17812         /* "domain_color[C]" is non-zero for each color, default is ALL. */
17813         if (!def_green && !def_yellow &&
17814             domain_color[RTE_COLOR_GREEN] != domain_color[RTE_COLOR_YELLOW] &&
17815             !(action_flags[RTE_COLOR_GREEN] & MLX5_FLOW_ACTION_DROP) &&
17816             !(action_flags[RTE_COLOR_YELLOW] & MLX5_FLOW_ACTION_DROP))
17817                 return -rte_mtr_error_set(error, EINVAL,
17818                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
17819                                           NULL, "policy domains conflict");
17820         /*
17821          * At least one color policy is listed in the actions, the domains
17822          * to be supported should be the intersection.
17823          */
17824         *domain_bitmap = domain_color[RTE_COLOR_GREEN] &
17825                          domain_color[RTE_COLOR_YELLOW];
17826         return 0;
17827 }
17828
17829 static int
17830 flow_dv_sync_domain(struct rte_eth_dev *dev, uint32_t domains, uint32_t flags)
17831 {
17832         struct mlx5_priv *priv = dev->data->dev_private;
17833         int ret = 0;
17834
17835         if ((domains & MLX5_DOMAIN_BIT_NIC_RX) && priv->sh->rx_domain != NULL) {
17836                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->rx_domain,
17837                                                 flags);
17838                 if (ret != 0)
17839                         return ret;
17840         }
17841         if ((domains & MLX5_DOMAIN_BIT_NIC_TX) && priv->sh->tx_domain != NULL) {
17842                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->tx_domain, flags);
17843                 if (ret != 0)
17844                         return ret;
17845         }
17846         if ((domains & MLX5_DOMAIN_BIT_FDB) && priv->sh->fdb_domain != NULL) {
17847                 ret = mlx5_os_flow_dr_sync_domain(priv->sh->fdb_domain, flags);
17848                 if (ret != 0)
17849                         return ret;
17850         }
17851         return 0;
17852 }
17853
17854 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
17855         .validate = flow_dv_validate,
17856         .prepare = flow_dv_prepare,
17857         .translate = flow_dv_translate,
17858         .apply = flow_dv_apply,
17859         .remove = flow_dv_remove,
17860         .destroy = flow_dv_destroy,
17861         .query = flow_dv_query,
17862         .create_mtr_tbls = flow_dv_create_mtr_tbls,
17863         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbls,
17864         .destroy_mtr_drop_tbls = flow_dv_destroy_mtr_drop_tbls,
17865         .create_meter = flow_dv_mtr_alloc,
17866         .free_meter = flow_dv_aso_mtr_release_to_pool,
17867         .validate_mtr_acts = flow_dv_validate_mtr_policy_acts,
17868         .create_mtr_acts = flow_dv_create_mtr_policy_acts,
17869         .destroy_mtr_acts = flow_dv_destroy_mtr_policy_acts,
17870         .create_policy_rules = flow_dv_create_policy_rules,
17871         .destroy_policy_rules = flow_dv_destroy_policy_rules,
17872         .create_def_policy = flow_dv_create_def_policy,
17873         .destroy_def_policy = flow_dv_destroy_def_policy,
17874         .meter_sub_policy_rss_prepare = flow_dv_meter_sub_policy_rss_prepare,
17875         .meter_hierarchy_rule_create = flow_dv_meter_hierarchy_rule_create,
17876         .destroy_sub_policy_with_rxq = flow_dv_destroy_sub_policy_with_rxq,
17877         .counter_alloc = flow_dv_counter_allocate,
17878         .counter_free = flow_dv_counter_free,
17879         .counter_query = flow_dv_counter_query,
17880         .get_aged_flows = flow_get_aged_flows,
17881         .action_validate = flow_dv_action_validate,
17882         .action_create = flow_dv_action_create,
17883         .action_destroy = flow_dv_action_destroy,
17884         .action_update = flow_dv_action_update,
17885         .action_query = flow_dv_action_query,
17886         .sync_domain = flow_dv_sync_domain,
17887 };
17888
17889 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
17890