drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <rte_ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_ip.h>
  19 #include <rte_gre.h>
  20 #include <rte_vxlan.h>
  21 #include <rte_gtp.h>
  22 #include <rte_eal_paging.h>
  23 #include <rte_mpls.h>
  24
  25 #include <mlx5_glue.h>
  26 #include <mlx5_devx_cmds.h>
  27 #include <mlx5_prm.h>
  28 #include <mlx5_malloc.h>
  29
  30 #include "mlx5_defs.h"
  31 #include "mlx5.h"
  32 #include "mlx5_common_os.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_flow_os.h"
  35 #include "mlx5_rxtx.h"
  36
  37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  38
  39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  41 #endif
  42
  43 #ifndef HAVE_MLX5DV_DR_ESWITCH
  44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  46 #endif
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR
  50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  51 #endif
  52
  53 /* VLAN header definitions */
  54 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  55 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  56 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  57 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  58 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  59
  60 union flow_dv_attr {
  61         struct {
  62                 uint32_t valid:1;
  63                 uint32_t ipv4:1;
  64                 uint32_t ipv6:1;
  65                 uint32_t tcp:1;
  66                 uint32_t udp:1;
  67                 uint32_t reserved:27;
  68         };
  69         uint32_t attr;
  70 };
  71
  72 static int
  73 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
  74                              struct mlx5_flow_tbl_resource *tbl);
  75
  76 static int
  77 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev);
  78
  79 static int
  80 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  81                                       uint32_t encap_decap_idx);
  82
  83 static int
  84 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  85                                         uint32_t port_id);
  86
  87 /**
  88  * Initialize flow attributes structure according to flow items' types.
  89  *
  90  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  91  * mode. For tunnel mode, the items to be modified are the outermost ones.
  92  *
  93  * @param[in] item
  94  *   Pointer to item specification.
  95  * @param[out] attr
  96  *   Pointer to flow attributes structure.
  97  * @param[in] dev_flow
  98  *   Pointer to the sub flow.
  99  * @param[in] tunnel_decap
 100  *   Whether action is after tunnel decapsulation.
 101  */
 102 static void
 103 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 104                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 105 {
 106         uint64_t layers = dev_flow->handle->layers;
 107
 108         /*
 109          * If layers is already initialized, it means this dev_flow is the
 110          * suffix flow, the layers flags is set by the prefix flow. Need to
 111          * use the layer flags from prefix flow as the suffix flow may not
 112          * have the user defined items as the flow is split.
 113          */
 114         if (layers) {
 115                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 116                         attr->ipv4 = 1;
 117                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 118                         attr->ipv6 = 1;
 119                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 120                         attr->tcp = 1;
 121                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 122                         attr->udp = 1;
 123                 attr->valid = 1;
 124                 return;
 125         }
 126         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 127                 uint8_t next_protocol = 0xff;
 128                 switch (item->type) {
 129                 case RTE_FLOW_ITEM_TYPE_GRE:
 130                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 131                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 132                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 133                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 134                 case RTE_FLOW_ITEM_TYPE_MPLS:
 135                         if (tunnel_decap)
 136                                 attr->attr = 0;
 137                         break;
 138                 case RTE_FLOW_ITEM_TYPE_IPV4:
 139                         if (!attr->ipv6)
 140                                 attr->ipv4 = 1;
 141                         if (item->mask != NULL &&
 142                             ((const struct rte_flow_item_ipv4 *)
 143                             item->mask)->hdr.next_proto_id)
 144                                 next_protocol =
 145                                     ((const struct rte_flow_item_ipv4 *)
 146                                       (item->spec))->hdr.next_proto_id &
 147                                     ((const struct rte_flow_item_ipv4 *)
 148                                       (item->mask))->hdr.next_proto_id;
 149                         if ((next_protocol == IPPROTO_IPIP ||
 150                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 151                                 attr->attr = 0;
 152                         break;
 153                 case RTE_FLOW_ITEM_TYPE_IPV6:
 154                         if (!attr->ipv4)
 155                                 attr->ipv6 = 1;
 156                         if (item->mask != NULL &&
 157                             ((const struct rte_flow_item_ipv6 *)
 158                             item->mask)->hdr.proto)
 159                                 next_protocol =
 160                                     ((const struct rte_flow_item_ipv6 *)
 161                                       (item->spec))->hdr.proto &
 162                                     ((const struct rte_flow_item_ipv6 *)
 163                                       (item->mask))->hdr.proto;
 164                         if ((next_protocol == IPPROTO_IPIP ||
 165                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 166                                 attr->attr = 0;
 167                         break;
 168                 case RTE_FLOW_ITEM_TYPE_UDP:
 169                         if (!attr->tcp)
 170                                 attr->udp = 1;
 171                         break;
 172                 case RTE_FLOW_ITEM_TYPE_TCP:
 173                         if (!attr->udp)
 174                                 attr->tcp = 1;
 175                         break;
 176                 default:
 177                         break;
 178                 }
 179         }
 180         attr->valid = 1;
 181 }
 182
 183 /**
 184  * Convert rte_mtr_color to mlx5 color.
 185  *
 186  * @param[in] rcol
 187  *   rte_mtr_color.
 188  *
 189  * @return
 190  *   mlx5 color.
 191  */
 192 static int
 193 rte_col_2_mlx5_col(enum rte_color rcol)
 194 {
 195         switch (rcol) {
 196         case RTE_COLOR_GREEN:
 197                 return MLX5_FLOW_COLOR_GREEN;
 198         case RTE_COLOR_YELLOW:
 199                 return MLX5_FLOW_COLOR_YELLOW;
 200         case RTE_COLOR_RED:
 201                 return MLX5_FLOW_COLOR_RED;
 202         default:
 203                 break;
 204         }
 205         return MLX5_FLOW_COLOR_UNDEFINED;
 206 }
 207
 208 struct field_modify_info {
 209         uint32_t size; /* Size of field in protocol header, in bytes. */
 210         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 211         enum mlx5_modification_field id;
 212 };
 213
 214 struct field_modify_info modify_eth[] = {
 215         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 216         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 217         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 218         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 219         {0, 0, 0},
 220 };
 221
 222 struct field_modify_info modify_vlan_out_first_vid[] = {
 223         /* Size in bits !!! */
 224         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 225         {0, 0, 0},
 226 };
 227
 228 struct field_modify_info modify_ipv4[] = {
 229         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 230         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 231         {4, 12, MLX5_MODI_OUT_SIPV4},
 232         {4, 16, MLX5_MODI_OUT_DIPV4},
 233         {0, 0, 0},
 234 };
 235
 236 struct field_modify_info modify_ipv6[] = {
 237         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 238         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 239         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 240         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 241         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 242         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 243         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 244         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 245         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 246         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 247         {0, 0, 0},
 248 };
 249
 250 struct field_modify_info modify_udp[] = {
 251         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 252         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 253         {0, 0, 0},
 254 };
 255
 256 struct field_modify_info modify_tcp[] = {
 257         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 258         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 259         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 260         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 261         {0, 0, 0},
 262 };
 263
 264 static void
 265 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 266                           uint8_t next_protocol, uint64_t *item_flags,
 267                           int *tunnel)
 268 {
 269         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 270                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 271         if (next_protocol == IPPROTO_IPIP) {
 272                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 273                 *tunnel = 1;
 274         }
 275         if (next_protocol == IPPROTO_IPV6) {
 276                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 277                 *tunnel = 1;
 278         }
 279 }
 280
 281 /**
 282  * Acquire the synchronizing object to protect multithreaded access
 283  * to shared dv context. Lock occurs only if context is actually
 284  * shared, i.e. we have multiport IB device and representors are
 285  * created.
 286  *
 287  * @param[in] dev
 288  *   Pointer to the rte_eth_dev structure.
 289  */
 290 static void
 291 flow_dv_shared_lock(struct rte_eth_dev *dev)
 292 {
 293         struct mlx5_priv *priv = dev->data->dev_private;
 294         struct mlx5_dev_ctx_shared *sh = priv->sh;
 295
 296         if (sh->refcnt > 1) {
 297                 int ret;
 298
 299                 ret = pthread_mutex_lock(&sh->dv_mutex);
 300                 MLX5_ASSERT(!ret);
 301                 (void)ret;
 302         }
 303 }
 304
 305 static void
 306 flow_dv_shared_unlock(struct rte_eth_dev *dev)
 307 {
 308         struct mlx5_priv *priv = dev->data->dev_private;
 309         struct mlx5_dev_ctx_shared *sh = priv->sh;
 310
 311         if (sh->refcnt > 1) {
 312                 int ret;
 313
 314                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 315                 MLX5_ASSERT(!ret);
 316                 (void)ret;
 317         }
 318 }
 319
 320 /* Update VLAN's VID/PCP based on input rte_flow_action.
 321  *
 322  * @param[in] action
 323  *   Pointer to struct rte_flow_action.
 324  * @param[out] vlan
 325  *   Pointer to struct rte_vlan_hdr.
 326  */
 327 static void
 328 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 329                          struct rte_vlan_hdr *vlan)
 330 {
 331         uint16_t vlan_tci;
 332         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 333                 vlan_tci =
 334                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 335                                                action->conf)->vlan_pcp;
 336                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 337                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 338                 vlan->vlan_tci |= vlan_tci;
 339         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 340                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 341                 vlan->vlan_tci |= rte_be_to_cpu_16
 342                     (((const struct rte_flow_action_of_set_vlan_vid *)
 343                                              action->conf)->vlan_vid);
 344         }
 345 }
 346
 347 /**
 348  * Fetch 1, 2, 3 or 4 byte field from the byte array
 349  * and return as unsigned integer in host-endian format.
 350  *
 351  * @param[in] data
 352  *   Pointer to data array.
 353  * @param[in] size
 354  *   Size of field to extract.
 355  *
 356  * @return
 357  *   converted field in host endian format.
 358  */
 359 static inline uint32_t
 360 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 361 {
 362         uint32_t ret;
 363
 364         switch (size) {
 365         case 1:
 366                 ret = *data;
 367                 break;
 368         case 2:
 369                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 370                 break;
 371         case 3:
 372                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 373                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 374                 break;
 375         case 4:
 376                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 377                 break;
 378         default:
 379                 MLX5_ASSERT(false);
 380                 ret = 0;
 381                 break;
 382         }
 383         return ret;
 384 }
 385
 386 /**
 387  * Convert modify-header action to DV specification.
 388  *
 389  * Data length of each action is determined by provided field description
 390  * and the item mask. Data bit offset and width of each action is determined
 391  * by provided item mask.
 392  *
 393  * @param[in] item
 394  *   Pointer to item specification.
 395  * @param[in] field
 396  *   Pointer to field modification information.
 397  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 398  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 399  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 400  * @param[in] dcopy
 401  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 402  *   Negative offset value sets the same offset as source offset.
 403  *   size field is ignored, value is taken from source field.
 404  * @param[in,out] resource
 405  *   Pointer to the modify-header resource.
 406  * @param[in] type
 407  *   Type of modification.
 408  * @param[out] error
 409  *   Pointer to the error structure.
 410  *
 411  * @return
 412  *   0 on success, a negative errno value otherwise and rte_errno is set.
 413  */
 414 static int
 415 flow_dv_convert_modify_action(struct rte_flow_item *item,
 416                               struct field_modify_info *field,
 417                               struct field_modify_info *dcopy,
 418                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 419                               uint32_t type, struct rte_flow_error *error)
 420 {
 421         uint32_t i = resource->actions_num;
 422         struct mlx5_modification_cmd *actions = resource->actions;
 423
 424         /*
 425          * The item and mask are provided in big-endian format.
 426          * The fields should be presented as in big-endian format either.
 427          * Mask must be always present, it defines the actual field width.
 428          */
 429         MLX5_ASSERT(item->mask);
 430         MLX5_ASSERT(field->size);
 431         do {
 432                 unsigned int size_b;
 433                 unsigned int off_b;
 434                 uint32_t mask;
 435                 uint32_t data;
 436
 437                 if (i >= MLX5_MAX_MODIFY_NUM)
 438                         return rte_flow_error_set(error, EINVAL,
 439                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 440                                  "too many items to modify");
 441                 /* Fetch variable byte size mask from the array. */
 442                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 443                                            field->offset, field->size);
 444                 if (!mask) {
 445                         ++field;
 446                         continue;
 447                 }
 448                 /* Deduce actual data width in bits from mask value. */
 449                 off_b = rte_bsf32(mask);
 450                 size_b = sizeof(uint32_t) * CHAR_BIT -
 451                          off_b - __builtin_clz(mask);
 452                 MLX5_ASSERT(size_b);
 453                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 454                 actions[i] = (struct mlx5_modification_cmd) {
 455                         .action_type = type,
 456                         .field = field->id,
 457                         .offset = off_b,
 458                         .length = size_b,
 459                 };
 460                 /* Convert entire record to expected big-endian format. */
 461                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 462                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 463                         MLX5_ASSERT(dcopy);
 464                         actions[i].dst_field = dcopy->id;
 465                         actions[i].dst_offset =
 466                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 467                         /* Convert entire record to big-endian format. */
 468                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 469                 } else {
 470                         MLX5_ASSERT(item->spec);
 471                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 472                                                    field->offset, field->size);
 473                         /* Shift out the trailing masked bits from data. */
 474                         data = (data & mask) >> off_b;
 475                         actions[i].data1 = rte_cpu_to_be_32(data);
 476                 }
 477                 ++i;
 478                 ++field;
 479         } while (field->size);
 480         if (resource->actions_num == i)
 481                 return rte_flow_error_set(error, EINVAL,
 482                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 483                                           "invalid modification flow item");
 484         resource->actions_num = i;
 485         return 0;
 486 }
 487
 488 /**
 489  * Convert modify-header set IPv4 address action to DV specification.
 490  *
 491  * @param[in,out] resource
 492  *   Pointer to the modify-header resource.
 493  * @param[in] action
 494  *   Pointer to action specification.
 495  * @param[out] error
 496  *   Pointer to the error structure.
 497  *
 498  * @return
 499  *   0 on success, a negative errno value otherwise and rte_errno is set.
 500  */
 501 static int
 502 flow_dv_convert_action_modify_ipv4
 503                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 504                          const struct rte_flow_action *action,
 505                          struct rte_flow_error *error)
 506 {
 507         const struct rte_flow_action_set_ipv4 *conf =
 508                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 509         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 510         struct rte_flow_item_ipv4 ipv4;
 511         struct rte_flow_item_ipv4 ipv4_mask;
 512
 513         memset(&ipv4, 0, sizeof(ipv4));
 514         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 515         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 516                 ipv4.hdr.src_addr = conf->ipv4_addr;
 517                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 518         } else {
 519                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 520                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 521         }
 522         item.spec = &ipv4;
 523         item.mask = &ipv4_mask;
 524         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 525                                              MLX5_MODIFICATION_TYPE_SET, error);
 526 }
 527
 528 /**
 529  * Convert modify-header set IPv6 address action to DV specification.
 530  *
 531  * @param[in,out] resource
 532  *   Pointer to the modify-header resource.
 533  * @param[in] action
 534  *   Pointer to action specification.
 535  * @param[out] error
 536  *   Pointer to the error structure.
 537  *
 538  * @return
 539  *   0 on success, a negative errno value otherwise and rte_errno is set.
 540  */
 541 static int
 542 flow_dv_convert_action_modify_ipv6
 543                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 544                          const struct rte_flow_action *action,
 545                          struct rte_flow_error *error)
 546 {
 547         const struct rte_flow_action_set_ipv6 *conf =
 548                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 549         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 550         struct rte_flow_item_ipv6 ipv6;
 551         struct rte_flow_item_ipv6 ipv6_mask;
 552
 553         memset(&ipv6, 0, sizeof(ipv6));
 554         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 555         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 556                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 557                        sizeof(ipv6.hdr.src_addr));
 558                 memcpy(&ipv6_mask.hdr.src_addr,
 559                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 560                        sizeof(ipv6.hdr.src_addr));
 561         } else {
 562                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 563                        sizeof(ipv6.hdr.dst_addr));
 564                 memcpy(&ipv6_mask.hdr.dst_addr,
 565                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 566                        sizeof(ipv6.hdr.dst_addr));
 567         }
 568         item.spec = &ipv6;
 569         item.mask = &ipv6_mask;
 570         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 571                                              MLX5_MODIFICATION_TYPE_SET, error);
 572 }
 573
 574 /**
 575  * Convert modify-header set MAC address action to DV specification.
 576  *
 577  * @param[in,out] resource
 578  *   Pointer to the modify-header resource.
 579  * @param[in] action
 580  *   Pointer to action specification.
 581  * @param[out] error
 582  *   Pointer to the error structure.
 583  *
 584  * @return
 585  *   0 on success, a negative errno value otherwise and rte_errno is set.
 586  */
 587 static int
 588 flow_dv_convert_action_modify_mac
 589                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 590                          const struct rte_flow_action *action,
 591                          struct rte_flow_error *error)
 592 {
 593         const struct rte_flow_action_set_mac *conf =
 594                 (const struct rte_flow_action_set_mac *)(action->conf);
 595         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 596         struct rte_flow_item_eth eth;
 597         struct rte_flow_item_eth eth_mask;
 598
 599         memset(&eth, 0, sizeof(eth));
 600         memset(&eth_mask, 0, sizeof(eth_mask));
 601         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 602                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 603                        sizeof(eth.src.addr_bytes));
 604                 memcpy(&eth_mask.src.addr_bytes,
 605                        &rte_flow_item_eth_mask.src.addr_bytes,
 606                        sizeof(eth_mask.src.addr_bytes));
 607         } else {
 608                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 609                        sizeof(eth.dst.addr_bytes));
 610                 memcpy(&eth_mask.dst.addr_bytes,
 611                        &rte_flow_item_eth_mask.dst.addr_bytes,
 612                        sizeof(eth_mask.dst.addr_bytes));
 613         }
 614         item.spec = &eth;
 615         item.mask = &eth_mask;
 616         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 617                                              MLX5_MODIFICATION_TYPE_SET, error);
 618 }
 619
 620 /**
 621  * Convert modify-header set VLAN VID action to DV specification.
 622  *
 623  * @param[in,out] resource
 624  *   Pointer to the modify-header resource.
 625  * @param[in] action
 626  *   Pointer to action specification.
 627  * @param[out] error
 628  *   Pointer to the error structure.
 629  *
 630  * @return
 631  *   0 on success, a negative errno value otherwise and rte_errno is set.
 632  */
 633 static int
 634 flow_dv_convert_action_modify_vlan_vid
 635                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 636                          const struct rte_flow_action *action,
 637                          struct rte_flow_error *error)
 638 {
 639         const struct rte_flow_action_of_set_vlan_vid *conf =
 640                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 641         int i = resource->actions_num;
 642         struct mlx5_modification_cmd *actions = resource->actions;
 643         struct field_modify_info *field = modify_vlan_out_first_vid;
 644
 645         if (i >= MLX5_MAX_MODIFY_NUM)
 646                 return rte_flow_error_set(error, EINVAL,
 647                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 648                          "too many items to modify");
 649         actions[i] = (struct mlx5_modification_cmd) {
 650                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 651                 .field = field->id,
 652                 .length = field->size,
 653                 .offset = field->offset,
 654         };
 655         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 656         actions[i].data1 = conf->vlan_vid;
 657         actions[i].data1 = actions[i].data1 << 16;
 658         resource->actions_num = ++i;
 659         return 0;
 660 }
 661
 662 /**
 663  * Convert modify-header set TP action to DV specification.
 664  *
 665  * @param[in,out] resource
 666  *   Pointer to the modify-header resource.
 667  * @param[in] action
 668  *   Pointer to action specification.
 669  * @param[in] items
 670  *   Pointer to rte_flow_item objects list.
 671  * @param[in] attr
 672  *   Pointer to flow attributes structure.
 673  * @param[in] dev_flow
 674  *   Pointer to the sub flow.
 675  * @param[in] tunnel_decap
 676  *   Whether action is after tunnel decapsulation.
 677  * @param[out] error
 678  *   Pointer to the error structure.
 679  *
 680  * @return
 681  *   0 on success, a negative errno value otherwise and rte_errno is set.
 682  */
 683 static int
 684 flow_dv_convert_action_modify_tp
 685                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 686                          const struct rte_flow_action *action,
 687                          const struct rte_flow_item *items,
 688                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 689                          bool tunnel_decap, struct rte_flow_error *error)
 690 {
 691         const struct rte_flow_action_set_tp *conf =
 692                 (const struct rte_flow_action_set_tp *)(action->conf);
 693         struct rte_flow_item item;
 694         struct rte_flow_item_udp udp;
 695         struct rte_flow_item_udp udp_mask;
 696         struct rte_flow_item_tcp tcp;
 697         struct rte_flow_item_tcp tcp_mask;
 698         struct field_modify_info *field;
 699
 700         if (!attr->valid)
 701                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 702         if (attr->udp) {
 703                 memset(&udp, 0, sizeof(udp));
 704                 memset(&udp_mask, 0, sizeof(udp_mask));
 705                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 706                         udp.hdr.src_port = conf->port;
 707                         udp_mask.hdr.src_port =
 708                                         rte_flow_item_udp_mask.hdr.src_port;
 709                 } else {
 710                         udp.hdr.dst_port = conf->port;
 711                         udp_mask.hdr.dst_port =
 712                                         rte_flow_item_udp_mask.hdr.dst_port;
 713                 }
 714                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 715                 item.spec = &udp;
 716                 item.mask = &udp_mask;
 717                 field = modify_udp;
 718         } else {
 719                 MLX5_ASSERT(attr->tcp);
 720                 memset(&tcp, 0, sizeof(tcp));
 721                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 722                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 723                         tcp.hdr.src_port = conf->port;
 724                         tcp_mask.hdr.src_port =
 725                                         rte_flow_item_tcp_mask.hdr.src_port;
 726                 } else {
 727                         tcp.hdr.dst_port = conf->port;
 728                         tcp_mask.hdr.dst_port =
 729                                         rte_flow_item_tcp_mask.hdr.dst_port;
 730                 }
 731                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 732                 item.spec = &tcp;
 733                 item.mask = &tcp_mask;
 734                 field = modify_tcp;
 735         }
 736         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 737                                              MLX5_MODIFICATION_TYPE_SET, error);
 738 }
 739
 740 /**
 741  * Convert modify-header set TTL action to DV specification.
 742  *
 743  * @param[in,out] resource
 744  *   Pointer to the modify-header resource.
 745  * @param[in] action
 746  *   Pointer to action specification.
 747  * @param[in] items
 748  *   Pointer to rte_flow_item objects list.
 749  * @param[in] attr
 750  *   Pointer to flow attributes structure.
 751  * @param[in] dev_flow
 752  *   Pointer to the sub flow.
 753  * @param[in] tunnel_decap
 754  *   Whether action is after tunnel decapsulation.
 755  * @param[out] error
 756  *   Pointer to the error structure.
 757  *
 758  * @return
 759  *   0 on success, a negative errno value otherwise and rte_errno is set.
 760  */
 761 static int
 762 flow_dv_convert_action_modify_ttl
 763                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 764                          const struct rte_flow_action *action,
 765                          const struct rte_flow_item *items,
 766                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 767                          bool tunnel_decap, struct rte_flow_error *error)
 768 {
 769         const struct rte_flow_action_set_ttl *conf =
 770                 (const struct rte_flow_action_set_ttl *)(action->conf);
 771         struct rte_flow_item item;
 772         struct rte_flow_item_ipv4 ipv4;
 773         struct rte_flow_item_ipv4 ipv4_mask;
 774         struct rte_flow_item_ipv6 ipv6;
 775         struct rte_flow_item_ipv6 ipv6_mask;
 776         struct field_modify_info *field;
 777
 778         if (!attr->valid)
 779                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 780         if (attr->ipv4) {
 781                 memset(&ipv4, 0, sizeof(ipv4));
 782                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 783                 ipv4.hdr.time_to_live = conf->ttl_value;
 784                 ipv4_mask.hdr.time_to_live = 0xFF;
 785                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 786                 item.spec = &ipv4;
 787                 item.mask = &ipv4_mask;
 788                 field = modify_ipv4;
 789         } else {
 790                 MLX5_ASSERT(attr->ipv6);
 791                 memset(&ipv6, 0, sizeof(ipv6));
 792                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 793                 ipv6.hdr.hop_limits = conf->ttl_value;
 794                 ipv6_mask.hdr.hop_limits = 0xFF;
 795                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 796                 item.spec = &ipv6;
 797                 item.mask = &ipv6_mask;
 798                 field = modify_ipv6;
 799         }
 800         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 801                                              MLX5_MODIFICATION_TYPE_SET, error);
 802 }
 803
 804 /**
 805  * Convert modify-header decrement TTL action to DV specification.
 806  *
 807  * @param[in,out] resource
 808  *   Pointer to the modify-header resource.
 809  * @param[in] action
 810  *   Pointer to action specification.
 811  * @param[in] items
 812  *   Pointer to rte_flow_item objects list.
 813  * @param[in] attr
 814  *   Pointer to flow attributes structure.
 815  * @param[in] dev_flow
 816  *   Pointer to the sub flow.
 817  * @param[in] tunnel_decap
 818  *   Whether action is after tunnel decapsulation.
 819  * @param[out] error
 820  *   Pointer to the error structure.
 821  *
 822  * @return
 823  *   0 on success, a negative errno value otherwise and rte_errno is set.
 824  */
 825 static int
 826 flow_dv_convert_action_modify_dec_ttl
 827                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 828                          const struct rte_flow_item *items,
 829                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 830                          bool tunnel_decap, struct rte_flow_error *error)
 831 {
 832         struct rte_flow_item item;
 833         struct rte_flow_item_ipv4 ipv4;
 834         struct rte_flow_item_ipv4 ipv4_mask;
 835         struct rte_flow_item_ipv6 ipv6;
 836         struct rte_flow_item_ipv6 ipv6_mask;
 837         struct field_modify_info *field;
 838
 839         if (!attr->valid)
 840                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 841         if (attr->ipv4) {
 842                 memset(&ipv4, 0, sizeof(ipv4));
 843                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 844                 ipv4.hdr.time_to_live = 0xFF;
 845                 ipv4_mask.hdr.time_to_live = 0xFF;
 846                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 847                 item.spec = &ipv4;
 848                 item.mask = &ipv4_mask;
 849                 field = modify_ipv4;
 850         } else {
 851                 MLX5_ASSERT(attr->ipv6);
 852                 memset(&ipv6, 0, sizeof(ipv6));
 853                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 854                 ipv6.hdr.hop_limits = 0xFF;
 855                 ipv6_mask.hdr.hop_limits = 0xFF;
 856                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 857                 item.spec = &ipv6;
 858                 item.mask = &ipv6_mask;
 859                 field = modify_ipv6;
 860         }
 861         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 862                                              MLX5_MODIFICATION_TYPE_ADD, error);
 863 }
 864
 865 /**
 866  * Convert modify-header increment/decrement TCP Sequence number
 867  * to DV specification.
 868  *
 869  * @param[in,out] resource
 870  *   Pointer to the modify-header resource.
 871  * @param[in] action
 872  *   Pointer to action specification.
 873  * @param[out] error
 874  *   Pointer to the error structure.
 875  *
 876  * @return
 877  *   0 on success, a negative errno value otherwise and rte_errno is set.
 878  */
 879 static int
 880 flow_dv_convert_action_modify_tcp_seq
 881                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 882                          const struct rte_flow_action *action,
 883                          struct rte_flow_error *error)
 884 {
 885         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 886         uint64_t value = rte_be_to_cpu_32(*conf);
 887         struct rte_flow_item item;
 888         struct rte_flow_item_tcp tcp;
 889         struct rte_flow_item_tcp tcp_mask;
 890
 891         memset(&tcp, 0, sizeof(tcp));
 892         memset(&tcp_mask, 0, sizeof(tcp_mask));
 893         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 894                 /*
 895                  * The HW has no decrement operation, only increment operation.
 896                  * To simulate decrement X from Y using increment operation
 897                  * we need to add UINT32_MAX X times to Y.
 898                  * Each adding of UINT32_MAX decrements Y by 1.
 899                  */
 900                 value *= UINT32_MAX;
 901         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 902         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 903         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 904         item.spec = &tcp;
 905         item.mask = &tcp_mask;
 906         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 907                                              MLX5_MODIFICATION_TYPE_ADD, error);
 908 }
 909
 910 /**
 911  * Convert modify-header increment/decrement TCP Acknowledgment number
 912  * to DV specification.
 913  *
 914  * @param[in,out] resource
 915  *   Pointer to the modify-header resource.
 916  * @param[in] action
 917  *   Pointer to action specification.
 918  * @param[out] error
 919  *   Pointer to the error structure.
 920  *
 921  * @return
 922  *   0 on success, a negative errno value otherwise and rte_errno is set.
 923  */
 924 static int
 925 flow_dv_convert_action_modify_tcp_ack
 926                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 927                          const struct rte_flow_action *action,
 928                          struct rte_flow_error *error)
 929 {
 930         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 931         uint64_t value = rte_be_to_cpu_32(*conf);
 932         struct rte_flow_item item;
 933         struct rte_flow_item_tcp tcp;
 934         struct rte_flow_item_tcp tcp_mask;
 935
 936         memset(&tcp, 0, sizeof(tcp));
 937         memset(&tcp_mask, 0, sizeof(tcp_mask));
 938         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 939                 /*
 940                  * The HW has no decrement operation, only increment operation.
 941                  * To simulate decrement X from Y using increment operation
 942                  * we need to add UINT32_MAX X times to Y.
 943                  * Each adding of UINT32_MAX decrements Y by 1.
 944                  */
 945                 value *= UINT32_MAX;
 946         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 947         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 948         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 949         item.spec = &tcp;
 950         item.mask = &tcp_mask;
 951         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 952                                              MLX5_MODIFICATION_TYPE_ADD, error);
 953 }
 954
 955 static enum mlx5_modification_field reg_to_field[] = {
 956         [REG_NON] = MLX5_MODI_OUT_NONE,
 957         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 958         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 959         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 960         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 961         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 962         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 963         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 964         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 965         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 966         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 967 };
 968
 969 /**
 970  * Convert register set to DV specification.
 971  *
 972  * @param[in,out] resource
 973  *   Pointer to the modify-header resource.
 974  * @param[in] action
 975  *   Pointer to action specification.
 976  * @param[out] error
 977  *   Pointer to the error structure.
 978  *
 979  * @return
 980  *   0 on success, a negative errno value otherwise and rte_errno is set.
 981  */
 982 static int
 983 flow_dv_convert_action_set_reg
 984                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 985                          const struct rte_flow_action *action,
 986                          struct rte_flow_error *error)
 987 {
 988         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 989         struct mlx5_modification_cmd *actions = resource->actions;
 990         uint32_t i = resource->actions_num;
 991
 992         if (i >= MLX5_MAX_MODIFY_NUM)
 993                 return rte_flow_error_set(error, EINVAL,
 994                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 995                                           "too many items to modify");
 996         MLX5_ASSERT(conf->id != REG_NON);
 997         MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field));
 998         actions[i] = (struct mlx5_modification_cmd) {
 999                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1000                 .field = reg_to_field[conf->id],
1001         };
1002         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1003         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1004         ++i;
1005         resource->actions_num = i;
1006         return 0;
1007 }
1008
1009 /**
1010  * Convert SET_TAG action to DV specification.
1011  *
1012  * @param[in] dev
1013  *   Pointer to the rte_eth_dev structure.
1014  * @param[in,out] resource
1015  *   Pointer to the modify-header resource.
1016  * @param[in] conf
1017  *   Pointer to action specification.
1018  * @param[out] error
1019  *   Pointer to the error structure.
1020  *
1021  * @return
1022  *   0 on success, a negative errno value otherwise and rte_errno is set.
1023  */
1024 static int
1025 flow_dv_convert_action_set_tag
1026                         (struct rte_eth_dev *dev,
1027                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1028                          const struct rte_flow_action_set_tag *conf,
1029                          struct rte_flow_error *error)
1030 {
1031         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1032         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1033         struct rte_flow_item item = {
1034                 .spec = &data,
1035                 .mask = &mask,
1036         };
1037         struct field_modify_info reg_c_x[] = {
1038                 [1] = {0, 0, 0},
1039         };
1040         enum mlx5_modification_field reg_type;
1041         int ret;
1042
1043         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1044         if (ret < 0)
1045                 return ret;
1046         MLX5_ASSERT(ret != REG_NON);
1047         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1048         reg_type = reg_to_field[ret];
1049         MLX5_ASSERT(reg_type > 0);
1050         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1051         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1052                                              MLX5_MODIFICATION_TYPE_SET, error);
1053 }
1054
1055 /**
1056  * Convert internal COPY_REG action to DV specification.
1057  *
1058  * @param[in] dev
1059  *   Pointer to the rte_eth_dev structure.
1060  * @param[in,out] res
1061  *   Pointer to the modify-header resource.
1062  * @param[in] action
1063  *   Pointer to action specification.
1064  * @param[out] error
1065  *   Pointer to the error structure.
1066  *
1067  * @return
1068  *   0 on success, a negative errno value otherwise and rte_errno is set.
1069  */
1070 static int
1071 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1072                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1073                                  const struct rte_flow_action *action,
1074                                  struct rte_flow_error *error)
1075 {
1076         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1077         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1078         struct rte_flow_item item = {
1079                 .spec = NULL,
1080                 .mask = &mask,
1081         };
1082         struct field_modify_info reg_src[] = {
1083                 {4, 0, reg_to_field[conf->src]},
1084                 {0, 0, 0},
1085         };
1086         struct field_modify_info reg_dst = {
1087                 .offset = 0,
1088                 .id = reg_to_field[conf->dst],
1089         };
1090         /* Adjust reg_c[0] usage according to reported mask. */
1091         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1092                 struct mlx5_priv *priv = dev->data->dev_private;
1093                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1094
1095                 MLX5_ASSERT(reg_c0);
1096                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1097                 if (conf->dst == REG_C_0) {
1098                         /* Copy to reg_c[0], within mask only. */
1099                         reg_dst.offset = rte_bsf32(reg_c0);
1100                         /*
1101                          * Mask is ignoring the enianness, because
1102                          * there is no conversion in datapath.
1103                          */
1104 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1105                         /* Copy from destination lower bits to reg_c[0]. */
1106                         mask = reg_c0 >> reg_dst.offset;
1107 #else
1108                         /* Copy from destination upper bits to reg_c[0]. */
1109                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1110                                           rte_fls_u32(reg_c0));
1111 #endif
1112                 } else {
1113                         mask = rte_cpu_to_be_32(reg_c0);
1114 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1115                         /* Copy from reg_c[0] to destination lower bits. */
1116                         reg_dst.offset = 0;
1117 #else
1118                         /* Copy from reg_c[0] to destination upper bits. */
1119                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1120                                          (rte_fls_u32(reg_c0) -
1121                                           rte_bsf32(reg_c0));
1122 #endif
1123                 }
1124         }
1125         return flow_dv_convert_modify_action(&item,
1126                                              reg_src, &reg_dst, res,
1127                                              MLX5_MODIFICATION_TYPE_COPY,
1128                                              error);
1129 }
1130
1131 /**
1132  * Convert MARK action to DV specification. This routine is used
1133  * in extensive metadata only and requires metadata register to be
1134  * handled. In legacy mode hardware tag resource is engaged.
1135  *
1136  * @param[in] dev
1137  *   Pointer to the rte_eth_dev structure.
1138  * @param[in] conf
1139  *   Pointer to MARK action specification.
1140  * @param[in,out] resource
1141  *   Pointer to the modify-header resource.
1142  * @param[out] error
1143  *   Pointer to the error structure.
1144  *
1145  * @return
1146  *   0 on success, a negative errno value otherwise and rte_errno is set.
1147  */
1148 static int
1149 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1150                             const struct rte_flow_action_mark *conf,
1151                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1152                             struct rte_flow_error *error)
1153 {
1154         struct mlx5_priv *priv = dev->data->dev_private;
1155         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1156                                            priv->sh->dv_mark_mask);
1157         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1158         struct rte_flow_item item = {
1159                 .spec = &data,
1160                 .mask = &mask,
1161         };
1162         struct field_modify_info reg_c_x[] = {
1163                 [1] = {0, 0, 0},
1164         };
1165         int reg;
1166
1167         if (!mask)
1168                 return rte_flow_error_set(error, EINVAL,
1169                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1170                                           NULL, "zero mark action mask");
1171         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1172         if (reg < 0)
1173                 return reg;
1174         MLX5_ASSERT(reg > 0);
1175         if (reg == REG_C_0) {
1176                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1177                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1178
1179                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1180                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1181                 mask = rte_cpu_to_be_32(mask << shl_c0);
1182         }
1183         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1184         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1185                                              MLX5_MODIFICATION_TYPE_SET, error);
1186 }
1187
1188 /**
1189  * Get metadata register index for specified steering domain.
1190  *
1191  * @param[in] dev
1192  *   Pointer to the rte_eth_dev structure.
1193  * @param[in] attr
1194  *   Attributes of flow to determine steering domain.
1195  * @param[out] error
1196  *   Pointer to the error structure.
1197  *
1198  * @return
1199  *   positive index on success, a negative errno value otherwise
1200  *   and rte_errno is set.
1201  */
1202 static enum modify_reg
1203 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1204                          const struct rte_flow_attr *attr,
1205                          struct rte_flow_error *error)
1206 {
1207         int reg =
1208                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1209                                           MLX5_METADATA_FDB :
1210                                             attr->egress ?
1211                                             MLX5_METADATA_TX :
1212                                             MLX5_METADATA_RX, 0, error);
1213         if (reg < 0)
1214                 return rte_flow_error_set(error,
1215                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1216                                           NULL, "unavailable "
1217                                           "metadata register");
1218         return reg;
1219 }
1220
1221 /**
1222  * Convert SET_META action to DV specification.
1223  *
1224  * @param[in] dev
1225  *   Pointer to the rte_eth_dev structure.
1226  * @param[in,out] resource
1227  *   Pointer to the modify-header resource.
1228  * @param[in] attr
1229  *   Attributes of flow that includes this item.
1230  * @param[in] conf
1231  *   Pointer to action specification.
1232  * @param[out] error
1233  *   Pointer to the error structure.
1234  *
1235  * @return
1236  *   0 on success, a negative errno value otherwise and rte_errno is set.
1237  */
1238 static int
1239 flow_dv_convert_action_set_meta
1240                         (struct rte_eth_dev *dev,
1241                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1242                          const struct rte_flow_attr *attr,
1243                          const struct rte_flow_action_set_meta *conf,
1244                          struct rte_flow_error *error)
1245 {
1246         uint32_t data = conf->data;
1247         uint32_t mask = conf->mask;
1248         struct rte_flow_item item = {
1249                 .spec = &data,
1250                 .mask = &mask,
1251         };
1252         struct field_modify_info reg_c_x[] = {
1253                 [1] = {0, 0, 0},
1254         };
1255         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1256
1257         if (reg < 0)
1258                 return reg;
1259         /*
1260          * In datapath code there is no endianness
1261          * coversions for perfromance reasons, all
1262          * pattern conversions are done in rte_flow.
1263          */
1264         if (reg == REG_C_0) {
1265                 struct mlx5_priv *priv = dev->data->dev_private;
1266                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1267                 uint32_t shl_c0;
1268
1269                 MLX5_ASSERT(msk_c0);
1270 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1271                 shl_c0 = rte_bsf32(msk_c0);
1272 #else
1273                 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1274 #endif
1275                 mask <<= shl_c0;
1276                 data <<= shl_c0;
1277                 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1278         }
1279         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1280         /* The routine expects parameters in memory as big-endian ones. */
1281         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1282                                              MLX5_MODIFICATION_TYPE_SET, error);
1283 }
1284
1285 /**
1286  * Convert modify-header set IPv4 DSCP action to DV specification.
1287  *
1288  * @param[in,out] resource
1289  *   Pointer to the modify-header resource.
1290  * @param[in] action
1291  *   Pointer to action specification.
1292  * @param[out] error
1293  *   Pointer to the error structure.
1294  *
1295  * @return
1296  *   0 on success, a negative errno value otherwise and rte_errno is set.
1297  */
1298 static int
1299 flow_dv_convert_action_modify_ipv4_dscp
1300                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1301                          const struct rte_flow_action *action,
1302                          struct rte_flow_error *error)
1303 {
1304         const struct rte_flow_action_set_dscp *conf =
1305                 (const struct rte_flow_action_set_dscp *)(action->conf);
1306         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1307         struct rte_flow_item_ipv4 ipv4;
1308         struct rte_flow_item_ipv4 ipv4_mask;
1309
1310         memset(&ipv4, 0, sizeof(ipv4));
1311         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1312         ipv4.hdr.type_of_service = conf->dscp;
1313         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1314         item.spec = &ipv4;
1315         item.mask = &ipv4_mask;
1316         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1317                                              MLX5_MODIFICATION_TYPE_SET, error);
1318 }
1319
1320 /**
1321  * Convert modify-header set IPv6 DSCP action to DV specification.
1322  *
1323  * @param[in,out] resource
1324  *   Pointer to the modify-header resource.
1325  * @param[in] action
1326  *   Pointer to action specification.
1327  * @param[out] error
1328  *   Pointer to the error structure.
1329  *
1330  * @return
1331  *   0 on success, a negative errno value otherwise and rte_errno is set.
1332  */
1333 static int
1334 flow_dv_convert_action_modify_ipv6_dscp
1335                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1336                          const struct rte_flow_action *action,
1337                          struct rte_flow_error *error)
1338 {
1339         const struct rte_flow_action_set_dscp *conf =
1340                 (const struct rte_flow_action_set_dscp *)(action->conf);
1341         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1342         struct rte_flow_item_ipv6 ipv6;
1343         struct rte_flow_item_ipv6 ipv6_mask;
1344
1345         memset(&ipv6, 0, sizeof(ipv6));
1346         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1347         /*
1348          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1349          * rdma-core only accept the DSCP bits byte aligned start from
1350          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1351          * bits in IPv6 case as rdma-core requires byte aligned value.
1352          */
1353         ipv6.hdr.vtc_flow = conf->dscp;
1354         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1355         item.spec = &ipv6;
1356         item.mask = &ipv6_mask;
1357         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1358                                              MLX5_MODIFICATION_TYPE_SET, error);
1359 }
1360
1361 /**
1362  * Validate MARK item.
1363  *
1364  * @param[in] dev
1365  *   Pointer to the rte_eth_dev structure.
1366  * @param[in] item
1367  *   Item specification.
1368  * @param[in] attr
1369  *   Attributes of flow that includes this item.
1370  * @param[out] error
1371  *   Pointer to error structure.
1372  *
1373  * @return
1374  *   0 on success, a negative errno value otherwise and rte_errno is set.
1375  */
1376 static int
1377 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1378                            const struct rte_flow_item *item,
1379                            const struct rte_flow_attr *attr __rte_unused,
1380                            struct rte_flow_error *error)
1381 {
1382         struct mlx5_priv *priv = dev->data->dev_private;
1383         struct mlx5_dev_config *config = &priv->config;
1384         const struct rte_flow_item_mark *spec = item->spec;
1385         const struct rte_flow_item_mark *mask = item->mask;
1386         const struct rte_flow_item_mark nic_mask = {
1387                 .id = priv->sh->dv_mark_mask,
1388         };
1389         int ret;
1390
1391         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1392                 return rte_flow_error_set(error, ENOTSUP,
1393                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1394                                           "extended metadata feature"
1395                                           " isn't enabled");
1396         if (!mlx5_flow_ext_mreg_supported(dev))
1397                 return rte_flow_error_set(error, ENOTSUP,
1398                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1399                                           "extended metadata register"
1400                                           " isn't supported");
1401         if (!nic_mask.id)
1402                 return rte_flow_error_set(error, ENOTSUP,
1403                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1404                                           "extended metadata register"
1405                                           " isn't available");
1406         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1407         if (ret < 0)
1408                 return ret;
1409         if (!spec)
1410                 return rte_flow_error_set(error, EINVAL,
1411                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1412                                           item->spec,
1413                                           "data cannot be empty");
1414         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1415                 return rte_flow_error_set(error, EINVAL,
1416                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1417                                           &spec->id,
1418                                           "mark id exceeds the limit");
1419         if (!mask)
1420                 mask = &nic_mask;
1421         if (!mask->id)
1422                 return rte_flow_error_set(error, EINVAL,
1423                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1424                                         "mask cannot be zero");
1425
1426         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1427                                         (const uint8_t *)&nic_mask,
1428                                         sizeof(struct rte_flow_item_mark),
1429                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1430         if (ret < 0)
1431                 return ret;
1432         return 0;
1433 }
1434
1435 /**
1436  * Validate META item.
1437  *
1438  * @param[in] dev
1439  *   Pointer to the rte_eth_dev structure.
1440  * @param[in] item
1441  *   Item specification.
1442  * @param[in] attr
1443  *   Attributes of flow that includes this item.
1444  * @param[out] error
1445  *   Pointer to error structure.
1446  *
1447  * @return
1448  *   0 on success, a negative errno value otherwise and rte_errno is set.
1449  */
1450 static int
1451 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1452                            const struct rte_flow_item *item,
1453                            const struct rte_flow_attr *attr,
1454                            struct rte_flow_error *error)
1455 {
1456         struct mlx5_priv *priv = dev->data->dev_private;
1457         struct mlx5_dev_config *config = &priv->config;
1458         const struct rte_flow_item_meta *spec = item->spec;
1459         const struct rte_flow_item_meta *mask = item->mask;
1460         struct rte_flow_item_meta nic_mask = {
1461                 .data = UINT32_MAX
1462         };
1463         int reg;
1464         int ret;
1465
1466         if (!spec)
1467                 return rte_flow_error_set(error, EINVAL,
1468                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1469                                           item->spec,
1470                                           "data cannot be empty");
1471         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1472                 if (!mlx5_flow_ext_mreg_supported(dev))
1473                         return rte_flow_error_set(error, ENOTSUP,
1474                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1475                                           "extended metadata register"
1476                                           " isn't supported");
1477                 reg = flow_dv_get_metadata_reg(dev, attr, error);
1478                 if (reg < 0)
1479                         return reg;
1480                 if (reg == REG_B)
1481                         return rte_flow_error_set(error, ENOTSUP,
1482                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1483                                           "match on reg_b "
1484                                           "isn't supported");
1485                 if (reg != REG_A)
1486                         nic_mask.data = priv->sh->dv_meta_mask;
1487         } else if (attr->transfer) {
1488                 return rte_flow_error_set(error, ENOTSUP,
1489                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1490                                         "extended metadata feature "
1491                                         "should be enabled when "
1492                                         "meta item is requested "
1493                                         "with e-switch mode ");
1494         }
1495         if (!mask)
1496                 mask = &rte_flow_item_meta_mask;
1497         if (!mask->data)
1498                 return rte_flow_error_set(error, EINVAL,
1499                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1500                                         "mask cannot be zero");
1501
1502         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1503                                         (const uint8_t *)&nic_mask,
1504                                         sizeof(struct rte_flow_item_meta),
1505                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1506         return ret;
1507 }
1508
1509 /**
1510  * Validate TAG item.
1511  *
1512  * @param[in] dev
1513  *   Pointer to the rte_eth_dev structure.
1514  * @param[in] item
1515  *   Item specification.
1516  * @param[in] attr
1517  *   Attributes of flow that includes this item.
1518  * @param[out] error
1519  *   Pointer to error structure.
1520  *
1521  * @return
1522  *   0 on success, a negative errno value otherwise and rte_errno is set.
1523  */
1524 static int
1525 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1526                           const struct rte_flow_item *item,
1527                           const struct rte_flow_attr *attr __rte_unused,
1528                           struct rte_flow_error *error)
1529 {
1530         const struct rte_flow_item_tag *spec = item->spec;
1531         const struct rte_flow_item_tag *mask = item->mask;
1532         const struct rte_flow_item_tag nic_mask = {
1533                 .data = RTE_BE32(UINT32_MAX),
1534                 .index = 0xff,
1535         };
1536         int ret;
1537
1538         if (!mlx5_flow_ext_mreg_supported(dev))
1539                 return rte_flow_error_set(error, ENOTSUP,
1540                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1541                                           "extensive metadata register"
1542                                           " isn't supported");
1543         if (!spec)
1544                 return rte_flow_error_set(error, EINVAL,
1545                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1546                                           item->spec,
1547                                           "data cannot be empty");
1548         if (!mask)
1549                 mask = &rte_flow_item_tag_mask;
1550         if (!mask->data)
1551                 return rte_flow_error_set(error, EINVAL,
1552                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1553                                         "mask cannot be zero");
1554
1555         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1556                                         (const uint8_t *)&nic_mask,
1557                                         sizeof(struct rte_flow_item_tag),
1558                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1559         if (ret < 0)
1560                 return ret;
1561         if (mask->index != 0xff)
1562                 return rte_flow_error_set(error, EINVAL,
1563                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1564                                           "partial mask for tag index"
1565                                           " is not supported");
1566         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1567         if (ret < 0)
1568                 return ret;
1569         MLX5_ASSERT(ret != REG_NON);
1570         return 0;
1571 }
1572
1573 /**
1574  * Validate vport item.
1575  *
1576  * @param[in] dev
1577  *   Pointer to the rte_eth_dev structure.
1578  * @param[in] item
1579  *   Item specification.
1580  * @param[in] attr
1581  *   Attributes of flow that includes this item.
1582  * @param[in] item_flags
1583  *   Bit-fields that holds the items detected until now.
1584  * @param[out] error
1585  *   Pointer to error structure.
1586  *
1587  * @return
1588  *   0 on success, a negative errno value otherwise and rte_errno is set.
1589  */
1590 static int
1591 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1592                               const struct rte_flow_item *item,
1593                               const struct rte_flow_attr *attr,
1594                               uint64_t item_flags,
1595                               struct rte_flow_error *error)
1596 {
1597         const struct rte_flow_item_port_id *spec = item->spec;
1598         const struct rte_flow_item_port_id *mask = item->mask;
1599         const struct rte_flow_item_port_id switch_mask = {
1600                         .id = 0xffffffff,
1601         };
1602         struct mlx5_priv *esw_priv;
1603         struct mlx5_priv *dev_priv;
1604         int ret;
1605
1606         if (!attr->transfer)
1607                 return rte_flow_error_set(error, EINVAL,
1608                                           RTE_FLOW_ERROR_TYPE_ITEM,
1609                                           NULL,
1610                                           "match on port id is valid only"
1611                                           " when transfer flag is enabled");
1612         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1613                 return rte_flow_error_set(error, ENOTSUP,
1614                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1615                                           "multiple source ports are not"
1616                                           " supported");
1617         if (!mask)
1618                 mask = &switch_mask;
1619         if (mask->id != 0xffffffff)
1620                 return rte_flow_error_set(error, ENOTSUP,
1621                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1622                                            mask,
1623                                            "no support for partial mask on"
1624                                            " \"id\" field");
1625         ret = mlx5_flow_item_acceptable
1626                                 (item, (const uint8_t *)mask,
1627                                  (const uint8_t *)&rte_flow_item_port_id_mask,
1628                                  sizeof(struct rte_flow_item_port_id),
1629                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1630         if (ret)
1631                 return ret;
1632         if (!spec)
1633                 return 0;
1634         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1635         if (!esw_priv)
1636                 return rte_flow_error_set(error, rte_errno,
1637                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1638                                           "failed to obtain E-Switch info for"
1639                                           " port");
1640         dev_priv = mlx5_dev_to_eswitch_info(dev);
1641         if (!dev_priv)
1642                 return rte_flow_error_set(error, rte_errno,
1643                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1644                                           NULL,
1645                                           "failed to obtain E-Switch info");
1646         if (esw_priv->domain_id != dev_priv->domain_id)
1647                 return rte_flow_error_set(error, EINVAL,
1648                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1649                                           "cannot match on a port from a"
1650                                           " different E-Switch");
1651         return 0;
1652 }
1653
1654 /**
1655  * Validate VLAN item.
1656  *
1657  * @param[in] item
1658  *   Item specification.
1659  * @param[in] item_flags
1660  *   Bit-fields that holds the items detected until now.
1661  * @param[in] dev
1662  *   Ethernet device flow is being created on.
1663  * @param[out] error
1664  *   Pointer to error structure.
1665  *
1666  * @return
1667  *   0 on success, a negative errno value otherwise and rte_errno is set.
1668  */
1669 static int
1670 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
1671                            uint64_t item_flags,
1672                            struct rte_eth_dev *dev,
1673                            struct rte_flow_error *error)
1674 {
1675         const struct rte_flow_item_vlan *mask = item->mask;
1676         const struct rte_flow_item_vlan nic_mask = {
1677                 .tci = RTE_BE16(UINT16_MAX),
1678                 .inner_type = RTE_BE16(UINT16_MAX),
1679                 .has_more_vlan = 1,
1680         };
1681         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1682         int ret;
1683         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1684                                         MLX5_FLOW_LAYER_INNER_L4) :
1685                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1686                                         MLX5_FLOW_LAYER_OUTER_L4);
1687         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1688                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1689
1690         if (item_flags & vlanm)
1691                 return rte_flow_error_set(error, EINVAL,
1692                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1693                                           "multiple VLAN layers not supported");
1694         else if ((item_flags & l34m) != 0)
1695                 return rte_flow_error_set(error, EINVAL,
1696                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1697                                           "VLAN cannot follow L3/L4 layer");
1698         if (!mask)
1699                 mask = &rte_flow_item_vlan_mask;
1700         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1701                                         (const uint8_t *)&nic_mask,
1702                                         sizeof(struct rte_flow_item_vlan),
1703                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1704         if (ret)
1705                 return ret;
1706         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1707                 struct mlx5_priv *priv = dev->data->dev_private;
1708
1709                 if (priv->vmwa_context) {
1710                         /*
1711                          * Non-NULL context means we have a virtual machine
1712                          * and SR-IOV enabled, we have to create VLAN interface
1713                          * to make hypervisor to setup E-Switch vport
1714                          * context correctly. We avoid creating the multiple
1715                          * VLAN interfaces, so we cannot support VLAN tag mask.
1716                          */
1717                         return rte_flow_error_set(error, EINVAL,
1718                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1719                                                   item,
1720                                                   "VLAN tag mask is not"
1721                                                   " supported in virtual"
1722                                                   " environment");
1723                 }
1724         }
1725         return 0;
1726 }
1727
1728 /*
1729  * GTP flags are contained in 1 byte of the format:
1730  * -------------------------------------------
1731  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
1732  * |-----------------------------------------|
1733  * | value | Version | PT | Res | E | S | PN |
1734  * -------------------------------------------
1735  *
1736  * Matching is supported only for GTP flags E, S, PN.
1737  */
1738 #define MLX5_GTP_FLAGS_MASK     0x07
1739
1740 /**
1741  * Validate GTP item.
1742  *
1743  * @param[in] dev
1744  *   Pointer to the rte_eth_dev structure.
1745  * @param[in] item
1746  *   Item specification.
1747  * @param[in] item_flags
1748  *   Bit-fields that holds the items detected until now.
1749  * @param[out] error
1750  *   Pointer to error structure.
1751  *
1752  * @return
1753  *   0 on success, a negative errno value otherwise and rte_errno is set.
1754  */
1755 static int
1756 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
1757                           const struct rte_flow_item *item,
1758                           uint64_t item_flags,
1759                           struct rte_flow_error *error)
1760 {
1761         struct mlx5_priv *priv = dev->data->dev_private;
1762         const struct rte_flow_item_gtp *spec = item->spec;
1763         const struct rte_flow_item_gtp *mask = item->mask;
1764         const struct rte_flow_item_gtp nic_mask = {
1765                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
1766                 .msg_type = 0xff,
1767                 .teid = RTE_BE32(0xffffffff),
1768         };
1769
1770         if (!priv->config.hca_attr.tunnel_stateless_gtp)
1771                 return rte_flow_error_set(error, ENOTSUP,
1772                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1773                                           "GTP support is not enabled");
1774         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1775                 return rte_flow_error_set(error, ENOTSUP,
1776                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1777                                           "multiple tunnel layers not"
1778                                           " supported");
1779         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1780                 return rte_flow_error_set(error, EINVAL,
1781                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1782                                           "no outer UDP layer found");
1783         if (!mask)
1784                 mask = &rte_flow_item_gtp_mask;
1785         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
1786                 return rte_flow_error_set(error, ENOTSUP,
1787                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1788                                           "Match is supported for GTP"
1789                                           " flags only");
1790         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1791                                          (const uint8_t *)&nic_mask,
1792                                          sizeof(struct rte_flow_item_gtp),
1793                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1794 }
1795
1796 /**
1797  * Validate IPV4 item.
1798  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
1799  * add specific validation of fragment_offset field,
1800  *
1801  * @param[in] item
1802  *   Item specification.
1803  * @param[in] item_flags
1804  *   Bit-fields that holds the items detected until now.
1805  * @param[out] error
1806  *   Pointer to error structure.
1807  *
1808  * @return
1809  *   0 on success, a negative errno value otherwise and rte_errno is set.
1810  */
1811 static int
1812 flow_dv_validate_item_ipv4(const struct rte_flow_item *item,
1813                            uint64_t item_flags,
1814                            uint64_t last_item,
1815                            uint16_t ether_type,
1816                            struct rte_flow_error *error)
1817 {
1818         int ret;
1819         const struct rte_flow_item_ipv4 *spec = item->spec;
1820         const struct rte_flow_item_ipv4 *last = item->last;
1821         const struct rte_flow_item_ipv4 *mask = item->mask;
1822         rte_be16_t fragment_offset_spec = 0;
1823         rte_be16_t fragment_offset_last = 0;
1824         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
1825                 .hdr = {
1826                         .src_addr = RTE_BE32(0xffffffff),
1827                         .dst_addr = RTE_BE32(0xffffffff),
1828                         .type_of_service = 0xff,
1829                         .fragment_offset = RTE_BE16(0xffff),
1830                         .next_proto_id = 0xff,
1831                         .time_to_live = 0xff,
1832                 },
1833         };
1834
1835         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
1836                                            ether_type, &nic_ipv4_mask,
1837                                            MLX5_ITEM_RANGE_ACCEPTED, error);
1838         if (ret < 0)
1839                 return ret;
1840         if (spec && mask)
1841                 fragment_offset_spec = spec->hdr.fragment_offset &
1842                                        mask->hdr.fragment_offset;
1843         if (!fragment_offset_spec)
1844                 return 0;
1845         /*
1846          * spec and mask are valid, enforce using full mask to make sure the
1847          * complete value is used correctly.
1848          */
1849         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1850                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1851                 return rte_flow_error_set(error, EINVAL,
1852                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1853                                           item, "must use full mask for"
1854                                           " fragment_offset");
1855         /*
1856          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
1857          * indicating this is 1st fragment of fragmented packet.
1858          * This is not yet supported in MLX5, return appropriate error message.
1859          */
1860         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
1861                 return rte_flow_error_set(error, ENOTSUP,
1862                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1863                                           "match on first fragment not "
1864                                           "supported");
1865         if (fragment_offset_spec && !last)
1866                 return rte_flow_error_set(error, ENOTSUP,
1867                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1868                                           "specified value not supported");
1869         /* spec and last are valid, validate the specified range. */
1870         fragment_offset_last = last->hdr.fragment_offset &
1871                                mask->hdr.fragment_offset;
1872         /*
1873          * Match on fragment_offset spec 0x2001 and last 0x3fff
1874          * means MF is 1 and frag-offset is > 0.
1875          * This packet is fragment 2nd and onward, excluding last.
1876          * This is not yet supported in MLX5, return appropriate
1877          * error message.
1878          */
1879         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
1880             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1881                 return rte_flow_error_set(error, ENOTSUP,
1882                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1883                                           last, "match on following "
1884                                           "fragments not supported");
1885         /*
1886          * Match on fragment_offset spec 0x0001 and last 0x1fff
1887          * means MF is 0 and frag-offset is > 0.
1888          * This packet is last fragment of fragmented packet.
1889          * This is not yet supported in MLX5, return appropriate
1890          * error message.
1891          */
1892         if (fragment_offset_spec == RTE_BE16(1) &&
1893             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
1894                 return rte_flow_error_set(error, ENOTSUP,
1895                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1896                                           last, "match on last "
1897                                           "fragment not supported");
1898         /*
1899          * Match on fragment_offset spec 0x0001 and last 0x3fff
1900          * means MF and/or frag-offset is not 0.
1901          * This is a fragmented packet.
1902          * Other range values are invalid and rejected.
1903          */
1904         if (!(fragment_offset_spec == RTE_BE16(1) &&
1905               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
1906                 return rte_flow_error_set(error, ENOTSUP,
1907                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
1908                                           "specified range not supported");
1909         return 0;
1910 }
1911
1912 /**
1913  * Validate IPV6 fragment extension item.
1914  *
1915  * @param[in] item
1916  *   Item specification.
1917  * @param[in] item_flags
1918  *   Bit-fields that holds the items detected until now.
1919  * @param[out] error
1920  *   Pointer to error structure.
1921  *
1922  * @return
1923  *   0 on success, a negative errno value otherwise and rte_errno is set.
1924  */
1925 static int
1926 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
1927                                     uint64_t item_flags,
1928                                     struct rte_flow_error *error)
1929 {
1930         const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
1931         const struct rte_flow_item_ipv6_frag_ext *last = item->last;
1932         const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
1933         rte_be16_t frag_data_spec = 0;
1934         rte_be16_t frag_data_last = 0;
1935         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1936         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1937                                       MLX5_FLOW_LAYER_OUTER_L4;
1938         int ret = 0;
1939         struct rte_flow_item_ipv6_frag_ext nic_mask = {
1940                 .hdr = {
1941                         .next_header = 0xff,
1942                         .frag_data = RTE_BE16(0xffff),
1943                 },
1944         };
1945
1946         if (item_flags & l4m)
1947                 return rte_flow_error_set(error, EINVAL,
1948                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1949                                           "ipv6 fragment extension item cannot "
1950                                           "follow L4 item.");
1951         if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
1952             (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
1953                 return rte_flow_error_set(error, EINVAL,
1954                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1955                                           "ipv6 fragment extension item must "
1956                                           "follow ipv6 item");
1957         if (spec && mask)
1958                 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
1959         if (!frag_data_spec)
1960                 return 0;
1961         /*
1962          * spec and mask are valid, enforce using full mask to make sure the
1963          * complete value is used correctly.
1964          */
1965         if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
1966                                 RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
1967                 return rte_flow_error_set(error, EINVAL,
1968                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1969                                           item, "must use full mask for"
1970                                           " frag_data");
1971         /*
1972          * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
1973          * This is 1st fragment of fragmented packet.
1974          */
1975         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
1976                 return rte_flow_error_set(error, ENOTSUP,
1977                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1978                                           "match on first fragment not "
1979                                           "supported");
1980         if (frag_data_spec && !last)
1981                 return rte_flow_error_set(error, EINVAL,
1982                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1983                                           "specified value not supported");
1984         ret = mlx5_flow_item_acceptable
1985                                 (item, (const uint8_t *)mask,
1986                                  (const uint8_t *)&nic_mask,
1987                                  sizeof(struct rte_flow_item_ipv6_frag_ext),
1988                                  MLX5_ITEM_RANGE_ACCEPTED, error);
1989         if (ret)
1990                 return ret;
1991         /* spec and last are valid, validate the specified range. */
1992         frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
1993         /*
1994          * Match on frag_data spec 0x0009 and last 0xfff9
1995          * means M is 1 and frag-offset is > 0.
1996          * This packet is fragment 2nd and onward, excluding last.
1997          * This is not yet supported in MLX5, return appropriate
1998          * error message.
1999          */
2000         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
2001                                        RTE_IPV6_EHDR_MF_MASK) &&
2002             frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2003                 return rte_flow_error_set(error, ENOTSUP,
2004                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2005                                           last, "match on following "
2006                                           "fragments not supported");
2007         /*
2008          * Match on frag_data spec 0x0008 and last 0xfff8
2009          * means M is 0 and frag-offset is > 0.
2010          * This packet is last fragment of fragmented packet.
2011          * This is not yet supported in MLX5, return appropriate
2012          * error message.
2013          */
2014         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
2015             frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
2016                 return rte_flow_error_set(error, ENOTSUP,
2017                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2018                                           last, "match on last "
2019                                           "fragment not supported");
2020         /* Other range values are invalid and rejected. */
2021         return rte_flow_error_set(error, EINVAL,
2022                                   RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2023                                   "specified range not supported");
2024 }
2025
2026 /**
2027  * Validate the pop VLAN action.
2028  *
2029  * @param[in] dev
2030  *   Pointer to the rte_eth_dev structure.
2031  * @param[in] action_flags
2032  *   Holds the actions detected until now.
2033  * @param[in] action
2034  *   Pointer to the pop vlan action.
2035  * @param[in] item_flags
2036  *   The items found in this flow rule.
2037  * @param[in] attr
2038  *   Pointer to flow attributes.
2039  * @param[out] error
2040  *   Pointer to error structure.
2041  *
2042  * @return
2043  *   0 on success, a negative errno value otherwise and rte_errno is set.
2044  */
2045 static int
2046 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
2047                                  uint64_t action_flags,
2048                                  const struct rte_flow_action *action,
2049                                  uint64_t item_flags,
2050                                  const struct rte_flow_attr *attr,
2051                                  struct rte_flow_error *error)
2052 {
2053         const struct mlx5_priv *priv = dev->data->dev_private;
2054
2055         (void)action;
2056         (void)attr;
2057         if (!priv->sh->pop_vlan_action)
2058                 return rte_flow_error_set(error, ENOTSUP,
2059                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2060                                           NULL,
2061                                           "pop vlan action is not supported");
2062         if (attr->egress)
2063                 return rte_flow_error_set(error, ENOTSUP,
2064                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2065                                           NULL,
2066                                           "pop vlan action not supported for "
2067                                           "egress");
2068         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
2069                 return rte_flow_error_set(error, ENOTSUP,
2070                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2071                                           "no support for multiple VLAN "
2072                                           "actions");
2073         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
2074         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
2075             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
2076                 return rte_flow_error_set(error, ENOTSUP,
2077                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2078                                           NULL,
2079                                           "cannot pop vlan after decap without "
2080                                           "match on inner vlan in the flow");
2081         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
2082         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
2083             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2084                 return rte_flow_error_set(error, ENOTSUP,
2085                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2086                                           NULL,
2087                                           "cannot pop vlan without a "
2088                                           "match on (outer) vlan in the flow");
2089         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2090                 return rte_flow_error_set(error, EINVAL,
2091                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2092                                           "wrong action order, port_id should "
2093                                           "be after pop VLAN action");
2094         if (!attr->transfer && priv->representor)
2095                 return rte_flow_error_set(error, ENOTSUP,
2096                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2097                                           "pop vlan action for VF representor "
2098                                           "not supported on NIC table");
2099         return 0;
2100 }
2101
2102 /**
2103  * Get VLAN default info from vlan match info.
2104  *
2105  * @param[in] items
2106  *   the list of item specifications.
2107  * @param[out] vlan
2108  *   pointer VLAN info to fill to.
2109  *
2110  * @return
2111  *   0 on success, a negative errno value otherwise and rte_errno is set.
2112  */
2113 static void
2114 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2115                                   struct rte_vlan_hdr *vlan)
2116 {
2117         const struct rte_flow_item_vlan nic_mask = {
2118                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2119                                 MLX5DV_FLOW_VLAN_VID_MASK),
2120                 .inner_type = RTE_BE16(0xffff),
2121         };
2122
2123         if (items == NULL)
2124                 return;
2125         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2126                 int type = items->type;
2127
2128                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2129                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2130                         break;
2131         }
2132         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2133                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2134                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2135
2136                 /* If VLAN item in pattern doesn't contain data, return here. */
2137                 if (!vlan_v)
2138                         return;
2139                 if (!vlan_m)
2140                         vlan_m = &nic_mask;
2141                 /* Only full match values are accepted */
2142                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2143                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2144                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2145                         vlan->vlan_tci |=
2146                                 rte_be_to_cpu_16(vlan_v->tci &
2147                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2148                 }
2149                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2150                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2151                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2152                         vlan->vlan_tci |=
2153                                 rte_be_to_cpu_16(vlan_v->tci &
2154                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2155                 }
2156                 if (vlan_m->inner_type == nic_mask.inner_type)
2157                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2158                                                            vlan_m->inner_type);
2159         }
2160 }
2161
2162 /**
2163  * Validate the push VLAN action.
2164  *
2165  * @param[in] dev
2166  *   Pointer to the rte_eth_dev structure.
2167  * @param[in] action_flags
2168  *   Holds the actions detected until now.
2169  * @param[in] item_flags
2170  *   The items found in this flow rule.
2171  * @param[in] action
2172  *   Pointer to the action structure.
2173  * @param[in] attr
2174  *   Pointer to flow attributes
2175  * @param[out] error
2176  *   Pointer to error structure.
2177  *
2178  * @return
2179  *   0 on success, a negative errno value otherwise and rte_errno is set.
2180  */
2181 static int
2182 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2183                                   uint64_t action_flags,
2184                                   const struct rte_flow_item_vlan *vlan_m,
2185                                   const struct rte_flow_action *action,
2186                                   const struct rte_flow_attr *attr,
2187                                   struct rte_flow_error *error)
2188 {
2189         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2190         const struct mlx5_priv *priv = dev->data->dev_private;
2191
2192         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2193             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2194                 return rte_flow_error_set(error, EINVAL,
2195                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2196                                           "invalid vlan ethertype");
2197         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2198                 return rte_flow_error_set(error, EINVAL,
2199                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2200                                           "wrong action order, port_id should "
2201                                           "be after push VLAN");
2202         if (!attr->transfer && priv->representor)
2203                 return rte_flow_error_set(error, ENOTSUP,
2204                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2205                                           "push vlan action for VF representor "
2206                                           "not supported on NIC table");
2207         if (vlan_m &&
2208             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2209             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2210                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2211             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2212             !(mlx5_flow_find_action
2213                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2214                 return rte_flow_error_set(error, EINVAL,
2215                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2216                                           "not full match mask on VLAN PCP and "
2217                                           "there is no of_set_vlan_pcp action, "
2218                                           "push VLAN action cannot figure out "
2219                                           "PCP value");
2220         if (vlan_m &&
2221             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2222             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2223                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2224             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2225             !(mlx5_flow_find_action
2226                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2227                 return rte_flow_error_set(error, EINVAL,
2228                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2229                                           "not full match mask on VLAN VID and "
2230                                           "there is no of_set_vlan_vid action, "
2231                                           "push VLAN action cannot figure out "
2232                                           "VID value");
2233         (void)attr;
2234         return 0;
2235 }
2236
2237 /**
2238  * Validate the set VLAN PCP.
2239  *
2240  * @param[in] action_flags
2241  *   Holds the actions detected until now.
2242  * @param[in] actions
2243  *   Pointer to the list of actions remaining in the flow rule.
2244  * @param[out] error
2245  *   Pointer to error structure.
2246  *
2247  * @return
2248  *   0 on success, a negative errno value otherwise and rte_errno is set.
2249  */
2250 static int
2251 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2252                                      const struct rte_flow_action actions[],
2253                                      struct rte_flow_error *error)
2254 {
2255         const struct rte_flow_action *action = actions;
2256         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2257
2258         if (conf->vlan_pcp > 7)
2259                 return rte_flow_error_set(error, EINVAL,
2260                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2261                                           "VLAN PCP value is too big");
2262         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2263                 return rte_flow_error_set(error, ENOTSUP,
2264                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2265                                           "set VLAN PCP action must follow "
2266                                           "the push VLAN action");
2267         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2268                 return rte_flow_error_set(error, ENOTSUP,
2269                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2270                                           "Multiple VLAN PCP modification are "
2271                                           "not supported");
2272         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2273                 return rte_flow_error_set(error, EINVAL,
2274                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2275                                           "wrong action order, port_id should "
2276                                           "be after set VLAN PCP");
2277         return 0;
2278 }
2279
2280 /**
2281  * Validate the set VLAN VID.
2282  *
2283  * @param[in] item_flags
2284  *   Holds the items detected in this rule.
2285  * @param[in] action_flags
2286  *   Holds the actions detected until now.
2287  * @param[in] actions
2288  *   Pointer to the list of actions remaining in the flow rule.
2289  * @param[out] error
2290  *   Pointer to error structure.
2291  *
2292  * @return
2293  *   0 on success, a negative errno value otherwise and rte_errno is set.
2294  */
2295 static int
2296 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2297                                      uint64_t action_flags,
2298                                      const struct rte_flow_action actions[],
2299                                      struct rte_flow_error *error)
2300 {
2301         const struct rte_flow_action *action = actions;
2302         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2303
2304         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2305                 return rte_flow_error_set(error, EINVAL,
2306                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2307                                           "VLAN VID value is too big");
2308         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2309             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2310                 return rte_flow_error_set(error, ENOTSUP,
2311                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2312                                           "set VLAN VID action must follow push"
2313                                           " VLAN action or match on VLAN item");
2314         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2315                 return rte_flow_error_set(error, ENOTSUP,
2316                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2317                                           "Multiple VLAN VID modifications are "
2318                                           "not supported");
2319         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2320                 return rte_flow_error_set(error, EINVAL,
2321                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2322                                           "wrong action order, port_id should "
2323                                           "be after set VLAN VID");
2324         return 0;
2325 }
2326
2327 /*
2328  * Validate the FLAG action.
2329  *
2330  * @param[in] dev
2331  *   Pointer to the rte_eth_dev structure.
2332  * @param[in] action_flags
2333  *   Holds the actions detected until now.
2334  * @param[in] attr
2335  *   Pointer to flow attributes
2336  * @param[out] error
2337  *   Pointer to error structure.
2338  *
2339  * @return
2340  *   0 on success, a negative errno value otherwise and rte_errno is set.
2341  */
2342 static int
2343 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
2344                              uint64_t action_flags,
2345                              const struct rte_flow_attr *attr,
2346                              struct rte_flow_error *error)
2347 {
2348         struct mlx5_priv *priv = dev->data->dev_private;
2349         struct mlx5_dev_config *config = &priv->config;
2350         int ret;
2351
2352         /* Fall back if no extended metadata register support. */
2353         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2354                 return mlx5_flow_validate_action_flag(action_flags, attr,
2355                                                       error);
2356         /* Extensive metadata mode requires registers. */
2357         if (!mlx5_flow_ext_mreg_supported(dev))
2358                 return rte_flow_error_set(error, ENOTSUP,
2359                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2360                                           "no metadata registers "
2361                                           "to support flag action");
2362         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
2363                 return rte_flow_error_set(error, ENOTSUP,
2364                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2365                                           "extended metadata register"
2366                                           " isn't available");
2367         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2368         if (ret < 0)
2369                 return ret;
2370         MLX5_ASSERT(ret > 0);
2371         if (action_flags & MLX5_FLOW_ACTION_MARK)
2372                 return rte_flow_error_set(error, EINVAL,
2373                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2374                                           "can't mark and flag in same flow");
2375         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2376                 return rte_flow_error_set(error, EINVAL,
2377                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2378                                           "can't have 2 flag"
2379                                           " actions in same flow");
2380         return 0;
2381 }
2382
2383 /**
2384  * Validate MARK action.
2385  *
2386  * @param[in] dev
2387  *   Pointer to the rte_eth_dev structure.
2388  * @param[in] action
2389  *   Pointer to action.
2390  * @param[in] action_flags
2391  *   Holds the actions detected until now.
2392  * @param[in] attr
2393  *   Pointer to flow attributes
2394  * @param[out] error
2395  *   Pointer to error structure.
2396  *
2397  * @return
2398  *   0 on success, a negative errno value otherwise and rte_errno is set.
2399  */
2400 static int
2401 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
2402                              const struct rte_flow_action *action,
2403                              uint64_t action_flags,
2404                              const struct rte_flow_attr *attr,
2405                              struct rte_flow_error *error)
2406 {
2407         struct mlx5_priv *priv = dev->data->dev_private;
2408         struct mlx5_dev_config *config = &priv->config;
2409         const struct rte_flow_action_mark *mark = action->conf;
2410         int ret;
2411
2412         /* Fall back if no extended metadata register support. */
2413         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2414                 return mlx5_flow_validate_action_mark(action, action_flags,
2415                                                       attr, error);
2416         /* Extensive metadata mode requires registers. */
2417         if (!mlx5_flow_ext_mreg_supported(dev))
2418                 return rte_flow_error_set(error, ENOTSUP,
2419                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2420                                           "no metadata registers "
2421                                           "to support mark action");
2422         if (!priv->sh->dv_mark_mask)
2423                 return rte_flow_error_set(error, ENOTSUP,
2424                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2425                                           "extended metadata register"
2426                                           " isn't available");
2427         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2428         if (ret < 0)
2429                 return ret;
2430         MLX5_ASSERT(ret > 0);
2431         if (!mark)
2432                 return rte_flow_error_set(error, EINVAL,
2433                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2434                                           "configuration cannot be null");
2435         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
2436                 return rte_flow_error_set(error, EINVAL,
2437                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2438                                           &mark->id,
2439                                           "mark id exceeds the limit");
2440         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2441                 return rte_flow_error_set(error, EINVAL,
2442                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2443                                           "can't flag and mark in same flow");
2444         if (action_flags & MLX5_FLOW_ACTION_MARK)
2445                 return rte_flow_error_set(error, EINVAL,
2446                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2447                                           "can't have 2 mark actions in same"
2448                                           " flow");
2449         return 0;
2450 }
2451
2452 /**
2453  * Validate SET_META action.
2454  *
2455  * @param[in] dev
2456  *   Pointer to the rte_eth_dev structure.
2457  * @param[in] action
2458  *   Pointer to the action structure.
2459  * @param[in] action_flags
2460  *   Holds the actions detected until now.
2461  * @param[in] attr
2462  *   Pointer to flow attributes
2463  * @param[out] error
2464  *   Pointer to error structure.
2465  *
2466  * @return
2467  *   0 on success, a negative errno value otherwise and rte_errno is set.
2468  */
2469 static int
2470 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
2471                                  const struct rte_flow_action *action,
2472                                  uint64_t action_flags __rte_unused,
2473                                  const struct rte_flow_attr *attr,
2474                                  struct rte_flow_error *error)
2475 {
2476         const struct rte_flow_action_set_meta *conf;
2477         uint32_t nic_mask = UINT32_MAX;
2478         int reg;
2479
2480         if (!mlx5_flow_ext_mreg_supported(dev))
2481                 return rte_flow_error_set(error, ENOTSUP,
2482                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2483                                           "extended metadata register"
2484                                           " isn't supported");
2485         reg = flow_dv_get_metadata_reg(dev, attr, error);
2486         if (reg < 0)
2487                 return reg;
2488         if (reg != REG_A && reg != REG_B) {
2489                 struct mlx5_priv *priv = dev->data->dev_private;
2490
2491                 nic_mask = priv->sh->dv_meta_mask;
2492         }
2493         if (!(action->conf))
2494                 return rte_flow_error_set(error, EINVAL,
2495                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2496                                           "configuration cannot be null");
2497         conf = (const struct rte_flow_action_set_meta *)action->conf;
2498         if (!conf->mask)
2499                 return rte_flow_error_set(error, EINVAL,
2500                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2501                                           "zero mask doesn't have any effect");
2502         if (conf->mask & ~nic_mask)
2503                 return rte_flow_error_set(error, EINVAL,
2504                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2505                                           "meta data must be within reg C0");
2506         return 0;
2507 }
2508
2509 /**
2510  * Validate SET_TAG action.
2511  *
2512  * @param[in] dev
2513  *   Pointer to the rte_eth_dev structure.
2514  * @param[in] action
2515  *   Pointer to the action structure.
2516  * @param[in] action_flags
2517  *   Holds the actions detected until now.
2518  * @param[in] attr
2519  *   Pointer to flow attributes
2520  * @param[out] error
2521  *   Pointer to error structure.
2522  *
2523  * @return
2524  *   0 on success, a negative errno value otherwise and rte_errno is set.
2525  */
2526 static int
2527 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
2528                                 const struct rte_flow_action *action,
2529                                 uint64_t action_flags,
2530                                 const struct rte_flow_attr *attr,
2531                                 struct rte_flow_error *error)
2532 {
2533         const struct rte_flow_action_set_tag *conf;
2534         const uint64_t terminal_action_flags =
2535                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
2536                 MLX5_FLOW_ACTION_RSS;
2537         int ret;
2538
2539         if (!mlx5_flow_ext_mreg_supported(dev))
2540                 return rte_flow_error_set(error, ENOTSUP,
2541                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2542                                           "extensive metadata register"
2543                                           " isn't supported");
2544         if (!(action->conf))
2545                 return rte_flow_error_set(error, EINVAL,
2546                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2547                                           "configuration cannot be null");
2548         conf = (const struct rte_flow_action_set_tag *)action->conf;
2549         if (!conf->mask)
2550                 return rte_flow_error_set(error, EINVAL,
2551                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2552                                           "zero mask doesn't have any effect");
2553         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
2554         if (ret < 0)
2555                 return ret;
2556         if (!attr->transfer && attr->ingress &&
2557             (action_flags & terminal_action_flags))
2558                 return rte_flow_error_set(error, EINVAL,
2559                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2560                                           "set_tag has no effect"
2561                                           " with terminal actions");
2562         return 0;
2563 }
2564
2565 /**
2566  * Validate count action.
2567  *
2568  * @param[in] dev
2569  *   Pointer to rte_eth_dev structure.
2570  * @param[out] error
2571  *   Pointer to error structure.
2572  *
2573  * @return
2574  *   0 on success, a negative errno value otherwise and rte_errno is set.
2575  */
2576 static int
2577 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2578                               struct rte_flow_error *error)
2579 {
2580         struct mlx5_priv *priv = dev->data->dev_private;
2581
2582         if (!priv->config.devx)
2583                 goto notsup_err;
2584 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2585         return 0;
2586 #endif
2587 notsup_err:
2588         return rte_flow_error_set
2589                       (error, ENOTSUP,
2590                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2591                        NULL,
2592                        "count action not supported");
2593 }
2594
2595 /**
2596  * Validate the L2 encap action.
2597  *
2598  * @param[in] dev
2599  *   Pointer to the rte_eth_dev structure.
2600  * @param[in] action_flags
2601  *   Holds the actions detected until now.
2602  * @param[in] action
2603  *   Pointer to the action structure.
2604  * @param[in] attr
2605  *   Pointer to flow attributes.
2606  * @param[out] error
2607  *   Pointer to error structure.
2608  *
2609  * @return
2610  *   0 on success, a negative errno value otherwise and rte_errno is set.
2611  */
2612 static int
2613 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
2614                                  uint64_t action_flags,
2615                                  const struct rte_flow_action *action,
2616                                  const struct rte_flow_attr *attr,
2617                                  struct rte_flow_error *error)
2618 {
2619         const struct mlx5_priv *priv = dev->data->dev_private;
2620
2621         if (!(action->conf))
2622                 return rte_flow_error_set(error, EINVAL,
2623                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2624                                           "configuration cannot be null");
2625         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
2626                 return rte_flow_error_set(error, EINVAL,
2627                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2628                                           "can only have a single encap action "
2629                                           "in a flow");
2630         if (!attr->transfer && priv->representor)
2631                 return rte_flow_error_set(error, ENOTSUP,
2632                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2633                                           "encap action for VF representor "
2634                                           "not supported on NIC table");
2635         return 0;
2636 }
2637
2638 /**
2639  * Validate a decap action.
2640  *
2641  * @param[in] dev
2642  *   Pointer to the rte_eth_dev structure.
2643  * @param[in] action_flags
2644  *   Holds the actions detected until now.
2645  * @param[in] attr
2646  *   Pointer to flow attributes
2647  * @param[out] error
2648  *   Pointer to error structure.
2649  *
2650  * @return
2651  *   0 on success, a negative errno value otherwise and rte_errno is set.
2652  */
2653 static int
2654 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
2655                               uint64_t action_flags,
2656                               const struct rte_flow_attr *attr,
2657                               struct rte_flow_error *error)
2658 {
2659         const struct mlx5_priv *priv = dev->data->dev_private;
2660
2661         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
2662             !priv->config.decap_en)
2663                 return rte_flow_error_set(error, ENOTSUP,
2664                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2665                                           "decap is not enabled");
2666         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
2667                 return rte_flow_error_set(error, ENOTSUP,
2668                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2669                                           action_flags &
2670                                           MLX5_FLOW_ACTION_DECAP ? "can only "
2671                                           "have a single decap action" : "decap "
2672                                           "after encap is not supported");
2673         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2674                 return rte_flow_error_set(error, EINVAL,
2675                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2676                                           "can't have decap action after"
2677                                           " modify action");
2678         if (attr->egress)
2679                 return rte_flow_error_set(error, ENOTSUP,
2680                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2681                                           NULL,
2682                                           "decap action not supported for "
2683                                           "egress");
2684         if (!attr->transfer && priv->representor)
2685                 return rte_flow_error_set(error, ENOTSUP,
2686                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2687                                           "decap action for VF representor "
2688                                           "not supported on NIC table");
2689         return 0;
2690 }
2691
2692 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
2693
2694 /**
2695  * Validate the raw encap and decap actions.
2696  *
2697  * @param[in] dev
2698  *   Pointer to the rte_eth_dev structure.
2699  * @param[in] decap
2700  *   Pointer to the decap action.
2701  * @param[in] encap
2702  *   Pointer to the encap action.
2703  * @param[in] attr
2704  *   Pointer to flow attributes
2705  * @param[in/out] action_flags
2706  *   Holds the actions detected until now.
2707  * @param[out] actions_n
2708  *   pointer to the number of actions counter.
2709  * @param[out] error
2710  *   Pointer to error structure.
2711  *
2712  * @return
2713  *   0 on success, a negative errno value otherwise and rte_errno is set.
2714  */
2715 static int
2716 flow_dv_validate_action_raw_encap_decap
2717         (struct rte_eth_dev *dev,
2718          const struct rte_flow_action_raw_decap *decap,
2719          const struct rte_flow_action_raw_encap *encap,
2720          const struct rte_flow_attr *attr, uint64_t *action_flags,
2721          int *actions_n, struct rte_flow_error *error)
2722 {
2723         const struct mlx5_priv *priv = dev->data->dev_private;
2724         int ret;
2725
2726         if (encap && (!encap->size || !encap->data))
2727                 return rte_flow_error_set(error, EINVAL,
2728                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2729                                           "raw encap data cannot be empty");
2730         if (decap && encap) {
2731                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
2732                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2733                         /* L3 encap. */
2734                         decap = NULL;
2735                 else if (encap->size <=
2736                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2737                            decap->size >
2738                            MLX5_ENCAPSULATION_DECISION_SIZE)
2739                         /* L3 decap. */
2740                         encap = NULL;
2741                 else if (encap->size >
2742                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2743                            decap->size >
2744                            MLX5_ENCAPSULATION_DECISION_SIZE)
2745                         /* 2 L2 actions: encap and decap. */
2746                         ;
2747                 else
2748                         return rte_flow_error_set(error,
2749                                 ENOTSUP,
2750                                 RTE_FLOW_ERROR_TYPE_ACTION,
2751                                 NULL, "unsupported too small "
2752                                 "raw decap and too small raw "
2753                                 "encap combination");
2754         }
2755         if (decap) {
2756                 ret = flow_dv_validate_action_decap(dev, *action_flags, attr,
2757                                                     error);
2758                 if (ret < 0)
2759                         return ret;
2760                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
2761                 ++(*actions_n);
2762         }
2763         if (encap) {
2764                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
2765                         return rte_flow_error_set(error, ENOTSUP,
2766                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2767                                                   NULL,
2768                                                   "small raw encap size");
2769                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
2770                         return rte_flow_error_set(error, EINVAL,
2771                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2772                                                   NULL,
2773                                                   "more than one encap action");
2774                 if (!attr->transfer && priv->representor)
2775                         return rte_flow_error_set
2776                                         (error, ENOTSUP,
2777                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2778                                          "encap action for VF representor "
2779                                          "not supported on NIC table");
2780                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
2781                 ++(*actions_n);
2782         }
2783         return 0;
2784 }
2785
2786 /**
2787  * Match encap_decap resource.
2788  *
2789  * @param entry
2790  *   Pointer to exist resource entry object.
2791  * @param ctx
2792  *   Pointer to new encap_decap resource.
2793  *
2794  * @return
2795  *   0 on matching, -1 otherwise.
2796  */
2797 static int
2798 flow_dv_encap_decap_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
2799 {
2800         struct mlx5_flow_dv_encap_decap_resource *resource;
2801         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2802
2803         resource = (struct mlx5_flow_dv_encap_decap_resource *)ctx;
2804         cache_resource = container_of(entry,
2805                                       struct mlx5_flow_dv_encap_decap_resource,
2806                                       entry);
2807         if (resource->entry.key == cache_resource->entry.key &&
2808             resource->reformat_type == cache_resource->reformat_type &&
2809             resource->ft_type == cache_resource->ft_type &&
2810             resource->flags == cache_resource->flags &&
2811             resource->size == cache_resource->size &&
2812             !memcmp((const void *)resource->buf,
2813                     (const void *)cache_resource->buf,
2814                     resource->size))
2815                 return 0;
2816         return -1;
2817 }
2818
2819 /**
2820  * Find existing encap/decap resource or create and register a new one.
2821  *
2822  * @param[in, out] dev
2823  *   Pointer to rte_eth_dev structure.
2824  * @param[in, out] resource
2825  *   Pointer to encap/decap resource.
2826  * @parm[in, out] dev_flow
2827  *   Pointer to the dev_flow.
2828  * @param[out] error
2829  *   pointer to error structure.
2830  *
2831  * @return
2832  *   0 on success otherwise -errno and errno is set.
2833  */
2834 static int
2835 flow_dv_encap_decap_resource_register
2836                         (struct rte_eth_dev *dev,
2837                          struct mlx5_flow_dv_encap_decap_resource *resource,
2838                          struct mlx5_flow *dev_flow,
2839                          struct rte_flow_error *error)
2840 {
2841         struct mlx5_priv *priv = dev->data->dev_private;
2842         struct mlx5_dev_ctx_shared *sh = priv->sh;
2843         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2844         struct mlx5dv_dr_domain *domain;
2845         struct mlx5_hlist_entry *entry;
2846         union mlx5_flow_encap_decap_key encap_decap_key = {
2847                 {
2848                         .ft_type = resource->ft_type,
2849                         .refmt_type = resource->reformat_type,
2850                         .buf_size = resource->size,
2851                         .table_level = !!dev_flow->dv.group,
2852                         .cksum = 0,
2853                 }
2854         };
2855         int ret;
2856
2857         resource->flags = dev_flow->dv.group ? 0 : 1;
2858         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2859                 domain = sh->fdb_domain;
2860         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2861                 domain = sh->rx_domain;
2862         else
2863                 domain = sh->tx_domain;
2864         encap_decap_key.cksum = __rte_raw_cksum(resource->buf,
2865                                                 resource->size, 0);
2866         resource->entry.key = encap_decap_key.v64;
2867         /* Lookup a matching resource from cache. */
2868         entry = mlx5_hlist_lookup_ex(sh->encaps_decaps, resource->entry.key,
2869                                      flow_dv_encap_decap_resource_match,
2870                                      (void *)resource);
2871         if (entry) {
2872                 cache_resource = container_of(entry,
2873                         struct mlx5_flow_dv_encap_decap_resource, entry);
2874                 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2875                         (void *)cache_resource,
2876                         __atomic_load_n(&cache_resource->refcnt,
2877                                         __ATOMIC_RELAXED));
2878                 __atomic_fetch_add(&cache_resource->refcnt, 1,
2879                                    __ATOMIC_RELAXED);
2880                 dev_flow->handle->dvh.rix_encap_decap = cache_resource->idx;
2881                 dev_flow->dv.encap_decap = cache_resource;
2882                 return 0;
2883         }
2884         /* Register new encap/decap resource. */
2885         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2886                                        &dev_flow->handle->dvh.rix_encap_decap);
2887         if (!cache_resource)
2888                 return rte_flow_error_set(error, ENOMEM,
2889                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2890                                           "cannot allocate resource memory");
2891         *cache_resource = *resource;
2892         cache_resource->idx = dev_flow->handle->dvh.rix_encap_decap;
2893         ret = mlx5_flow_os_create_flow_action_packet_reformat
2894                                         (sh->ctx, domain, cache_resource,
2895                                          &cache_resource->action);
2896         if (ret) {
2897                 mlx5_free(cache_resource);
2898                 return rte_flow_error_set(error, ENOMEM,
2899                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2900                                           NULL, "cannot create action");
2901         }
2902         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
2903         if (mlx5_hlist_insert_ex(sh->encaps_decaps, &cache_resource->entry,
2904                                  flow_dv_encap_decap_resource_match,
2905                                  (void *)cache_resource)) {
2906                 claim_zero(mlx5_flow_os_destroy_flow_action
2907                                                 (cache_resource->action));
2908                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2909                                 cache_resource->idx);
2910                 return rte_flow_error_set(error, EEXIST,
2911                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2912                                           NULL, "action exist");
2913         }
2914         dev_flow->dv.encap_decap = cache_resource;
2915         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2916                 (void *)cache_resource,
2917                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
2918         return 0;
2919 }
2920
2921 /**
2922  * Find existing table jump resource or create and register a new one.
2923  *
2924  * @param[in, out] dev
2925  *   Pointer to rte_eth_dev structure.
2926  * @param[in, out] tbl
2927  *   Pointer to flow table resource.
2928  * @parm[in, out] dev_flow
2929  *   Pointer to the dev_flow.
2930  * @param[out] error
2931  *   pointer to error structure.
2932  *
2933  * @return
2934  *   0 on success otherwise -errno and errno is set.
2935  */
2936 static int
2937 flow_dv_jump_tbl_resource_register
2938                         (struct rte_eth_dev *dev __rte_unused,
2939                          struct mlx5_flow_tbl_resource *tbl,
2940                          struct mlx5_flow *dev_flow,
2941                          struct rte_flow_error *error)
2942 {
2943         struct mlx5_flow_tbl_data_entry *tbl_data =
2944                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2945         int cnt, ret;
2946
2947         MLX5_ASSERT(tbl);
2948         cnt = __atomic_load_n(&tbl_data->jump.refcnt, __ATOMIC_ACQUIRE);
2949         if (!cnt) {
2950                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
2951                                 (tbl->obj, &tbl_data->jump.action);
2952                 if (ret)
2953                         return rte_flow_error_set(error, ENOMEM,
2954                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2955                                         NULL, "cannot create jump action");
2956                 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2957                         (void *)&tbl_data->jump, cnt);
2958         } else {
2959                 /* old jump should not make the table ref++. */
2960                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
2961                 MLX5_ASSERT(tbl_data->jump.action);
2962                 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2963                         (void *)&tbl_data->jump, cnt);
2964         }
2965         __atomic_fetch_add(&tbl_data->jump.refcnt, 1, __ATOMIC_RELEASE);
2966         dev_flow->handle->rix_jump = tbl_data->idx;
2967         dev_flow->dv.jump = &tbl_data->jump;
2968         return 0;
2969 }
2970
2971 /**
2972  * Find existing default miss resource or create and register a new one.
2973  *
2974  * @param[in, out] dev
2975  *   Pointer to rte_eth_dev structure.
2976  * @param[out] error
2977  *   pointer to error structure.
2978  *
2979  * @return
2980  *   0 on success otherwise -errno and errno is set.
2981  */
2982 static int
2983 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
2984                 struct rte_flow_error *error)
2985 {
2986         struct mlx5_priv *priv = dev->data->dev_private;
2987         struct mlx5_dev_ctx_shared *sh = priv->sh;
2988         struct mlx5_flow_default_miss_resource *cache_resource =
2989                         &sh->default_miss;
2990         int cnt = __atomic_load_n(&cache_resource->refcnt, __ATOMIC_ACQUIRE);
2991
2992         if (!cnt) {
2993                 MLX5_ASSERT(cache_resource->action);
2994                 cache_resource->action =
2995                 mlx5_glue->dr_create_flow_action_default_miss();
2996                 if (!cache_resource->action)
2997                         return rte_flow_error_set(error, ENOMEM,
2998                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2999                                         "cannot create default miss action");
3000                 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++",
3001                                 (void *)cache_resource->action, cnt);
3002         }
3003         __atomic_fetch_add(&cache_resource->refcnt, 1, __ATOMIC_RELEASE);
3004         return 0;
3005 }
3006
3007 /**
3008  * Find existing table port ID resource or create and register a new one.
3009  *
3010  * @param[in, out] dev
3011  *   Pointer to rte_eth_dev structure.
3012  * @param[in, out] resource
3013  *   Pointer to port ID action resource.
3014  * @parm[in, out] dev_flow
3015  *   Pointer to the dev_flow.
3016  * @param[out] error
3017  *   pointer to error structure.
3018  *
3019  * @return
3020  *   0 on success otherwise -errno and errno is set.
3021  */
3022 static int
3023 flow_dv_port_id_action_resource_register
3024                         (struct rte_eth_dev *dev,
3025                          struct mlx5_flow_dv_port_id_action_resource *resource,
3026                          struct mlx5_flow *dev_flow,
3027                          struct rte_flow_error *error)
3028 {
3029         struct mlx5_priv *priv = dev->data->dev_private;
3030         struct mlx5_dev_ctx_shared *sh = priv->sh;
3031         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
3032         uint32_t idx = 0;
3033         int ret;
3034
3035         /* Lookup a matching resource from cache. */
3036         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list,
3037                       idx, cache_resource, next) {
3038                 if (resource->port_id == cache_resource->port_id) {
3039                         DRV_LOG(DEBUG, "port id action resource resource %p: "
3040                                 "refcnt %d++",
3041                                 (void *)cache_resource,
3042                                 __atomic_load_n(&cache_resource->refcnt,
3043                                                 __ATOMIC_RELAXED));
3044                         __atomic_fetch_add(&cache_resource->refcnt, 1,
3045                                            __ATOMIC_RELAXED);
3046                         dev_flow->handle->rix_port_id_action = idx;
3047                         dev_flow->dv.port_id_action = cache_resource;
3048                         return 0;
3049                 }
3050         }
3051         /* Register new port id action resource. */
3052         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID],
3053                                        &dev_flow->handle->rix_port_id_action);
3054         if (!cache_resource)
3055                 return rte_flow_error_set(error, ENOMEM,
3056                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3057                                           "cannot allocate resource memory");
3058         *cache_resource = *resource;
3059         ret = mlx5_flow_os_create_flow_action_dest_port
3060                                 (priv->sh->fdb_domain, resource->port_id,
3061                                  &cache_resource->action);
3062         if (ret) {
3063                 mlx5_free(cache_resource);
3064                 return rte_flow_error_set(error, ENOMEM,
3065                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3066                                           NULL, "cannot create action");
3067         }
3068         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
3069         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
3070                      dev_flow->handle->rix_port_id_action, cache_resource,
3071                      next);
3072         dev_flow->dv.port_id_action = cache_resource;
3073         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
3074                 (void *)cache_resource,
3075                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
3076         return 0;
3077 }
3078
3079 /**
3080  * Find existing push vlan resource or create and register a new one.
3081  *
3082  * @param [in, out] dev
3083  *   Pointer to rte_eth_dev structure.
3084  * @param[in, out] resource
3085  *   Pointer to port ID action resource.
3086  * @parm[in, out] dev_flow
3087  *   Pointer to the dev_flow.
3088  * @param[out] error
3089  *   pointer to error structure.
3090  *
3091  * @return
3092  *   0 on success otherwise -errno and errno is set.
3093  */
3094 static int
3095 flow_dv_push_vlan_action_resource_register
3096                        (struct rte_eth_dev *dev,
3097                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
3098                         struct mlx5_flow *dev_flow,
3099                         struct rte_flow_error *error)
3100 {
3101         struct mlx5_priv *priv = dev->data->dev_private;
3102         struct mlx5_dev_ctx_shared *sh = priv->sh;
3103         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
3104         struct mlx5dv_dr_domain *domain;
3105         uint32_t idx = 0;
3106         int ret;
3107
3108         /* Lookup a matching resource from cache. */
3109         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3110                       sh->push_vlan_action_list, idx, cache_resource, next) {
3111                 if (resource->vlan_tag == cache_resource->vlan_tag &&
3112                     resource->ft_type == cache_resource->ft_type) {
3113                         DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
3114                                 "refcnt %d++",
3115                                 (void *)cache_resource,
3116                                 __atomic_load_n(&cache_resource->refcnt,
3117                                                 __ATOMIC_RELAXED));
3118                         __atomic_fetch_add(&cache_resource->refcnt, 1,
3119                                            __ATOMIC_RELAXED);
3120                         dev_flow->handle->dvh.rix_push_vlan = idx;
3121                         dev_flow->dv.push_vlan_res = cache_resource;
3122                         return 0;
3123                 }
3124         }
3125         /* Register new push_vlan action resource. */
3126         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3127                                        &dev_flow->handle->dvh.rix_push_vlan);
3128         if (!cache_resource)
3129                 return rte_flow_error_set(error, ENOMEM,
3130                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3131                                           "cannot allocate resource memory");
3132         *cache_resource = *resource;
3133         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3134                 domain = sh->fdb_domain;
3135         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3136                 domain = sh->rx_domain;
3137         else
3138                 domain = sh->tx_domain;
3139         ret = mlx5_flow_os_create_flow_action_push_vlan
3140                                         (domain, resource->vlan_tag,
3141                                          &cache_resource->action);
3142         if (ret) {
3143                 mlx5_free(cache_resource);
3144                 return rte_flow_error_set(error, ENOMEM,
3145                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3146                                           NULL, "cannot create action");
3147         }
3148         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
3149         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3150                      &sh->push_vlan_action_list,
3151                      dev_flow->handle->dvh.rix_push_vlan,
3152                      cache_resource, next);
3153         dev_flow->dv.push_vlan_res = cache_resource;
3154         DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
3155                 (void *)cache_resource,
3156                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
3157         return 0;
3158 }
3159 /**
3160  * Get the size of specific rte_flow_item_type hdr size
3161  *
3162  * @param[in] item_type
3163  *   Tested rte_flow_item_type.
3164  *
3165  * @return
3166  *   sizeof struct item_type, 0 if void or irrelevant.
3167  */
3168 static size_t
3169 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
3170 {
3171         size_t retval;
3172
3173         switch (item_type) {
3174         case RTE_FLOW_ITEM_TYPE_ETH:
3175                 retval = sizeof(struct rte_ether_hdr);
3176                 break;
3177         case RTE_FLOW_ITEM_TYPE_VLAN:
3178                 retval = sizeof(struct rte_vlan_hdr);
3179                 break;
3180         case RTE_FLOW_ITEM_TYPE_IPV4:
3181                 retval = sizeof(struct rte_ipv4_hdr);
3182                 break;
3183         case RTE_FLOW_ITEM_TYPE_IPV6:
3184                 retval = sizeof(struct rte_ipv6_hdr);
3185                 break;
3186         case RTE_FLOW_ITEM_TYPE_UDP:
3187                 retval = sizeof(struct rte_udp_hdr);
3188                 break;
3189         case RTE_FLOW_ITEM_TYPE_TCP:
3190                 retval = sizeof(struct rte_tcp_hdr);
3191                 break;
3192         case RTE_FLOW_ITEM_TYPE_VXLAN:
3193         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3194                 retval = sizeof(struct rte_vxlan_hdr);
3195                 break;
3196         case RTE_FLOW_ITEM_TYPE_GRE:
3197         case RTE_FLOW_ITEM_TYPE_NVGRE:
3198                 retval = sizeof(struct rte_gre_hdr);
3199                 break;
3200         case RTE_FLOW_ITEM_TYPE_MPLS:
3201                 retval = sizeof(struct rte_mpls_hdr);
3202                 break;
3203         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
3204         default:
3205                 retval = 0;
3206                 break;
3207         }
3208         return retval;
3209 }
3210
3211 #define MLX5_ENCAP_IPV4_VERSION         0x40
3212 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
3213 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
3214 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
3215 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
3216 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
3217 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
3218
3219 /**
3220  * Convert the encap action data from list of rte_flow_item to raw buffer
3221  *
3222  * @param[in] items
3223  *   Pointer to rte_flow_item objects list.
3224  * @param[out] buf
3225  *   Pointer to the output buffer.
3226  * @param[out] size
3227  *   Pointer to the output buffer size.
3228  * @param[out] error
3229  *   Pointer to the error structure.
3230  *
3231  * @return
3232  *   0 on success, a negative errno value otherwise and rte_errno is set.
3233  */
3234 static int
3235 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
3236                            size_t *size, struct rte_flow_error *error)
3237 {
3238         struct rte_ether_hdr *eth = NULL;
3239         struct rte_vlan_hdr *vlan = NULL;
3240         struct rte_ipv4_hdr *ipv4 = NULL;
3241         struct rte_ipv6_hdr *ipv6 = NULL;
3242         struct rte_udp_hdr *udp = NULL;
3243         struct rte_vxlan_hdr *vxlan = NULL;
3244         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
3245         struct rte_gre_hdr *gre = NULL;
3246         size_t len;
3247         size_t temp_size = 0;
3248
3249         if (!items)
3250                 return rte_flow_error_set(error, EINVAL,
3251                                           RTE_FLOW_ERROR_TYPE_ACTION,
3252                                           NULL, "invalid empty data");
3253         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3254                 len = flow_dv_get_item_hdr_len(items->type);
3255                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
3256                         return rte_flow_error_set(error, EINVAL,
3257                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3258                                                   (void *)items->type,
3259                                                   "items total size is too big"
3260                                                   " for encap action");
3261                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
3262                 switch (items->type) {
3263                 case RTE_FLOW_ITEM_TYPE_ETH:
3264                         eth = (struct rte_ether_hdr *)&buf[temp_size];
3265                         break;
3266                 case RTE_FLOW_ITEM_TYPE_VLAN:
3267                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
3268                         if (!eth)
3269                                 return rte_flow_error_set(error, EINVAL,
3270                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3271                                                 (void *)items->type,
3272                                                 "eth header not found");
3273                         if (!eth->ether_type)
3274                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
3275                         break;
3276                 case RTE_FLOW_ITEM_TYPE_IPV4:
3277                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
3278                         if (!vlan && !eth)
3279                                 return rte_flow_error_set(error, EINVAL,
3280                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3281                                                 (void *)items->type,
3282                                                 "neither eth nor vlan"
3283                                                 " header found");
3284                         if (vlan && !vlan->eth_proto)
3285                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3286                         else if (eth && !eth->ether_type)
3287                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3288                         if (!ipv4->version_ihl)
3289                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
3290                                                     MLX5_ENCAP_IPV4_IHL_MIN;
3291                         if (!ipv4->time_to_live)
3292                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
3293                         break;
3294                 case RTE_FLOW_ITEM_TYPE_IPV6:
3295                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
3296                         if (!vlan && !eth)
3297                                 return rte_flow_error_set(error, EINVAL,
3298                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3299                                                 (void *)items->type,
3300                                                 "neither eth nor vlan"
3301                                                 " header found");
3302                         if (vlan && !vlan->eth_proto)
3303                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3304                         else if (eth && !eth->ether_type)
3305                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3306                         if (!ipv6->vtc_flow)
3307                                 ipv6->vtc_flow =
3308                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
3309                         if (!ipv6->hop_limits)
3310                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
3311                         break;
3312                 case RTE_FLOW_ITEM_TYPE_UDP:
3313                         udp = (struct rte_udp_hdr *)&buf[temp_size];
3314                         if (!ipv4 && !ipv6)
3315                                 return rte_flow_error_set(error, EINVAL,
3316                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3317                                                 (void *)items->type,
3318                                                 "ip header not found");
3319                         if (ipv4 && !ipv4->next_proto_id)
3320                                 ipv4->next_proto_id = IPPROTO_UDP;
3321                         else if (ipv6 && !ipv6->proto)
3322                                 ipv6->proto = IPPROTO_UDP;
3323                         break;
3324                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3325                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
3326                         if (!udp)
3327                                 return rte_flow_error_set(error, EINVAL,
3328                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3329                                                 (void *)items->type,
3330                                                 "udp header not found");
3331                         if (!udp->dst_port)
3332                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
3333                         if (!vxlan->vx_flags)
3334                                 vxlan->vx_flags =
3335                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
3336                         break;
3337                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3338                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
3339                         if (!udp)
3340                                 return rte_flow_error_set(error, EINVAL,
3341                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3342                                                 (void *)items->type,
3343                                                 "udp header not found");
3344                         if (!vxlan_gpe->proto)
3345                                 return rte_flow_error_set(error, EINVAL,
3346                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3347                                                 (void *)items->type,
3348                                                 "next protocol not found");
3349                         if (!udp->dst_port)
3350                                 udp->dst_port =
3351                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
3352                         if (!vxlan_gpe->vx_flags)
3353                                 vxlan_gpe->vx_flags =
3354                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
3355                         break;
3356                 case RTE_FLOW_ITEM_TYPE_GRE:
3357                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3358                         gre = (struct rte_gre_hdr *)&buf[temp_size];
3359                         if (!gre->proto)
3360                                 return rte_flow_error_set(error, EINVAL,
3361                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3362                                                 (void *)items->type,
3363                                                 "next protocol not found");
3364                         if (!ipv4 && !ipv6)
3365                                 return rte_flow_error_set(error, EINVAL,
3366                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3367                                                 (void *)items->type,
3368                                                 "ip header not found");
3369                         if (ipv4 && !ipv4->next_proto_id)
3370                                 ipv4->next_proto_id = IPPROTO_GRE;
3371                         else if (ipv6 && !ipv6->proto)
3372                                 ipv6->proto = IPPROTO_GRE;
3373                         break;
3374                 case RTE_FLOW_ITEM_TYPE_VOID:
3375                         break;
3376                 default:
3377                         return rte_flow_error_set(error, EINVAL,
3378                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3379                                                   (void *)items->type,
3380                                                   "unsupported item type");
3381                         break;
3382                 }
3383                 temp_size += len;
3384         }
3385         *size = temp_size;
3386         return 0;
3387 }
3388
3389 static int
3390 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
3391 {
3392         struct rte_ether_hdr *eth = NULL;
3393         struct rte_vlan_hdr *vlan = NULL;
3394         struct rte_ipv6_hdr *ipv6 = NULL;
3395         struct rte_udp_hdr *udp = NULL;
3396         char *next_hdr;
3397         uint16_t proto;
3398
3399         eth = (struct rte_ether_hdr *)data;
3400         next_hdr = (char *)(eth + 1);
3401         proto = RTE_BE16(eth->ether_type);
3402
3403         /* VLAN skipping */
3404         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
3405                 vlan = (struct rte_vlan_hdr *)next_hdr;
3406                 proto = RTE_BE16(vlan->eth_proto);
3407                 next_hdr += sizeof(struct rte_vlan_hdr);
3408         }
3409
3410         /* HW calculates IPv4 csum. no need to proceed */
3411         if (proto == RTE_ETHER_TYPE_IPV4)
3412                 return 0;
3413
3414         /* non IPv4/IPv6 header. not supported */
3415         if (proto != RTE_ETHER_TYPE_IPV6) {
3416                 return rte_flow_error_set(error, ENOTSUP,
3417                                           RTE_FLOW_ERROR_TYPE_ACTION,
3418                                           NULL, "Cannot offload non IPv4/IPv6");
3419         }
3420
3421         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
3422
3423         /* ignore non UDP */
3424         if (ipv6->proto != IPPROTO_UDP)
3425                 return 0;
3426
3427         udp = (struct rte_udp_hdr *)(ipv6 + 1);
3428         udp->dgram_cksum = 0;
3429
3430         return 0;
3431 }
3432
3433 /**
3434  * Convert L2 encap action to DV specification.
3435  *
3436  * @param[in] dev
3437  *   Pointer to rte_eth_dev structure.
3438  * @param[in] action
3439  *   Pointer to action structure.
3440  * @param[in, out] dev_flow
3441  *   Pointer to the mlx5_flow.
3442  * @param[in] transfer
3443  *   Mark if the flow is E-Switch flow.
3444  * @param[out] error
3445  *   Pointer to the error structure.
3446  *
3447  * @return
3448  *   0 on success, a negative errno value otherwise and rte_errno is set.
3449  */
3450 static int
3451 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
3452                                const struct rte_flow_action *action,
3453                                struct mlx5_flow *dev_flow,
3454                                uint8_t transfer,
3455                                struct rte_flow_error *error)
3456 {
3457         const struct rte_flow_item *encap_data;
3458         const struct rte_flow_action_raw_encap *raw_encap_data;
3459         struct mlx5_flow_dv_encap_decap_resource res = {
3460                 .reformat_type =
3461                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
3462                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3463                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
3464         };
3465
3466         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3467                 raw_encap_data =
3468                         (const struct rte_flow_action_raw_encap *)action->conf;
3469                 res.size = raw_encap_data->size;
3470                 memcpy(res.buf, raw_encap_data->data, res.size);
3471         } else {
3472                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
3473                         encap_data =
3474                                 ((const struct rte_flow_action_vxlan_encap *)
3475                                                 action->conf)->definition;
3476                 else
3477                         encap_data =
3478                                 ((const struct rte_flow_action_nvgre_encap *)
3479                                                 action->conf)->definition;
3480                 if (flow_dv_convert_encap_data(encap_data, res.buf,
3481                                                &res.size, error))
3482                         return -rte_errno;
3483         }
3484         if (flow_dv_zero_encap_udp_csum(res.buf, error))
3485                 return -rte_errno;
3486         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3487                 return rte_flow_error_set(error, EINVAL,
3488                                           RTE_FLOW_ERROR_TYPE_ACTION,
3489                                           NULL, "can't create L2 encap action");
3490         return 0;
3491 }
3492
3493 /**
3494  * Convert L2 decap action to DV specification.
3495  *
3496  * @param[in] dev
3497  *   Pointer to rte_eth_dev structure.
3498  * @param[in, out] dev_flow
3499  *   Pointer to the mlx5_flow.
3500  * @param[in] transfer
3501  *   Mark if the flow is E-Switch flow.
3502  * @param[out] error
3503  *   Pointer to the error structure.
3504  *
3505  * @return
3506  *   0 on success, a negative errno value otherwise and rte_errno is set.
3507  */
3508 static int
3509 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
3510                                struct mlx5_flow *dev_flow,
3511                                uint8_t transfer,
3512                                struct rte_flow_error *error)
3513 {
3514         struct mlx5_flow_dv_encap_decap_resource res = {
3515                 .size = 0,
3516                 .reformat_type =
3517                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
3518                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3519                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
3520         };
3521
3522         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3523                 return rte_flow_error_set(error, EINVAL,
3524                                           RTE_FLOW_ERROR_TYPE_ACTION,
3525                                           NULL, "can't create L2 decap action");
3526         return 0;
3527 }
3528
3529 /**
3530  * Convert raw decap/encap (L3 tunnel) action to DV specification.
3531  *
3532  * @param[in] dev
3533  *   Pointer to rte_eth_dev structure.
3534  * @param[in] action
3535  *   Pointer to action structure.
3536  * @param[in, out] dev_flow
3537  *   Pointer to the mlx5_flow.
3538  * @param[in] attr
3539  *   Pointer to the flow attributes.
3540  * @param[out] error
3541  *   Pointer to the error structure.
3542  *
3543  * @return
3544  *   0 on success, a negative errno value otherwise and rte_errno is set.
3545  */
3546 static int
3547 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
3548                                 const struct rte_flow_action *action,
3549                                 struct mlx5_flow *dev_flow,
3550                                 const struct rte_flow_attr *attr,
3551                                 struct rte_flow_error *error)
3552 {
3553         const struct rte_flow_action_raw_encap *encap_data;
3554         struct mlx5_flow_dv_encap_decap_resource res;
3555
3556         memset(&res, 0, sizeof(res));
3557         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
3558         res.size = encap_data->size;
3559         memcpy(res.buf, encap_data->data, res.size);
3560         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
3561                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
3562                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3563         if (attr->transfer)
3564                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3565         else
3566                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3567                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3568         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3569                 return rte_flow_error_set(error, EINVAL,
3570                                           RTE_FLOW_ERROR_TYPE_ACTION,
3571                                           NULL, "can't create encap action");
3572         return 0;
3573 }
3574
3575 /**
3576  * Create action push VLAN.
3577  *
3578  * @param[in] dev
3579  *   Pointer to rte_eth_dev structure.
3580  * @param[in] attr
3581  *   Pointer to the flow attributes.
3582  * @param[in] vlan
3583  *   Pointer to the vlan to push to the Ethernet header.
3584  * @param[in, out] dev_flow
3585  *   Pointer to the mlx5_flow.
3586  * @param[out] error
3587  *   Pointer to the error structure.
3588  *
3589  * @return
3590  *   0 on success, a negative errno value otherwise and rte_errno is set.
3591  */
3592 static int
3593 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
3594                                 const struct rte_flow_attr *attr,
3595                                 const struct rte_vlan_hdr *vlan,
3596                                 struct mlx5_flow *dev_flow,
3597                                 struct rte_flow_error *error)
3598 {
3599         struct mlx5_flow_dv_push_vlan_action_resource res;
3600
3601         memset(&res, 0, sizeof(res));
3602         res.vlan_tag =
3603                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
3604                                  vlan->vlan_tci);
3605         if (attr->transfer)
3606                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3607         else
3608                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3609                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3610         return flow_dv_push_vlan_action_resource_register
3611                                             (dev, &res, dev_flow, error);
3612 }
3613
3614 static int fdb_mirror;
3615
3616 /**
3617  * Validate the modify-header actions.
3618  *
3619  * @param[in] action_flags
3620  *   Holds the actions detected until now.
3621  * @param[in] action
3622  *   Pointer to the modify action.
3623  * @param[out] error
3624  *   Pointer to error structure.
3625  *
3626  * @return
3627  *   0 on success, a negative errno value otherwise and rte_errno is set.
3628  */
3629 static int
3630 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
3631                                    const struct rte_flow_action *action,
3632                                    struct rte_flow_error *error)
3633 {
3634         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
3635                 return rte_flow_error_set(error, EINVAL,
3636                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3637                                           NULL, "action configuration not set");
3638         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3639                 return rte_flow_error_set(error, EINVAL,
3640                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3641                                           "can't have encap action before"
3642                                           " modify action");
3643         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror)
3644                 return rte_flow_error_set(error, EINVAL,
3645                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3646                                           "can't support sample action before"
3647                                           " modify action for E-Switch"
3648                                           " mirroring");
3649         return 0;
3650 }
3651
3652 /**
3653  * Validate the modify-header MAC address actions.
3654  *
3655  * @param[in] action_flags
3656  *   Holds the actions detected until now.
3657  * @param[in] action
3658  *   Pointer to the modify action.
3659  * @param[in] item_flags
3660  *   Holds the items detected.
3661  * @param[out] error
3662  *   Pointer to error structure.
3663  *
3664  * @return
3665  *   0 on success, a negative errno value otherwise and rte_errno is set.
3666  */
3667 static int
3668 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
3669                                    const struct rte_flow_action *action,
3670                                    const uint64_t item_flags,
3671                                    struct rte_flow_error *error)
3672 {
3673         int ret = 0;
3674
3675         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3676         if (!ret) {
3677                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
3678                         return rte_flow_error_set(error, EINVAL,
3679                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3680                                                   NULL,
3681                                                   "no L2 item in pattern");
3682         }
3683         return ret;
3684 }
3685
3686 /**
3687  * Validate the modify-header IPv4 address actions.
3688  *
3689  * @param[in] action_flags
3690  *   Holds the actions detected until now.
3691  * @param[in] action
3692  *   Pointer to the modify action.
3693  * @param[in] item_flags
3694  *   Holds the items detected.
3695  * @param[out] error
3696  *   Pointer to error structure.
3697  *
3698  * @return
3699  *   0 on success, a negative errno value otherwise and rte_errno is set.
3700  */
3701 static int
3702 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3703                                     const struct rte_flow_action *action,
3704                                     const uint64_t item_flags,
3705                                     struct rte_flow_error *error)
3706 {
3707         int ret = 0;
3708         uint64_t layer;
3709
3710         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3711         if (!ret) {
3712                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3713                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3714                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3715                 if (!(item_flags & layer))
3716                         return rte_flow_error_set(error, EINVAL,
3717                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3718                                                   NULL,
3719                                                   "no ipv4 item in pattern");
3720         }
3721         return ret;
3722 }
3723
3724 /**
3725  * Validate the modify-header IPv6 address actions.
3726  *
3727  * @param[in] action_flags
3728  *   Holds the actions detected until now.
3729  * @param[in] action
3730  *   Pointer to the modify action.
3731  * @param[in] item_flags
3732  *   Holds the items detected.
3733  * @param[out] error
3734  *   Pointer to error structure.
3735  *
3736  * @return
3737  *   0 on success, a negative errno value otherwise and rte_errno is set.
3738  */
3739 static int
3740 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3741                                     const struct rte_flow_action *action,
3742                                     const uint64_t item_flags,
3743                                     struct rte_flow_error *error)
3744 {
3745         int ret = 0;
3746         uint64_t layer;
3747
3748         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3749         if (!ret) {
3750                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3751                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3752                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3753                 if (!(item_flags & layer))
3754                         return rte_flow_error_set(error, EINVAL,
3755                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3756                                                   NULL,
3757                                                   "no ipv6 item in pattern");
3758         }
3759         return ret;
3760 }
3761
3762 /**
3763  * Validate the modify-header TP actions.
3764  *
3765  * @param[in] action_flags
3766  *   Holds the actions detected until now.
3767  * @param[in] action
3768  *   Pointer to the modify action.
3769  * @param[in] item_flags
3770  *   Holds the items detected.
3771  * @param[out] error
3772  *   Pointer to error structure.
3773  *
3774  * @return
3775  *   0 on success, a negative errno value otherwise and rte_errno is set.
3776  */
3777 static int
3778 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3779                                   const struct rte_flow_action *action,
3780                                   const uint64_t item_flags,
3781                                   struct rte_flow_error *error)
3782 {
3783         int ret = 0;
3784         uint64_t layer;
3785
3786         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3787         if (!ret) {
3788                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3789                                  MLX5_FLOW_LAYER_INNER_L4 :
3790                                  MLX5_FLOW_LAYER_OUTER_L4;
3791                 if (!(item_flags & layer))
3792                         return rte_flow_error_set(error, EINVAL,
3793                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3794                                                   NULL, "no transport layer "
3795                                                   "in pattern");
3796         }
3797         return ret;
3798 }
3799
3800 /**
3801  * Validate the modify-header actions of increment/decrement
3802  * TCP Sequence-number.
3803  *
3804  * @param[in] action_flags
3805  *   Holds the actions detected until now.
3806  * @param[in] action
3807  *   Pointer to the modify action.
3808  * @param[in] item_flags
3809  *   Holds the items detected.
3810  * @param[out] error
3811  *   Pointer to error structure.
3812  *
3813  * @return
3814  *   0 on success, a negative errno value otherwise and rte_errno is set.
3815  */
3816 static int
3817 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3818                                        const struct rte_flow_action *action,
3819                                        const uint64_t item_flags,
3820                                        struct rte_flow_error *error)
3821 {
3822         int ret = 0;
3823         uint64_t layer;
3824
3825         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3826         if (!ret) {
3827                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3828                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3829                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3830                 if (!(item_flags & layer))
3831                         return rte_flow_error_set(error, EINVAL,
3832                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3833                                                   NULL, "no TCP item in"
3834                                                   " pattern");
3835                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3836                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3837                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3838                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3839                         return rte_flow_error_set(error, EINVAL,
3840                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3841                                                   NULL,
3842                                                   "cannot decrease and increase"
3843                                                   " TCP sequence number"
3844                                                   " at the same time");
3845         }
3846         return ret;
3847 }
3848
3849 /**
3850  * Validate the modify-header actions of increment/decrement
3851  * TCP Acknowledgment number.
3852  *
3853  * @param[in] action_flags
3854  *   Holds the actions detected until now.
3855  * @param[in] action
3856  *   Pointer to the modify action.
3857  * @param[in] item_flags
3858  *   Holds the items detected.
3859  * @param[out] error
3860  *   Pointer to error structure.
3861  *
3862  * @return
3863  *   0 on success, a negative errno value otherwise and rte_errno is set.
3864  */
3865 static int
3866 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3867                                        const struct rte_flow_action *action,
3868                                        const uint64_t item_flags,
3869                                        struct rte_flow_error *error)
3870 {
3871         int ret = 0;
3872         uint64_t layer;
3873
3874         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3875         if (!ret) {
3876                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3877                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3878                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3879                 if (!(item_flags & layer))
3880                         return rte_flow_error_set(error, EINVAL,
3881                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3882                                                   NULL, "no TCP item in"
3883                                                   " pattern");
3884                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3885                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3886                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3887                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3888                         return rte_flow_error_set(error, EINVAL,
3889                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3890                                                   NULL,
3891                                                   "cannot decrease and increase"
3892                                                   " TCP acknowledgment number"
3893                                                   " at the same time");
3894         }
3895         return ret;
3896 }
3897
3898 /**
3899  * Validate the modify-header TTL actions.
3900  *
3901  * @param[in] action_flags
3902  *   Holds the actions detected until now.
3903  * @param[in] action
3904  *   Pointer to the modify action.
3905  * @param[in] item_flags
3906  *   Holds the items detected.
3907  * @param[out] error
3908  *   Pointer to error structure.
3909  *
3910  * @return
3911  *   0 on success, a negative errno value otherwise and rte_errno is set.
3912  */
3913 static int
3914 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3915                                    const struct rte_flow_action *action,
3916                                    const uint64_t item_flags,
3917                                    struct rte_flow_error *error)
3918 {
3919         int ret = 0;
3920         uint64_t layer;
3921
3922         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3923         if (!ret) {
3924                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3925                                  MLX5_FLOW_LAYER_INNER_L3 :
3926                                  MLX5_FLOW_LAYER_OUTER_L3;
3927                 if (!(item_flags & layer))
3928                         return rte_flow_error_set(error, EINVAL,
3929                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3930                                                   NULL,
3931                                                   "no IP protocol in pattern");
3932         }
3933         return ret;
3934 }
3935
3936 /**
3937  * Validate jump action.
3938  *
3939  * @param[in] action
3940  *   Pointer to the jump action.
3941  * @param[in] action_flags
3942  *   Holds the actions detected until now.
3943  * @param[in] attributes
3944  *   Pointer to flow attributes
3945  * @param[in] external
3946  *   Action belongs to flow rule created by request external to PMD.
3947  * @param[out] error
3948  *   Pointer to error structure.
3949  *
3950  * @return
3951  *   0 on success, a negative errno value otherwise and rte_errno is set.
3952  */
3953 static int
3954 flow_dv_validate_action_jump(struct rte_eth_dev *dev,
3955                              const struct mlx5_flow_tunnel *tunnel,
3956                              const struct rte_flow_action *action,
3957                              uint64_t action_flags,
3958                              const struct rte_flow_attr *attributes,
3959                              bool external, struct rte_flow_error *error)
3960 {
3961         uint32_t target_group, table;
3962         int ret = 0;
3963         struct flow_grp_info grp_info = {
3964                 .external = !!external,
3965                 .transfer = !!attributes->transfer,
3966                 .fdb_def_rule = 1,
3967                 .std_tbl_fix = 0
3968         };
3969         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3970                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3971                 return rte_flow_error_set(error, EINVAL,
3972                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3973                                           "can't have 2 fate actions in"
3974                                           " same flow");
3975         if (action_flags & MLX5_FLOW_ACTION_METER)
3976                 return rte_flow_error_set(error, ENOTSUP,
3977                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3978                                           "jump with meter not support");
3979         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror)
3980                 return rte_flow_error_set(error, EINVAL,
3981                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3982                                           "E-Switch mirroring can't support"
3983                                           " Sample action and jump action in"
3984                                           " same flow now");
3985         if (!action->conf)
3986                 return rte_flow_error_set(error, EINVAL,
3987                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3988                                           NULL, "action configuration not set");
3989         target_group =
3990                 ((const struct rte_flow_action_jump *)action->conf)->group;
3991         ret = mlx5_flow_group_to_table(dev, tunnel, target_group, &table,
3992                                        grp_info, error);
3993         if (ret)
3994                 return ret;
3995         if (attributes->group == target_group &&
3996             !(action_flags & (MLX5_FLOW_ACTION_TUNNEL_SET |
3997                               MLX5_FLOW_ACTION_TUNNEL_MATCH)))
3998                 return rte_flow_error_set(error, EINVAL,
3999                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4000                                           "target group must be other than"
4001                                           " the current flow group");
4002         return 0;
4003 }
4004
4005 /*
4006  * Validate the port_id action.
4007  *
4008  * @param[in] dev
4009  *   Pointer to rte_eth_dev structure.
4010  * @param[in] action_flags
4011  *   Bit-fields that holds the actions detected until now.
4012  * @param[in] action
4013  *   Port_id RTE action structure.
4014  * @param[in] attr
4015  *   Attributes of flow that includes this action.
4016  * @param[out] error
4017  *   Pointer to error structure.
4018  *
4019  * @return
4020  *   0 on success, a negative errno value otherwise and rte_errno is set.
4021  */
4022 static int
4023 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
4024                                 uint64_t action_flags,
4025                                 const struct rte_flow_action *action,
4026                                 const struct rte_flow_attr *attr,
4027                                 struct rte_flow_error *error)
4028 {
4029         const struct rte_flow_action_port_id *port_id;
4030         struct mlx5_priv *act_priv;
4031         struct mlx5_priv *dev_priv;
4032         uint16_t port;
4033
4034         if (!attr->transfer)
4035                 return rte_flow_error_set(error, ENOTSUP,
4036                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4037                                           NULL,
4038                                           "port id action is valid in transfer"
4039                                           " mode only");
4040         if (!action || !action->conf)
4041                 return rte_flow_error_set(error, ENOTSUP,
4042                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4043                                           NULL,
4044                                           "port id action parameters must be"
4045                                           " specified");
4046         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
4047                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4048                 return rte_flow_error_set(error, EINVAL,
4049                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4050                                           "can have only one fate actions in"
4051                                           " a flow");
4052         dev_priv = mlx5_dev_to_eswitch_info(dev);
4053         if (!dev_priv)
4054                 return rte_flow_error_set(error, rte_errno,
4055                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4056                                           NULL,
4057                                           "failed to obtain E-Switch info");
4058         port_id = action->conf;
4059         port = port_id->original ? dev->data->port_id : port_id->id;
4060         act_priv = mlx5_port_to_eswitch_info(port, false);
4061         if (!act_priv)
4062                 return rte_flow_error_set
4063                                 (error, rte_errno,
4064                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
4065                                  "failed to obtain E-Switch port id for port");
4066         if (act_priv->domain_id != dev_priv->domain_id)
4067                 return rte_flow_error_set
4068                                 (error, EINVAL,
4069                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4070                                  "port does not belong to"
4071                                  " E-Switch being configured");
4072         return 0;
4073 }
4074
4075 /**
4076  * Get the maximum number of modify header actions.
4077  *
4078  * @param dev
4079  *   Pointer to rte_eth_dev structure.
4080  * @param flags
4081  *   Flags bits to check if root level.
4082  *
4083  * @return
4084  *   Max number of modify header actions device can support.
4085  */
4086 static inline unsigned int
4087 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
4088                               uint64_t flags)
4089 {
4090         /*
4091          * There's no way to directly query the max capacity from FW.
4092          * The maximal value on root table should be assumed to be supported.
4093          */
4094         if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
4095                 return MLX5_MAX_MODIFY_NUM;
4096         else
4097                 return MLX5_ROOT_TBL_MODIFY_NUM;
4098 }
4099
4100 /**
4101  * Validate the meter action.
4102  *
4103  * @param[in] dev
4104  *   Pointer to rte_eth_dev structure.
4105  * @param[in] action_flags
4106  *   Bit-fields that holds the actions detected until now.
4107  * @param[in] action
4108  *   Pointer to the meter action.
4109  * @param[in] attr
4110  *   Attributes of flow that includes this action.
4111  * @param[out] error
4112  *   Pointer to error structure.
4113  *
4114  * @return
4115  *   0 on success, a negative errno value otherwise and rte_ernno is set.
4116  */
4117 static int
4118 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
4119                                 uint64_t action_flags,
4120                                 const struct rte_flow_action *action,
4121                                 const struct rte_flow_attr *attr,
4122                                 struct rte_flow_error *error)
4123 {
4124         struct mlx5_priv *priv = dev->data->dev_private;
4125         const struct rte_flow_action_meter *am = action->conf;
4126         struct mlx5_flow_meter *fm;
4127
4128         if (!am)
4129                 return rte_flow_error_set(error, EINVAL,
4130                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4131                                           "meter action conf is NULL");
4132
4133         if (action_flags & MLX5_FLOW_ACTION_METER)
4134                 return rte_flow_error_set(error, ENOTSUP,
4135                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4136                                           "meter chaining not support");
4137         if (action_flags & MLX5_FLOW_ACTION_JUMP)
4138                 return rte_flow_error_set(error, ENOTSUP,
4139                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4140                                           "meter with jump not support");
4141         if (!priv->mtr_en)
4142                 return rte_flow_error_set(error, ENOTSUP,
4143                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4144                                           NULL,
4145                                           "meter action not supported");
4146         fm = mlx5_flow_meter_find(priv, am->mtr_id);
4147         if (!fm)
4148                 return rte_flow_error_set(error, EINVAL,
4149                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4150                                           "Meter not found");
4151         if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
4152               (!fm->ingress && !attr->ingress && attr->egress) ||
4153               (!fm->egress && !attr->egress && attr->ingress))))
4154                 return rte_flow_error_set(error, EINVAL,
4155                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4156                                           "Flow attributes are either invalid "
4157                                           "or have a conflict with current "
4158                                           "meter attributes");
4159         return 0;
4160 }
4161
4162 /**
4163  * Validate the age action.
4164  *
4165  * @param[in] action_flags
4166  *   Holds the actions detected until now.
4167  * @param[in] action
4168  *   Pointer to the age action.
4169  * @param[in] dev
4170  *   Pointer to the Ethernet device structure.
4171  * @param[out] error
4172  *   Pointer to error structure.
4173  *
4174  * @return
4175  *   0 on success, a negative errno value otherwise and rte_errno is set.
4176  */
4177 static int
4178 flow_dv_validate_action_age(uint64_t action_flags,
4179                             const struct rte_flow_action *action,
4180                             struct rte_eth_dev *dev,
4181                             struct rte_flow_error *error)
4182 {
4183         struct mlx5_priv *priv = dev->data->dev_private;
4184         const struct rte_flow_action_age *age = action->conf;
4185
4186         if (!priv->config.devx || priv->sh->cmng.counter_fallback)
4187                 return rte_flow_error_set(error, ENOTSUP,
4188                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4189                                           NULL,
4190                                           "age action not supported");
4191         if (!(action->conf))
4192                 return rte_flow_error_set(error, EINVAL,
4193                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4194                                           "configuration cannot be null");
4195         if (!(age->timeout))
4196                 return rte_flow_error_set(error, EINVAL,
4197                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4198                                           "invalid timeout value 0");
4199         if (action_flags & MLX5_FLOW_ACTION_AGE)
4200                 return rte_flow_error_set(error, EINVAL,
4201                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4202                                           "duplicate age actions set");
4203         return 0;
4204 }
4205
4206 /**
4207  * Validate the modify-header IPv4 DSCP actions.
4208  *
4209  * @param[in] action_flags
4210  *   Holds the actions detected until now.
4211  * @param[in] action
4212  *   Pointer to the modify action.
4213  * @param[in] item_flags
4214  *   Holds the items detected.
4215  * @param[out] error
4216  *   Pointer to error structure.
4217  *
4218  * @return
4219  *   0 on success, a negative errno value otherwise and rte_errno is set.
4220  */
4221 static int
4222 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
4223                                          const struct rte_flow_action *action,
4224                                          const uint64_t item_flags,
4225                                          struct rte_flow_error *error)
4226 {
4227         int ret = 0;
4228
4229         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4230         if (!ret) {
4231                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
4232                         return rte_flow_error_set(error, EINVAL,
4233                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4234                                                   NULL,
4235                                                   "no ipv4 item in pattern");
4236         }
4237         return ret;
4238 }
4239
4240 /**
4241  * Validate the modify-header IPv6 DSCP actions.
4242  *
4243  * @param[in] action_flags
4244  *   Holds the actions detected until now.
4245  * @param[in] action
4246  *   Pointer to the modify action.
4247  * @param[in] item_flags
4248  *   Holds the items detected.
4249  * @param[out] error
4250  *   Pointer to error structure.
4251  *
4252  * @return
4253  *   0 on success, a negative errno value otherwise and rte_errno is set.
4254  */
4255 static int
4256 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
4257                                          const struct rte_flow_action *action,
4258                                          const uint64_t item_flags,
4259                                          struct rte_flow_error *error)
4260 {
4261         int ret = 0;
4262
4263         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4264         if (!ret) {
4265                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
4266                         return rte_flow_error_set(error, EINVAL,
4267                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4268                                                   NULL,
4269                                                   "no ipv6 item in pattern");
4270         }
4271         return ret;
4272 }
4273
4274 /**
4275  * Match modify-header resource.
4276  *
4277  * @param entry
4278  *   Pointer to exist resource entry object.
4279  * @param ctx
4280  *   Pointer to new modify-header resource.
4281  *
4282  * @return
4283  *   0 on matching, -1 otherwise.
4284  */
4285 static int
4286 flow_dv_modify_hdr_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
4287 {
4288         struct mlx5_flow_dv_modify_hdr_resource *resource;
4289         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4290         uint32_t actions_len;
4291
4292         resource = (struct mlx5_flow_dv_modify_hdr_resource *)ctx;
4293         cache_resource = container_of(entry,
4294                                       struct mlx5_flow_dv_modify_hdr_resource,
4295                                       entry);
4296         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4297         if (resource->entry.key == cache_resource->entry.key &&
4298             resource->ft_type == cache_resource->ft_type &&
4299             resource->actions_num == cache_resource->actions_num &&
4300             resource->flags == cache_resource->flags &&
4301             !memcmp((const void *)resource->actions,
4302                     (const void *)cache_resource->actions,
4303                     actions_len))
4304                 return 0;
4305         return -1;
4306 }
4307
4308 /**
4309  * Validate the sample action.
4310  *
4311  * @param[in] action_flags
4312  *   Holds the actions detected until now.
4313  * @param[in] action
4314  *   Pointer to the sample action.
4315  * @param[in] dev
4316  *   Pointer to the Ethernet device structure.
4317  * @param[in] attr
4318  *   Attributes of flow that includes this action.
4319  * @param[out] error
4320  *   Pointer to error structure.
4321  *
4322  * @return
4323  *   0 on success, a negative errno value otherwise and rte_errno is set.
4324  */
4325 static int
4326 flow_dv_validate_action_sample(uint64_t action_flags,
4327                                const struct rte_flow_action *action,
4328                                struct rte_eth_dev *dev,
4329                                const struct rte_flow_attr *attr,
4330                                struct rte_flow_error *error)
4331 {
4332         struct mlx5_priv *priv = dev->data->dev_private;
4333         struct mlx5_dev_config *dev_conf = &priv->config;
4334         const struct rte_flow_action_sample *sample = action->conf;
4335         const struct rte_flow_action *act;
4336         uint64_t sub_action_flags = 0;
4337         uint16_t queue_index = 0xFFFF;
4338         int actions_n = 0;
4339         int ret;
4340         fdb_mirror = 0;
4341
4342         if (!sample)
4343                 return rte_flow_error_set(error, EINVAL,
4344                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4345                                           "configuration cannot be NULL");
4346         if (sample->ratio == 0)
4347                 return rte_flow_error_set(error, EINVAL,
4348                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4349                                           "ratio value starts from 1");
4350         if (!priv->config.devx || (sample->ratio > 0 && !priv->sampler_en))
4351                 return rte_flow_error_set(error, ENOTSUP,
4352                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4353                                           NULL,
4354                                           "sample action not supported");
4355         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
4356                 return rte_flow_error_set(error, EINVAL,
4357                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4358                                           "Multiple sample actions not "
4359                                           "supported");
4360         if (action_flags & MLX5_FLOW_ACTION_METER)
4361                 return rte_flow_error_set(error, EINVAL,
4362                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4363                                           "wrong action order, meter should "
4364                                           "be after sample action");
4365         if (action_flags & MLX5_FLOW_ACTION_JUMP)
4366                 return rte_flow_error_set(error, EINVAL,
4367                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4368                                           "wrong action order, jump should "
4369                                           "be after sample action");
4370         act = sample->actions;
4371         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
4372                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
4373                         return rte_flow_error_set(error, ENOTSUP,
4374                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4375                                                   act, "too many actions");
4376                 switch (act->type) {
4377                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4378                         ret = mlx5_flow_validate_action_queue(act,
4379                                                               sub_action_flags,
4380                                                               dev,
4381                                                               attr, error);
4382                         if (ret < 0)
4383                                 return ret;
4384                         queue_index = ((const struct rte_flow_action_queue *)
4385                                                         (act->conf))->index;
4386                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
4387                         ++actions_n;
4388                         break;
4389                 case RTE_FLOW_ACTION_TYPE_MARK:
4390                         ret = flow_dv_validate_action_mark(dev, act,
4391                                                            sub_action_flags,
4392                                                            attr, error);
4393                         if (ret < 0)
4394                                 return ret;
4395                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
4396                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
4397                                                 MLX5_FLOW_ACTION_MARK_EXT;
4398                         else
4399                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
4400                         ++actions_n;
4401                         break;
4402                 case RTE_FLOW_ACTION_TYPE_COUNT:
4403                         ret = flow_dv_validate_action_count(dev, error);
4404                         if (ret < 0)
4405                                 return ret;
4406                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
4407                         ++actions_n;
4408                         break;
4409                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
4410                         ret = flow_dv_validate_action_port_id(dev,
4411                                                               sub_action_flags,
4412                                                               act,
4413                                                               attr,
4414                                                               error);
4415                         if (ret)
4416                                 return ret;
4417                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
4418                         ++actions_n;
4419                         break;
4420                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4421                         ret = flow_dv_validate_action_raw_encap_decap
4422                                 (dev, NULL, act->conf, attr, &sub_action_flags,
4423                                  &actions_n, error);
4424                         if (ret < 0)
4425                                 return ret;
4426                         ++actions_n;
4427                         break;
4428                 default:
4429                         return rte_flow_error_set(error, ENOTSUP,
4430                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4431                                                   NULL,
4432                                                   "Doesn't support optional "
4433                                                   "action");
4434                 }
4435         }
4436         if (attr->ingress && !attr->transfer) {
4437                 if (!(sub_action_flags & MLX5_FLOW_ACTION_QUEUE))
4438                         return rte_flow_error_set(error, EINVAL,
4439                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4440                                                   NULL,
4441                                                   "Ingress must has a dest "
4442                                                   "QUEUE for Sample");
4443         } else if (attr->egress && !attr->transfer) {
4444                 return rte_flow_error_set(error, ENOTSUP,
4445                                           RTE_FLOW_ERROR_TYPE_ACTION,
4446                                           NULL,
4447                                           "Sample Only support Ingress "
4448                                           "or E-Switch");
4449         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
4450                 MLX5_ASSERT(attr->transfer);
4451                 if (sample->ratio > 1)
4452                         return rte_flow_error_set(error, ENOTSUP,
4453                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4454                                                   NULL,
4455                                                   "E-Switch doesn't support "
4456                                                   "any optional action "
4457                                                   "for sampling");
4458                 fdb_mirror = 1;
4459                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
4460                         return rte_flow_error_set(error, ENOTSUP,
4461                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4462                                                   NULL,
4463                                                   "unsupported action QUEUE");
4464                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
4465                         return rte_flow_error_set(error, EINVAL,
4466                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4467                                                   NULL,
4468                                                   "E-Switch must has a dest "
4469                                                   "port for mirroring");
4470         }
4471         /* Continue validation for Xcap actions.*/
4472         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
4473             (queue_index == 0xFFFF ||
4474              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
4475                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
4476                      MLX5_FLOW_XCAP_ACTIONS)
4477                         return rte_flow_error_set(error, ENOTSUP,
4478                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4479                                                   NULL, "encap and decap "
4480                                                   "combination aren't "
4481                                                   "supported");
4482                 if (!attr->transfer && attr->ingress && (sub_action_flags &
4483                                                         MLX5_FLOW_ACTION_ENCAP))
4484                         return rte_flow_error_set(error, ENOTSUP,
4485                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4486                                                   NULL, "encap is not supported"
4487                                                   " for ingress traffic");
4488         }
4489         return 0;
4490 }
4491
4492 /**
4493  * Find existing modify-header resource or create and register a new one.
4494  *
4495  * @param dev[in, out]
4496  *   Pointer to rte_eth_dev structure.
4497  * @param[in, out] resource
4498  *   Pointer to modify-header resource.
4499  * @parm[in, out] dev_flow
4500  *   Pointer to the dev_flow.
4501  * @param[out] error
4502  *   pointer to error structure.
4503  *
4504  * @return
4505  *   0 on success otherwise -errno and errno is set.
4506  */
4507 static int
4508 flow_dv_modify_hdr_resource_register
4509                         (struct rte_eth_dev *dev,
4510                          struct mlx5_flow_dv_modify_hdr_resource *resource,
4511                          struct mlx5_flow *dev_flow,
4512                          struct rte_flow_error *error)
4513 {
4514         struct mlx5_priv *priv = dev->data->dev_private;
4515         struct mlx5_dev_ctx_shared *sh = priv->sh;
4516         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4517         struct mlx5dv_dr_domain *ns;
4518         uint32_t actions_len;
4519         struct mlx5_hlist_entry *entry;
4520         union mlx5_flow_modify_hdr_key hdr_mod_key = {
4521                 {
4522                         .ft_type = resource->ft_type,
4523                         .actions_num = resource->actions_num,
4524                         .group = dev_flow->dv.group,
4525                         .cksum = 0,
4526                 }
4527         };
4528         int ret;
4529
4530         resource->flags = dev_flow->dv.group ? 0 :
4531                           MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4532         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
4533                                     resource->flags))
4534                 return rte_flow_error_set(error, EOVERFLOW,
4535                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4536                                           "too many modify header items");
4537         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
4538                 ns = sh->fdb_domain;
4539         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
4540                 ns = sh->tx_domain;
4541         else
4542                 ns = sh->rx_domain;
4543         /* Lookup a matching resource from cache. */
4544         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4545         hdr_mod_key.cksum = __rte_raw_cksum(resource->actions, actions_len, 0);
4546         resource->entry.key = hdr_mod_key.v64;
4547         entry = mlx5_hlist_lookup_ex(sh->modify_cmds, resource->entry.key,
4548                                      flow_dv_modify_hdr_resource_match,
4549                                      (void *)resource);
4550         if (entry) {
4551                 cache_resource = container_of(entry,
4552                                         struct mlx5_flow_dv_modify_hdr_resource,
4553                                         entry);
4554                 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
4555                         (void *)cache_resource,
4556                         __atomic_load_n(&cache_resource->refcnt,
4557                                         __ATOMIC_RELAXED));
4558                 __atomic_fetch_add(&cache_resource->refcnt, 1,
4559                                    __ATOMIC_RELAXED);
4560                 dev_flow->handle->dvh.modify_hdr = cache_resource;
4561                 return 0;
4562
4563         }
4564         /* Register new modify-header resource. */
4565         cache_resource = mlx5_malloc(MLX5_MEM_ZERO,
4566                                     sizeof(*cache_resource) + actions_len, 0,
4567                                     SOCKET_ID_ANY);
4568         if (!cache_resource)
4569                 return rte_flow_error_set(error, ENOMEM,
4570                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4571                                           "cannot allocate resource memory");
4572         *cache_resource = *resource;
4573         rte_memcpy(cache_resource->actions, resource->actions, actions_len);
4574         ret = mlx5_flow_os_create_flow_action_modify_header
4575                                         (sh->ctx, ns, cache_resource,
4576                                          actions_len, &cache_resource->action);
4577         if (ret) {
4578                 mlx5_free(cache_resource);
4579                 return rte_flow_error_set(error, ENOMEM,
4580                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4581                                           NULL, "cannot create action");
4582         }
4583         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
4584         if (mlx5_hlist_insert_ex(sh->modify_cmds, &cache_resource->entry,
4585                                  flow_dv_modify_hdr_resource_match,
4586                                  (void *)cache_resource)) {
4587                 claim_zero(mlx5_flow_os_destroy_flow_action
4588                                                 (cache_resource->action));
4589                 mlx5_free(cache_resource);
4590                 return rte_flow_error_set(error, EEXIST,
4591                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4592                                           NULL, "action exist");
4593         }
4594         dev_flow->handle->dvh.modify_hdr = cache_resource;
4595         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
4596                 (void *)cache_resource,
4597                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
4598         return 0;
4599 }
4600
4601 /**
4602  * Get DV flow counter by index.
4603  *
4604  * @param[in] dev
4605  *   Pointer to the Ethernet device structure.
4606  * @param[in] idx
4607  *   mlx5 flow counter index in the container.
4608  * @param[out] ppool
4609  *   mlx5 flow counter pool in the container,
4610  *
4611  * @return
4612  *   Pointer to the counter, NULL otherwise.
4613  */
4614 static struct mlx5_flow_counter *
4615 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
4616                            uint32_t idx,
4617                            struct mlx5_flow_counter_pool **ppool)
4618 {
4619         struct mlx5_priv *priv = dev->data->dev_private;
4620         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4621         struct mlx5_flow_counter_pool *pool;
4622
4623         /* Decrease to original index and clear shared bit. */
4624         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
4625         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cmng->n);
4626         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
4627         MLX5_ASSERT(pool);
4628         if (ppool)
4629                 *ppool = pool;
4630         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
4631 }
4632
4633 /**
4634  * Check the devx counter belongs to the pool.
4635  *
4636  * @param[in] pool
4637  *   Pointer to the counter pool.
4638  * @param[in] id
4639  *   The counter devx ID.
4640  *
4641  * @return
4642  *   True if counter belongs to the pool, false otherwise.
4643  */
4644 static bool
4645 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
4646 {
4647         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
4648                    MLX5_COUNTERS_PER_POOL;
4649
4650         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
4651                 return true;
4652         return false;
4653 }
4654
4655 /**
4656  * Get a pool by devx counter ID.
4657  *
4658  * @param[in] cmng
4659  *   Pointer to the counter management.
4660  * @param[in] id
4661  *   The counter devx ID.
4662  *
4663  * @return
4664  *   The counter pool pointer if exists, NULL otherwise,
4665  */
4666 static struct mlx5_flow_counter_pool *
4667 flow_dv_find_pool_by_id(struct mlx5_flow_counter_mng *cmng, int id)
4668 {
4669         uint32_t i;
4670         struct mlx5_flow_counter_pool *pool = NULL;
4671
4672         rte_spinlock_lock(&cmng->pool_update_sl);
4673         /* Check last used pool. */
4674         if (cmng->last_pool_idx != POOL_IDX_INVALID &&
4675             flow_dv_is_counter_in_pool(cmng->pools[cmng->last_pool_idx], id)) {
4676                 pool = cmng->pools[cmng->last_pool_idx];
4677                 goto out;
4678         }
4679         /* ID out of range means no suitable pool in the container. */
4680         if (id > cmng->max_id || id < cmng->min_id)
4681                 goto out;
4682         /*
4683          * Find the pool from the end of the container, since mostly counter
4684          * ID is sequence increasing, and the last pool should be the needed
4685          * one.
4686          */
4687         i = cmng->n_valid;
4688         while (i--) {
4689                 struct mlx5_flow_counter_pool *pool_tmp = cmng->pools[i];
4690
4691                 if (flow_dv_is_counter_in_pool(pool_tmp, id)) {
4692                         pool = pool_tmp;
4693                         break;
4694                 }
4695         }
4696 out:
4697         rte_spinlock_unlock(&cmng->pool_update_sl);
4698         return pool;
4699 }
4700
4701 /**
4702  * Resize a counter container.
4703  *
4704  * @param[in] dev
4705  *   Pointer to the Ethernet device structure.
4706  *
4707  * @return
4708  *   0 on success, otherwise negative errno value and rte_errno is set.
4709  */
4710 static int
4711 flow_dv_container_resize(struct rte_eth_dev *dev)
4712 {
4713         struct mlx5_priv *priv = dev->data->dev_private;
4714         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4715         void *old_pools = cmng->pools;
4716         uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
4717         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
4718         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
4719
4720         if (!pools) {
4721                 rte_errno = ENOMEM;
4722                 return -ENOMEM;
4723         }
4724         if (old_pools)
4725                 memcpy(pools, old_pools, cmng->n *
4726                                        sizeof(struct mlx5_flow_counter_pool *));
4727         cmng->n = resize;
4728         cmng->pools = pools;
4729         if (old_pools)
4730                 mlx5_free(old_pools);
4731         return 0;
4732 }
4733
4734 /**
4735  * Query a devx flow counter.
4736  *
4737  * @param[in] dev
4738  *   Pointer to the Ethernet device structure.
4739  * @param[in] cnt
4740  *   Index to the flow counter.
4741  * @param[out] pkts
4742  *   The statistics value of packets.
4743  * @param[out] bytes
4744  *   The statistics value of bytes.
4745  *
4746  * @return
4747  *   0 on success, otherwise a negative errno value and rte_errno is set.
4748  */
4749 static inline int
4750 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
4751                      uint64_t *bytes)
4752 {
4753         struct mlx5_priv *priv = dev->data->dev_private;
4754         struct mlx5_flow_counter_pool *pool = NULL;
4755         struct mlx5_flow_counter *cnt;
4756         int offset;
4757
4758         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4759         MLX5_ASSERT(pool);
4760         if (priv->sh->cmng.counter_fallback)
4761                 return mlx5_devx_cmd_flow_counter_query(cnt->dcs_when_active, 0,
4762                                         0, pkts, bytes, 0, NULL, NULL, 0);
4763         rte_spinlock_lock(&pool->sl);
4764         if (!pool->raw) {
4765                 *pkts = 0;
4766                 *bytes = 0;
4767         } else {
4768                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
4769                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
4770                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
4771         }
4772         rte_spinlock_unlock(&pool->sl);
4773         return 0;
4774 }
4775
4776 /**
4777  * Create and initialize a new counter pool.
4778  *
4779  * @param[in] dev
4780  *   Pointer to the Ethernet device structure.
4781  * @param[out] dcs
4782  *   The devX counter handle.
4783  * @param[in] age
4784  *   Whether the pool is for counter that was allocated for aging.
4785  * @param[in/out] cont_cur
4786  *   Pointer to the container pointer, it will be update in pool resize.
4787  *
4788  * @return
4789  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
4790  */
4791 static struct mlx5_flow_counter_pool *
4792 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
4793                     uint32_t age)
4794 {
4795         struct mlx5_priv *priv = dev->data->dev_private;
4796         struct mlx5_flow_counter_pool *pool;
4797         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4798         bool fallback = priv->sh->cmng.counter_fallback;
4799         uint32_t size = sizeof(*pool);
4800
4801         size += MLX5_COUNTERS_PER_POOL * MLX5_CNT_SIZE;
4802         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * MLX5_AGE_SIZE);
4803         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
4804         if (!pool) {
4805                 rte_errno = ENOMEM;
4806                 return NULL;
4807         }
4808         pool->raw = NULL;
4809         pool->is_aged = !!age;
4810         pool->query_gen = 0;
4811         pool->min_dcs = dcs;
4812         rte_spinlock_init(&pool->sl);
4813         rte_spinlock_init(&pool->csl);
4814         TAILQ_INIT(&pool->counters[0]);
4815         TAILQ_INIT(&pool->counters[1]);
4816         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
4817         rte_spinlock_lock(&cmng->pool_update_sl);
4818         pool->index = cmng->n_valid;
4819         if (pool->index == cmng->n && flow_dv_container_resize(dev)) {
4820                 mlx5_free(pool);
4821                 rte_spinlock_unlock(&cmng->pool_update_sl);
4822                 return NULL;
4823         }
4824         cmng->pools[pool->index] = pool;
4825         cmng->n_valid++;
4826         if (unlikely(fallback)) {
4827                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
4828
4829                 if (base < cmng->min_id)
4830                         cmng->min_id = base;
4831                 if (base > cmng->max_id)
4832                         cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
4833                 cmng->last_pool_idx = pool->index;
4834         }
4835         rte_spinlock_unlock(&cmng->pool_update_sl);
4836         return pool;
4837 }
4838
4839 /**
4840  * Prepare a new counter and/or a new counter pool.
4841  *
4842  * @param[in] dev
4843  *   Pointer to the Ethernet device structure.
4844  * @param[out] cnt_free
4845  *   Where to put the pointer of a new counter.
4846  * @param[in] age
4847  *   Whether the pool is for counter that was allocated for aging.
4848  *
4849  * @return
4850  *   The counter pool pointer and @p cnt_free is set on success,
4851  *   NULL otherwise and rte_errno is set.
4852  */
4853 static struct mlx5_flow_counter_pool *
4854 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
4855                              struct mlx5_flow_counter **cnt_free,
4856                              uint32_t age)
4857 {
4858         struct mlx5_priv *priv = dev->data->dev_private;
4859         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4860         struct mlx5_flow_counter_pool *pool;
4861         struct mlx5_counters tmp_tq;
4862         struct mlx5_devx_obj *dcs = NULL;
4863         struct mlx5_flow_counter *cnt;
4864         enum mlx5_counter_type cnt_type =
4865                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
4866         bool fallback = priv->sh->cmng.counter_fallback;
4867         uint32_t i;
4868
4869         if (fallback) {
4870                 /* bulk_bitmap must be 0 for single counter allocation. */
4871                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
4872                 if (!dcs)
4873                         return NULL;
4874                 pool = flow_dv_find_pool_by_id(cmng, dcs->id);
4875                 if (!pool) {
4876                         pool = flow_dv_pool_create(dev, dcs, age);
4877                         if (!pool) {
4878                                 mlx5_devx_cmd_destroy(dcs);
4879                                 return NULL;
4880                         }
4881                 }
4882                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
4883                 cnt = MLX5_POOL_GET_CNT(pool, i);
4884                 cnt->pool = pool;
4885                 cnt->dcs_when_free = dcs;
4886                 *cnt_free = cnt;
4887                 return pool;
4888         }
4889         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
4890         if (!dcs) {
4891                 rte_errno = ENODATA;
4892                 return NULL;
4893         }
4894         pool = flow_dv_pool_create(dev, dcs, age);
4895         if (!pool) {
4896                 mlx5_devx_cmd_destroy(dcs);
4897                 return NULL;
4898         }
4899         TAILQ_INIT(&tmp_tq);
4900         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
4901                 cnt = MLX5_POOL_GET_CNT(pool, i);
4902                 cnt->pool = pool;
4903                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
4904         }
4905         rte_spinlock_lock(&cmng->csl[cnt_type]);
4906         TAILQ_CONCAT(&cmng->counters[cnt_type], &tmp_tq, next);
4907         rte_spinlock_unlock(&cmng->csl[cnt_type]);
4908         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
4909         (*cnt_free)->pool = pool;
4910         return pool;
4911 }
4912
4913 /**
4914  * Allocate a flow counter.
4915  *
4916  * @param[in] dev
4917  *   Pointer to the Ethernet device structure.
4918  * @param[in] age
4919  *   Whether the counter was allocated for aging.
4920  *
4921  * @return
4922  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
4923  */
4924 static uint32_t
4925 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t age)
4926 {
4927         struct mlx5_priv *priv = dev->data->dev_private;
4928         struct mlx5_flow_counter_pool *pool = NULL;
4929         struct mlx5_flow_counter *cnt_free = NULL;
4930         bool fallback = priv->sh->cmng.counter_fallback;
4931         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4932         enum mlx5_counter_type cnt_type =
4933                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
4934         uint32_t cnt_idx;
4935
4936         if (!priv->config.devx) {
4937                 rte_errno = ENOTSUP;
4938                 return 0;
4939         }
4940         /* Get free counters from container. */
4941         rte_spinlock_lock(&cmng->csl[cnt_type]);
4942         cnt_free = TAILQ_FIRST(&cmng->counters[cnt_type]);
4943         if (cnt_free)
4944                 TAILQ_REMOVE(&cmng->counters[cnt_type], cnt_free, next);
4945         rte_spinlock_unlock(&cmng->csl[cnt_type]);
4946         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, age))
4947                 goto err;
4948         pool = cnt_free->pool;
4949         if (fallback)
4950                 cnt_free->dcs_when_active = cnt_free->dcs_when_free;
4951         /* Create a DV counter action only in the first time usage. */
4952         if (!cnt_free->action) {
4953                 uint16_t offset;
4954                 struct mlx5_devx_obj *dcs;
4955                 int ret;
4956
4957                 if (!fallback) {
4958                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
4959                         dcs = pool->min_dcs;
4960                 } else {
4961                         offset = 0;
4962                         dcs = cnt_free->dcs_when_free;
4963                 }
4964                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
4965                                                             &cnt_free->action);
4966                 if (ret) {
4967                         rte_errno = errno;
4968                         goto err;
4969                 }
4970         }
4971         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
4972                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
4973         /* Update the counter reset values. */
4974         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
4975                                  &cnt_free->bytes))
4976                 goto err;
4977         if (!fallback && !priv->sh->cmng.query_thread_on)
4978                 /* Start the asynchronous batch query by the host thread. */
4979                 mlx5_set_query_alarm(priv->sh);
4980         return cnt_idx;
4981 err:
4982         if (cnt_free) {
4983                 cnt_free->pool = pool;
4984                 if (fallback)
4985                         cnt_free->dcs_when_free = cnt_free->dcs_when_active;
4986                 rte_spinlock_lock(&cmng->csl[cnt_type]);
4987                 TAILQ_INSERT_TAIL(&cmng->counters[cnt_type], cnt_free, next);
4988                 rte_spinlock_unlock(&cmng->csl[cnt_type]);
4989         }
4990         return 0;
4991 }
4992
4993 /**
4994  * Allocate a shared flow counter.
4995  *
4996  * @param[in] ctx
4997  *   Pointer to the shared counter configuration.
4998  * @param[in] data
4999  *   Pointer to save the allocated counter index.
5000  *
5001  * @return
5002  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
5003  */
5004
5005 static int32_t
5006 flow_dv_counter_alloc_shared_cb(void *ctx, union mlx5_l3t_data *data)
5007 {
5008         struct mlx5_shared_counter_conf *conf = ctx;
5009         struct rte_eth_dev *dev = conf->dev;
5010         struct mlx5_flow_counter *cnt;
5011
5012         data->dword = flow_dv_counter_alloc(dev, 0);
5013         data->dword |= MLX5_CNT_SHARED_OFFSET;
5014         cnt = flow_dv_counter_get_by_idx(dev, data->dword, NULL);
5015         cnt->shared_info.id = conf->id;
5016         return 0;
5017 }
5018
5019 /**
5020  * Get a shared flow counter.
5021  *
5022  * @param[in] dev
5023  *   Pointer to the Ethernet device structure.
5024  * @param[in] id
5025  *   Counter identifier.
5026  *
5027  * @return
5028  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
5029  */
5030 static uint32_t
5031 flow_dv_counter_get_shared(struct rte_eth_dev *dev, uint32_t id)
5032 {
5033         struct mlx5_priv *priv = dev->data->dev_private;
5034         struct mlx5_shared_counter_conf conf = {
5035                 .dev = dev,
5036                 .id = id,
5037         };
5038         union mlx5_l3t_data data = {
5039                 .dword = 0,
5040         };
5041
5042         mlx5_l3t_prepare_entry(priv->sh->cnt_id_tbl, id, &data,
5043                                flow_dv_counter_alloc_shared_cb, &conf);
5044         return data.dword;
5045 }
5046
5047 /**
5048  * Get age param from counter index.
5049  *
5050  * @param[in] dev
5051  *   Pointer to the Ethernet device structure.
5052  * @param[in] counter
5053  *   Index to the counter handler.
5054  *
5055  * @return
5056  *   The aging parameter specified for the counter index.
5057  */
5058 static struct mlx5_age_param*
5059 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
5060                                 uint32_t counter)
5061 {
5062         struct mlx5_flow_counter *cnt;
5063         struct mlx5_flow_counter_pool *pool = NULL;
5064
5065         flow_dv_counter_get_by_idx(dev, counter, &pool);
5066         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
5067         cnt = MLX5_POOL_GET_CNT(pool, counter);
5068         return MLX5_CNT_TO_AGE(cnt);
5069 }
5070
5071 /**
5072  * Remove a flow counter from aged counter list.
5073  *
5074  * @param[in] dev
5075  *   Pointer to the Ethernet device structure.
5076  * @param[in] counter
5077  *   Index to the counter handler.
5078  * @param[in] cnt
5079  *   Pointer to the counter handler.
5080  */
5081 static void
5082 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
5083                                 uint32_t counter, struct mlx5_flow_counter *cnt)
5084 {
5085         struct mlx5_age_info *age_info;
5086         struct mlx5_age_param *age_param;
5087         struct mlx5_priv *priv = dev->data->dev_private;
5088         uint16_t expected = AGE_CANDIDATE;
5089
5090         age_info = GET_PORT_AGE_INFO(priv);
5091         age_param = flow_dv_counter_idx_get_age(dev, counter);
5092         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
5093                                          AGE_FREE, false, __ATOMIC_RELAXED,
5094                                          __ATOMIC_RELAXED)) {
5095                 /**
5096                  * We need the lock even it is age timeout,
5097                  * since counter may still in process.
5098                  */
5099                 rte_spinlock_lock(&age_info->aged_sl);
5100                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
5101                 rte_spinlock_unlock(&age_info->aged_sl);
5102                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
5103         }
5104 }
5105
5106 /**
5107  * Release a flow counter.
5108  *
5109  * @param[in] dev
5110  *   Pointer to the Ethernet device structure.
5111  * @param[in] counter
5112  *   Index to the counter handler.
5113  */
5114 static void
5115 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
5116 {
5117         struct mlx5_priv *priv = dev->data->dev_private;
5118         struct mlx5_flow_counter_pool *pool = NULL;
5119         struct mlx5_flow_counter *cnt;
5120         enum mlx5_counter_type cnt_type;
5121
5122         if (!counter)
5123                 return;
5124         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
5125         MLX5_ASSERT(pool);
5126         if (IS_SHARED_CNT(counter) &&
5127             mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, cnt->shared_info.id))
5128                 return;
5129         if (pool->is_aged)
5130                 flow_dv_counter_remove_from_age(dev, counter, cnt);
5131         cnt->pool = pool;
5132         /*
5133          * Put the counter back to list to be updated in none fallback mode.
5134          * Currently, we are using two list alternately, while one is in query,
5135          * add the freed counter to the other list based on the pool query_gen
5136          * value. After query finishes, add counter the list to the global
5137          * container counter list. The list changes while query starts. In
5138          * this case, lock will not be needed as query callback and release
5139          * function both operate with the different list.
5140          *
5141          */
5142         if (!priv->sh->cmng.counter_fallback) {
5143                 rte_spinlock_lock(&pool->csl);
5144                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
5145                 rte_spinlock_unlock(&pool->csl);
5146         } else {
5147                 cnt->dcs_when_free = cnt->dcs_when_active;
5148                 cnt_type = pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
5149                                            MLX5_COUNTER_TYPE_ORIGIN;
5150                 rte_spinlock_lock(&priv->sh->cmng.csl[cnt_type]);
5151                 TAILQ_INSERT_TAIL(&priv->sh->cmng.counters[cnt_type],
5152                                   cnt, next);
5153                 rte_spinlock_unlock(&priv->sh->cmng.csl[cnt_type]);
5154         }
5155 }
5156
5157 /**
5158  * Verify the @p attributes will be correctly understood by the NIC and store
5159  * them in the @p flow if everything is correct.
5160  *
5161  * @param[in] dev
5162  *   Pointer to dev struct.
5163  * @param[in] attributes
5164  *   Pointer to flow attributes
5165  * @param[in] external
5166  *   This flow rule is created by request external to PMD.
5167  * @param[out] error
5168  *   Pointer to error structure.
5169  *
5170  * @return
5171  *   - 0 on success and non root table.
5172  *   - 1 on success and root table.
5173  *   - a negative errno value otherwise and rte_errno is set.
5174  */
5175 static int
5176 flow_dv_validate_attributes(struct rte_eth_dev *dev,
5177                             const struct mlx5_flow_tunnel *tunnel,
5178                             const struct rte_flow_attr *attributes,
5179                             struct flow_grp_info grp_info,
5180                             struct rte_flow_error *error)
5181 {
5182         struct mlx5_priv *priv = dev->data->dev_private;
5183         uint32_t priority_max = priv->config.flow_prio - 1;
5184         int ret = 0;
5185
5186 #ifndef HAVE_MLX5DV_DR
5187         RTE_SET_USED(tunnel);
5188         RTE_SET_USED(grp_info);
5189         if (attributes->group)
5190                 return rte_flow_error_set(error, ENOTSUP,
5191                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
5192                                           NULL,
5193                                           "groups are not supported");
5194 #else
5195         uint32_t table = 0;
5196
5197         ret = mlx5_flow_group_to_table(dev, tunnel, attributes->group, &table,
5198                                        grp_info, error);
5199         if (ret)
5200                 return ret;
5201         if (!table)
5202                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
5203 #endif
5204         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
5205             attributes->priority >= priority_max)
5206                 return rte_flow_error_set(error, ENOTSUP,
5207                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
5208                                           NULL,
5209                                           "priority out of range");
5210         if (attributes->transfer) {
5211                 if (!priv->config.dv_esw_en)
5212                         return rte_flow_error_set
5213                                 (error, ENOTSUP,
5214                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5215                                  "E-Switch dr is not supported");
5216                 if (!(priv->representor || priv->master))
5217                         return rte_flow_error_set
5218                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5219                                  NULL, "E-Switch configuration can only be"
5220                                  " done by a master or a representor device");
5221                 if (attributes->egress)
5222                         return rte_flow_error_set
5223                                 (error, ENOTSUP,
5224                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
5225                                  "egress is not supported");
5226         }
5227         if (!(attributes->egress ^ attributes->ingress))
5228                 return rte_flow_error_set(error, ENOTSUP,
5229                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
5230                                           "must specify exactly one of "
5231                                           "ingress or egress");
5232         return ret;
5233 }
5234
5235 /**
5236  * Internal validation function. For validating both actions and items.
5237  *
5238  * @param[in] dev
5239  *   Pointer to the rte_eth_dev structure.
5240  * @param[in] attr
5241  *   Pointer to the flow attributes.
5242  * @param[in] items
5243  *   Pointer to the list of items.
5244  * @param[in] actions
5245  *   Pointer to the list of actions.
5246  * @param[in] external
5247  *   This flow rule is created by request external to PMD.
5248  * @param[in] hairpin
5249  *   Number of hairpin TX actions, 0 means classic flow.
5250  * @param[out] error
5251  *   Pointer to the error structure.
5252  *
5253  * @return
5254  *   0 on success, a negative errno value otherwise and rte_errno is set.
5255  */
5256 static int
5257 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
5258                  const struct rte_flow_item items[],
5259                  const struct rte_flow_action actions[],
5260                  bool external, int hairpin, struct rte_flow_error *error)
5261 {
5262         int ret;
5263         uint64_t action_flags = 0;
5264         uint64_t item_flags = 0;
5265         uint64_t last_item = 0;
5266         uint8_t next_protocol = 0xff;
5267         uint16_t ether_type = 0;
5268         int actions_n = 0;
5269         uint8_t item_ipv6_proto = 0;
5270         const struct rte_flow_item *gre_item = NULL;
5271         const struct rte_flow_action_raw_decap *decap;
5272         const struct rte_flow_action_raw_encap *encap;
5273         const struct rte_flow_action_rss *rss;
5274         const struct rte_flow_item_tcp nic_tcp_mask = {
5275                 .hdr = {
5276                         .tcp_flags = 0xFF,
5277                         .src_port = RTE_BE16(UINT16_MAX),
5278                         .dst_port = RTE_BE16(UINT16_MAX),
5279                 }
5280         };
5281         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
5282                 .hdr = {
5283                         .src_addr =
5284                         "\xff\xff\xff\xff\xff\xff\xff\xff"
5285                         "\xff\xff\xff\xff\xff\xff\xff\xff",
5286                         .dst_addr =
5287                         "\xff\xff\xff\xff\xff\xff\xff\xff"
5288                         "\xff\xff\xff\xff\xff\xff\xff\xff",
5289                         .vtc_flow = RTE_BE32(0xffffffff),
5290                         .proto = 0xff,
5291                         .hop_limits = 0xff,
5292                 },
5293                 .has_frag_ext = 1,
5294         };
5295         const struct rte_flow_item_ecpri nic_ecpri_mask = {
5296                 .hdr = {
5297                         .common = {
5298                                 .u32 =
5299                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
5300                                         .type = 0xFF,
5301                                         }).u32),
5302                         },
5303                         .dummy[0] = 0xffffffff,
5304                 },
5305         };
5306         struct mlx5_priv *priv = dev->data->dev_private;
5307         struct mlx5_dev_config *dev_conf = &priv->config;
5308         uint16_t queue_index = 0xFFFF;
5309         const struct rte_flow_item_vlan *vlan_m = NULL;
5310         int16_t rw_act_num = 0;
5311         uint64_t is_root;
5312         const struct mlx5_flow_tunnel *tunnel;
5313         struct flow_grp_info grp_info = {
5314                 .external = !!external,
5315                 .transfer = !!attr->transfer,
5316                 .fdb_def_rule = !!priv->fdb_def_rule,
5317         };
5318         const struct rte_eth_hairpin_conf *conf;
5319
5320         if (items == NULL)
5321                 return -1;
5322         if (is_flow_tunnel_match_rule(dev, attr, items, actions)) {
5323                 tunnel = flow_items_to_tunnel(items);
5324                 action_flags |= MLX5_FLOW_ACTION_TUNNEL_MATCH |
5325                                 MLX5_FLOW_ACTION_DECAP;
5326         } else if (is_flow_tunnel_steer_rule(dev, attr, items, actions)) {
5327                 tunnel = flow_actions_to_tunnel(actions);
5328                 action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
5329         } else {
5330                 tunnel = NULL;
5331         }
5332         grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
5333                                 (dev, tunnel, attr, items, actions);
5334         ret = flow_dv_validate_attributes(dev, tunnel, attr, grp_info, error);
5335         if (ret < 0)
5336                 return ret;
5337         is_root = (uint64_t)ret;
5338         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5339                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
5340                 int type = items->type;
5341
5342                 if (!mlx5_flow_os_item_supported(type))
5343                         return rte_flow_error_set(error, ENOTSUP,
5344                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5345                                                   NULL, "item not supported");
5346                 switch (type) {
5347                 case MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL:
5348                         if (items[0].type != (typeof(items[0].type))
5349                                                 MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL)
5350                                 return rte_flow_error_set
5351                                                 (error, EINVAL,
5352                                                 RTE_FLOW_ERROR_TYPE_ITEM,
5353                                                 NULL, "MLX5 private items "
5354                                                 "must be the first");
5355                         break;
5356                 case RTE_FLOW_ITEM_TYPE_VOID:
5357                         break;
5358                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
5359                         ret = flow_dv_validate_item_port_id
5360                                         (dev, items, attr, item_flags, error);
5361                         if (ret < 0)
5362                                 return ret;
5363                         last_item = MLX5_FLOW_ITEM_PORT_ID;
5364                         break;
5365                 case RTE_FLOW_ITEM_TYPE_ETH:
5366                         ret = mlx5_flow_validate_item_eth(items, item_flags,
5367                                                           true, error);
5368                         if (ret < 0)
5369                                 return ret;
5370                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
5371                                              MLX5_FLOW_LAYER_OUTER_L2;
5372                         if (items->mask != NULL && items->spec != NULL) {
5373                                 ether_type =
5374                                         ((const struct rte_flow_item_eth *)
5375                                          items->spec)->type;
5376                                 ether_type &=
5377                                         ((const struct rte_flow_item_eth *)
5378                                          items->mask)->type;
5379                                 ether_type = rte_be_to_cpu_16(ether_type);
5380                         } else {
5381                                 ether_type = 0;
5382                         }
5383                         break;
5384                 case RTE_FLOW_ITEM_TYPE_VLAN:
5385                         ret = flow_dv_validate_item_vlan(items, item_flags,
5386                                                          dev, error);
5387                         if (ret < 0)
5388                                 return ret;
5389                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
5390                                              MLX5_FLOW_LAYER_OUTER_VLAN;
5391                         if (items->mask != NULL && items->spec != NULL) {
5392                                 ether_type =
5393                                         ((const struct rte_flow_item_vlan *)
5394                                          items->spec)->inner_type;
5395                                 ether_type &=
5396                                         ((const struct rte_flow_item_vlan *)
5397                                          items->mask)->inner_type;
5398                                 ether_type = rte_be_to_cpu_16(ether_type);
5399                         } else {
5400                                 ether_type = 0;
5401                         }
5402                         /* Store outer VLAN mask for of_push_vlan action. */
5403                         if (!tunnel)
5404                                 vlan_m = items->mask;
5405                         break;
5406                 case RTE_FLOW_ITEM_TYPE_IPV4:
5407                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5408                                                   &item_flags, &tunnel);
5409                         ret = flow_dv_validate_item_ipv4(items, item_flags,
5410                                                          last_item, ether_type,
5411                                                          error);
5412                         if (ret < 0)
5413                                 return ret;
5414                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5415                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5416                         if (items->mask != NULL &&
5417                             ((const struct rte_flow_item_ipv4 *)
5418                              items->mask)->hdr.next_proto_id) {
5419                                 next_protocol =
5420                                         ((const struct rte_flow_item_ipv4 *)
5421                                          (items->spec))->hdr.next_proto_id;
5422                                 next_protocol &=
5423                                         ((const struct rte_flow_item_ipv4 *)
5424                                          (items->mask))->hdr.next_proto_id;
5425                         } else {
5426                                 /* Reset for inner layer. */
5427                                 next_protocol = 0xff;
5428                         }
5429                         break;
5430                 case RTE_FLOW_ITEM_TYPE_IPV6:
5431                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5432                                                   &item_flags, &tunnel);
5433                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
5434                                                            last_item,
5435                                                            ether_type,
5436                                                            &nic_ipv6_mask,
5437                                                            error);
5438                         if (ret < 0)
5439                                 return ret;
5440                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5441                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5442                         if (items->mask != NULL &&
5443                             ((const struct rte_flow_item_ipv6 *)
5444                              items->mask)->hdr.proto) {
5445                                 item_ipv6_proto =
5446                                         ((const struct rte_flow_item_ipv6 *)
5447                                          items->spec)->hdr.proto;
5448                                 next_protocol =
5449                                         ((const struct rte_flow_item_ipv6 *)
5450                                          items->spec)->hdr.proto;
5451                                 next_protocol &=
5452                                         ((const struct rte_flow_item_ipv6 *)
5453                                          items->mask)->hdr.proto;
5454                         } else {
5455                                 /* Reset for inner layer. */
5456                                 next_protocol = 0xff;
5457                         }
5458                         break;
5459                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
5460                         ret = flow_dv_validate_item_ipv6_frag_ext(items,
5461                                                                   item_flags,
5462                                                                   error);
5463                         if (ret < 0)
5464                                 return ret;
5465                         last_item = tunnel ?
5466                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
5467                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
5468                         if (items->mask != NULL &&
5469                             ((const struct rte_flow_item_ipv6_frag_ext *)
5470                              items->mask)->hdr.next_header) {
5471                                 next_protocol =
5472                                 ((const struct rte_flow_item_ipv6_frag_ext *)
5473                                  items->spec)->hdr.next_header;
5474                                 next_protocol &=
5475                                 ((const struct rte_flow_item_ipv6_frag_ext *)
5476                                  items->mask)->hdr.next_header;
5477                         } else {
5478                                 /* Reset for inner layer. */
5479                                 next_protocol = 0xff;
5480                         }
5481                         break;
5482                 case RTE_FLOW_ITEM_TYPE_TCP:
5483                         ret = mlx5_flow_validate_item_tcp
5484                                                 (items, item_flags,
5485                                                  next_protocol,
5486                                                  &nic_tcp_mask,
5487                                                  error);
5488                         if (ret < 0)
5489                                 return ret;
5490                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5491                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5492                         break;
5493                 case RTE_FLOW_ITEM_TYPE_UDP:
5494                         ret = mlx5_flow_validate_item_udp(items, item_flags,
5495                                                           next_protocol,
5496                                                           error);
5497                         if (ret < 0)
5498                                 return ret;
5499                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5500                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5501                         break;
5502                 case RTE_FLOW_ITEM_TYPE_GRE:
5503                         ret = mlx5_flow_validate_item_gre(items, item_flags,
5504                                                           next_protocol, error);
5505                         if (ret < 0)
5506                                 return ret;
5507                         gre_item = items;
5508                         last_item = MLX5_FLOW_LAYER_GRE;
5509                         break;
5510                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5511                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
5512                                                             next_protocol,
5513                                                             error);
5514                         if (ret < 0)
5515                                 return ret;
5516                         last_item = MLX5_FLOW_LAYER_NVGRE;
5517                         break;
5518                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5519                         ret = mlx5_flow_validate_item_gre_key
5520                                 (items, item_flags, gre_item, error);
5521                         if (ret < 0)
5522                                 return ret;
5523                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5524                         break;
5525                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5526                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
5527                                                             error);
5528                         if (ret < 0)
5529                                 return ret;
5530                         last_item = MLX5_FLOW_LAYER_VXLAN;
5531                         break;
5532                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5533                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
5534                                                                 item_flags, dev,
5535                                                                 error);
5536                         if (ret < 0)
5537                                 return ret;
5538                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5539                         break;
5540                 case RTE_FLOW_ITEM_TYPE_GENEVE:
5541                         ret = mlx5_flow_validate_item_geneve(items,
5542                                                              item_flags, dev,
5543                                                              error);
5544                         if (ret < 0)
5545                                 return ret;
5546                         last_item = MLX5_FLOW_LAYER_GENEVE;
5547                         break;
5548                 case RTE_FLOW_ITEM_TYPE_MPLS:
5549                         ret = mlx5_flow_validate_item_mpls(dev, items,
5550                                                            item_flags,
5551                                                            last_item, error);
5552                         if (ret < 0)
5553                                 return ret;
5554                         last_item = MLX5_FLOW_LAYER_MPLS;
5555                         break;
5556
5557                 case RTE_FLOW_ITEM_TYPE_MARK:
5558                         ret = flow_dv_validate_item_mark(dev, items, attr,
5559                                                          error);
5560                         if (ret < 0)
5561                                 return ret;
5562                         last_item = MLX5_FLOW_ITEM_MARK;
5563                         break;
5564                 case RTE_FLOW_ITEM_TYPE_META:
5565                         ret = flow_dv_validate_item_meta(dev, items, attr,
5566                                                          error);
5567                         if (ret < 0)
5568                                 return ret;
5569                         last_item = MLX5_FLOW_ITEM_METADATA;
5570                         break;
5571                 case RTE_FLOW_ITEM_TYPE_ICMP:
5572                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
5573                                                            next_protocol,
5574                                                            error);
5575                         if (ret < 0)
5576                                 return ret;
5577                         last_item = MLX5_FLOW_LAYER_ICMP;
5578                         break;
5579                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5580                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
5581                                                             next_protocol,
5582                                                             error);
5583                         if (ret < 0)
5584                                 return ret;
5585                         item_ipv6_proto = IPPROTO_ICMPV6;
5586                         last_item = MLX5_FLOW_LAYER_ICMP6;
5587                         break;
5588                 case RTE_FLOW_ITEM_TYPE_TAG:
5589                         ret = flow_dv_validate_item_tag(dev, items,
5590                                                         attr, error);
5591                         if (ret < 0)
5592                                 return ret;
5593                         last_item = MLX5_FLOW_ITEM_TAG;
5594                         break;
5595                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
5596                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
5597                         break;
5598                 case RTE_FLOW_ITEM_TYPE_GTP:
5599                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
5600                                                         error);
5601                         if (ret < 0)
5602                                 return ret;
5603                         last_item = MLX5_FLOW_LAYER_GTP;
5604                         break;
5605                 case RTE_FLOW_ITEM_TYPE_ECPRI:
5606                         /* Capacity will be checked in the translate stage. */
5607                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
5608                                                             last_item,
5609                                                             ether_type,
5610                                                             &nic_ecpri_mask,
5611                                                             error);
5612                         if (ret < 0)
5613                                 return ret;
5614                         last_item = MLX5_FLOW_LAYER_ECPRI;
5615                         break;
5616                 default:
5617                         return rte_flow_error_set(error, ENOTSUP,
5618                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5619                                                   NULL, "item not supported");
5620                 }
5621                 item_flags |= last_item;
5622         }
5623         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5624                 int type = actions->type;
5625
5626                 if (!mlx5_flow_os_action_supported(type))
5627                         return rte_flow_error_set(error, ENOTSUP,
5628                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5629                                                   actions,
5630                                                   "action not supported");
5631                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5632                         return rte_flow_error_set(error, ENOTSUP,
5633                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5634                                                   actions, "too many actions");
5635                 switch (type) {
5636                 case RTE_FLOW_ACTION_TYPE_VOID:
5637                         break;
5638                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5639                         ret = flow_dv_validate_action_port_id(dev,
5640                                                               action_flags,
5641                                                               actions,
5642                                                               attr,
5643                                                               error);
5644                         if (ret)
5645                                 return ret;
5646                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5647                         ++actions_n;
5648                         break;
5649                 case RTE_FLOW_ACTION_TYPE_FLAG:
5650                         ret = flow_dv_validate_action_flag(dev, action_flags,
5651                                                            attr, error);
5652                         if (ret < 0)
5653                                 return ret;
5654                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5655                                 /* Count all modify-header actions as one. */
5656                                 if (!(action_flags &
5657                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5658                                         ++actions_n;
5659                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
5660                                                 MLX5_FLOW_ACTION_MARK_EXT;
5661                         } else {
5662                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
5663                                 ++actions_n;
5664                         }
5665                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5666                         break;
5667                 case RTE_FLOW_ACTION_TYPE_MARK:
5668                         ret = flow_dv_validate_action_mark(dev, actions,
5669                                                            action_flags,
5670                                                            attr, error);
5671                         if (ret < 0)
5672                                 return ret;
5673                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5674                                 /* Count all modify-header actions as one. */
5675                                 if (!(action_flags &
5676                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5677                                         ++actions_n;
5678                                 action_flags |= MLX5_FLOW_ACTION_MARK |
5679                                                 MLX5_FLOW_ACTION_MARK_EXT;
5680                         } else {
5681                                 action_flags |= MLX5_FLOW_ACTION_MARK;
5682                                 ++actions_n;
5683                         }
5684                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5685                         break;
5686                 case RTE_FLOW_ACTION_TYPE_SET_META:
5687                         ret = flow_dv_validate_action_set_meta(dev, actions,
5688                                                                action_flags,
5689                                                                attr, error);
5690                         if (ret < 0)
5691                                 return ret;
5692                         /* Count all modify-header actions as one action. */
5693                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5694                                 ++actions_n;
5695                         action_flags |= MLX5_FLOW_ACTION_SET_META;
5696                         rw_act_num += MLX5_ACT_NUM_SET_META;
5697                         break;
5698                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5699                         ret = flow_dv_validate_action_set_tag(dev, actions,
5700                                                               action_flags,
5701                                                               attr, error);
5702                         if (ret < 0)
5703                                 return ret;
5704                         /* Count all modify-header actions as one action. */
5705                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5706                                 ++actions_n;
5707                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
5708                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5709                         break;
5710                 case RTE_FLOW_ACTION_TYPE_DROP:
5711                         ret = mlx5_flow_validate_action_drop(action_flags,
5712                                                              attr, error);
5713                         if (ret < 0)
5714                                 return ret;
5715                         action_flags |= MLX5_FLOW_ACTION_DROP;
5716                         ++actions_n;
5717                         break;
5718                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5719                         ret = mlx5_flow_validate_action_queue(actions,
5720                                                               action_flags, dev,
5721                                                               attr, error);
5722                         if (ret < 0)
5723                                 return ret;
5724                         queue_index = ((const struct rte_flow_action_queue *)
5725                                                         (actions->conf))->index;
5726                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
5727                         ++actions_n;
5728                         break;
5729                 case RTE_FLOW_ACTION_TYPE_RSS:
5730                         rss = actions->conf;
5731                         ret = mlx5_flow_validate_action_rss(actions,
5732                                                             action_flags, dev,
5733                                                             attr, item_flags,
5734                                                             error);
5735                         if (ret < 0)
5736                                 return ret;
5737                         if (rss != NULL && rss->queue_num)
5738                                 queue_index = rss->queue[0];
5739                         action_flags |= MLX5_FLOW_ACTION_RSS;
5740                         ++actions_n;
5741                         break;
5742                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
5743                         ret =
5744                         mlx5_flow_validate_action_default_miss(action_flags,
5745                                         attr, error);
5746                         if (ret < 0)
5747                                 return ret;
5748                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
5749                         ++actions_n;
5750                         break;
5751                 case RTE_FLOW_ACTION_TYPE_COUNT:
5752                         ret = flow_dv_validate_action_count(dev, error);
5753                         if (ret < 0)
5754                                 return ret;
5755                         action_flags |= MLX5_FLOW_ACTION_COUNT;
5756                         ++actions_n;
5757                         break;
5758                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5759                         if (flow_dv_validate_action_pop_vlan(dev,
5760                                                              action_flags,
5761                                                              actions,
5762                                                              item_flags, attr,
5763                                                              error))
5764                                 return -rte_errno;
5765                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5766                         ++actions_n;
5767                         break;
5768                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5769                         ret = flow_dv_validate_action_push_vlan(dev,
5770                                                                 action_flags,
5771                                                                 vlan_m,
5772                                                                 actions, attr,
5773                                                                 error);
5774                         if (ret < 0)
5775                                 return ret;
5776                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5777                         ++actions_n;
5778                         break;
5779                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5780                         ret = flow_dv_validate_action_set_vlan_pcp
5781                                                 (action_flags, actions, error);
5782                         if (ret < 0)
5783                                 return ret;
5784                         /* Count PCP with push_vlan command. */
5785                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
5786                         break;
5787                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5788                         ret = flow_dv_validate_action_set_vlan_vid
5789                                                 (item_flags, action_flags,
5790                                                  actions, error);
5791                         if (ret < 0)
5792                                 return ret;
5793                         /* Count VID with push_vlan command. */
5794                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
5795                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
5796                         break;
5797                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5798                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5799                         ret = flow_dv_validate_action_l2_encap(dev,
5800                                                                action_flags,
5801                                                                actions, attr,
5802                                                                error);
5803                         if (ret < 0)
5804                                 return ret;
5805                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
5806                         ++actions_n;
5807                         break;
5808                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5809                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5810                         ret = flow_dv_validate_action_decap(dev, action_flags,
5811                                                             attr, error);
5812                         if (ret < 0)
5813                                 return ret;
5814                         action_flags |= MLX5_FLOW_ACTION_DECAP;
5815                         ++actions_n;
5816                         break;
5817                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5818                         ret = flow_dv_validate_action_raw_encap_decap
5819                                 (dev, NULL, actions->conf, attr, &action_flags,
5820                                  &actions_n, error);
5821                         if (ret < 0)
5822                                 return ret;
5823                         break;
5824                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5825                         decap = actions->conf;
5826                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5827                                 ;
5828                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5829                                 encap = NULL;
5830                                 actions--;
5831                         } else {
5832                                 encap = actions->conf;
5833                         }
5834                         ret = flow_dv_validate_action_raw_encap_decap
5835                                            (dev,
5836                                             decap ? decap : &empty_decap, encap,
5837                                             attr, &action_flags, &actions_n,
5838                                             error);
5839                         if (ret < 0)
5840                                 return ret;
5841                         break;
5842                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5843                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5844                         ret = flow_dv_validate_action_modify_mac(action_flags,
5845                                                                  actions,
5846                                                                  item_flags,
5847                                                                  error);
5848                         if (ret < 0)
5849                                 return ret;
5850                         /* Count all modify-header actions as one action. */
5851                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5852                                 ++actions_n;
5853                         action_flags |= actions->type ==
5854                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5855                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
5856                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
5857                         /*
5858                          * Even if the source and destination MAC addresses have
5859                          * overlap in the header with 4B alignment, the convert
5860                          * function will handle them separately and 4 SW actions
5861                          * will be created. And 2 actions will be added each
5862                          * time no matter how many bytes of address will be set.
5863                          */
5864                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
5865                         break;
5866                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5867                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5868                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
5869                                                                   actions,
5870                                                                   item_flags,
5871                                                                   error);
5872                         if (ret < 0)
5873                                 return ret;
5874                         /* Count all modify-header actions as one action. */
5875                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5876                                 ++actions_n;
5877                         action_flags |= actions->type ==
5878                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5879                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
5880                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
5881                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
5882                         break;
5883                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5884                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5885                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
5886                                                                   actions,
5887                                                                   item_flags,
5888                                                                   error);
5889                         if (ret < 0)
5890                                 return ret;
5891                         if (item_ipv6_proto == IPPROTO_ICMPV6)
5892                                 return rte_flow_error_set(error, ENOTSUP,
5893                                         RTE_FLOW_ERROR_TYPE_ACTION,
5894                                         actions,
5895                                         "Can't change header "
5896                                         "with ICMPv6 proto");
5897                         /* Count all modify-header actions as one action. */
5898                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5899                                 ++actions_n;
5900                         action_flags |= actions->type ==
5901                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5902                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
5903                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
5904                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
5905                         break;
5906                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5907                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5908                         ret = flow_dv_validate_action_modify_tp(action_flags,
5909                                                                 actions,
5910                                                                 item_flags,
5911                                                                 error);
5912                         if (ret < 0)
5913                                 return ret;
5914                         /* Count all modify-header actions as one action. */
5915                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5916                                 ++actions_n;
5917                         action_flags |= actions->type ==
5918                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5919                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
5920                                                 MLX5_FLOW_ACTION_SET_TP_DST;
5921                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
5922                         break;
5923                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5924                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5925                         ret = flow_dv_validate_action_modify_ttl(action_flags,
5926                                                                  actions,
5927                                                                  item_flags,
5928                                                                  error);
5929                         if (ret < 0)
5930                                 return ret;
5931                         /* Count all modify-header actions as one action. */
5932                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5933                                 ++actions_n;
5934                         action_flags |= actions->type ==
5935                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
5936                                                 MLX5_FLOW_ACTION_SET_TTL :
5937                                                 MLX5_FLOW_ACTION_DEC_TTL;
5938                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
5939                         break;
5940                 case RTE_FLOW_ACTION_TYPE_JUMP:
5941                         ret = flow_dv_validate_action_jump(dev, tunnel, actions,
5942                                                            action_flags,
5943                                                            attr, external,
5944                                                            error);
5945                         if (ret)
5946                                 return ret;
5947                         ++actions_n;
5948                         action_flags |= MLX5_FLOW_ACTION_JUMP;
5949                         break;
5950                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5951                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5952                         ret = flow_dv_validate_action_modify_tcp_seq
5953                                                                 (action_flags,
5954                                                                  actions,
5955                                                                  item_flags,
5956                                                                  error);
5957                         if (ret < 0)
5958                                 return ret;
5959                         /* Count all modify-header actions as one action. */
5960                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5961                                 ++actions_n;
5962                         action_flags |= actions->type ==
5963                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5964                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
5965                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5966                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
5967                         break;
5968                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5969                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5970                         ret = flow_dv_validate_action_modify_tcp_ack
5971                                                                 (action_flags,
5972                                                                  actions,
5973                                                                  item_flags,
5974                                                                  error);
5975                         if (ret < 0)
5976                                 return ret;
5977                         /* Count all modify-header actions as one action. */
5978                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5979                                 ++actions_n;
5980                         action_flags |= actions->type ==
5981                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
5982                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
5983                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
5984                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
5985                         break;
5986                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
5987                         break;
5988                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
5989                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
5990                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5991                         break;
5992                 case RTE_FLOW_ACTION_TYPE_METER:
5993                         ret = mlx5_flow_validate_action_meter(dev,
5994                                                               action_flags,
5995                                                               actions, attr,
5996                                                               error);
5997                         if (ret < 0)
5998                                 return ret;
5999                         action_flags |= MLX5_FLOW_ACTION_METER;
6000                         ++actions_n;
6001                         /* Meter action will add one more TAG action. */
6002                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
6003                         break;
6004                 case RTE_FLOW_ACTION_TYPE_AGE:
6005                         ret = flow_dv_validate_action_age(action_flags,
6006                                                           actions, dev,
6007                                                           error);
6008                         if (ret < 0)
6009                                 return ret;
6010                         action_flags |= MLX5_FLOW_ACTION_AGE;
6011                         ++actions_n;
6012                         break;
6013                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
6014                         ret = flow_dv_validate_action_modify_ipv4_dscp
6015                                                          (action_flags,
6016                                                           actions,
6017                                                           item_flags,
6018                                                           error);
6019                         if (ret < 0)
6020                                 return ret;
6021                         /* Count all modify-header actions as one action. */
6022                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6023                                 ++actions_n;
6024                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
6025                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
6026                         break;
6027                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
6028                         ret = flow_dv_validate_action_modify_ipv6_dscp
6029                                                                 (action_flags,
6030                                                                  actions,
6031                                                                  item_flags,
6032                                                                  error);
6033                         if (ret < 0)
6034                                 return ret;
6035                         /* Count all modify-header actions as one action. */
6036                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6037                                 ++actions_n;
6038                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
6039                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
6040                         break;
6041                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
6042                         ret = flow_dv_validate_action_sample(action_flags,
6043                                                              actions, dev,
6044                                                              attr, error);
6045                         if (ret < 0)
6046                                 return ret;
6047                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
6048                         ++actions_n;
6049                         break;
6050                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
6051                         if (actions[0].type != (typeof(actions[0].type))
6052                                 MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET)
6053                                 return rte_flow_error_set
6054                                                 (error, EINVAL,
6055                                                 RTE_FLOW_ERROR_TYPE_ACTION,
6056                                                 NULL, "MLX5 private action "
6057                                                 "must be the first");
6058
6059                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
6060                         break;
6061                 default:
6062                         return rte_flow_error_set(error, ENOTSUP,
6063                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6064                                                   actions,
6065                                                   "action not supported");
6066                 }
6067         }
6068         /*
6069          * Validate actions in flow rules
6070          * - Explicit decap action is prohibited by the tunnel offload API.
6071          * - Drop action in tunnel steer rule is prohibited by the API.
6072          * - Application cannot use MARK action because it's value can mask
6073          *   tunnel default miss nitification.
6074          * - JUMP in tunnel match rule has no support in current PMD
6075          *   implementation.
6076          * - TAG & META are reserved for future uses.
6077          */
6078         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_SET) {
6079                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_DECAP    |
6080                                             MLX5_FLOW_ACTION_MARK     |
6081                                             MLX5_FLOW_ACTION_SET_TAG  |
6082                                             MLX5_FLOW_ACTION_SET_META |
6083                                             MLX5_FLOW_ACTION_DROP;
6084
6085                 if (action_flags & bad_actions_mask)
6086                         return rte_flow_error_set
6087                                         (error, EINVAL,
6088                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
6089                                         "Invalid RTE action in tunnel "
6090                                         "set decap rule");
6091                 if (!(action_flags & MLX5_FLOW_ACTION_JUMP))
6092                         return rte_flow_error_set
6093                                         (error, EINVAL,
6094                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
6095                                         "tunnel set decap rule must terminate "
6096                                         "with JUMP");
6097                 if (!attr->ingress)
6098                         return rte_flow_error_set
6099                                         (error, EINVAL,
6100                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
6101                                         "tunnel flows for ingress traffic only");
6102         }
6103         if (action_flags & MLX5_FLOW_ACTION_TUNNEL_MATCH) {
6104                 uint64_t bad_actions_mask = MLX5_FLOW_ACTION_JUMP    |
6105                                             MLX5_FLOW_ACTION_MARK    |
6106                                             MLX5_FLOW_ACTION_SET_TAG |
6107                                             MLX5_FLOW_ACTION_SET_META;
6108
6109                 if (action_flags & bad_actions_mask)
6110                         return rte_flow_error_set
6111                                         (error, EINVAL,
6112                                         RTE_FLOW_ERROR_TYPE_ACTION, NULL,
6113                                         "Invalid RTE action in tunnel "
6114                                         "set match rule");
6115         }
6116         /*
6117          * Validate the drop action mutual exclusion with other actions.
6118          * Drop action is mutually-exclusive with any other action, except for
6119          * Count action.
6120          */
6121         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
6122             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
6123                 return rte_flow_error_set(error, EINVAL,
6124                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
6125                                           "Drop action is mutually-exclusive "
6126                                           "with any other action, except for "
6127                                           "Count action");
6128         /* Eswitch has few restrictions on using items and actions */
6129         if (attr->transfer) {
6130                 if (!mlx5_flow_ext_mreg_supported(dev) &&
6131                     action_flags & MLX5_FLOW_ACTION_FLAG)
6132                         return rte_flow_error_set(error, ENOTSUP,
6133                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6134                                                   NULL,
6135                                                   "unsupported action FLAG");
6136                 if (!mlx5_flow_ext_mreg_supported(dev) &&
6137                     action_flags & MLX5_FLOW_ACTION_MARK)
6138                         return rte_flow_error_set(error, ENOTSUP,
6139                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6140                                                   NULL,
6141                                                   "unsupported action MARK");
6142                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
6143                         return rte_flow_error_set(error, ENOTSUP,
6144                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6145                                                   NULL,
6146                                                   "unsupported action QUEUE");
6147                 if (action_flags & MLX5_FLOW_ACTION_RSS)
6148                         return rte_flow_error_set(error, ENOTSUP,
6149                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6150                                                   NULL,
6151                                                   "unsupported action RSS");
6152                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
6153                         return rte_flow_error_set(error, EINVAL,
6154                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6155                                                   actions,
6156                                                   "no fate action is found");
6157         } else {
6158                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
6159                         return rte_flow_error_set(error, EINVAL,
6160                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6161                                                   actions,
6162                                                   "no fate action is found");
6163         }
6164         /*
6165          * Continue validation for Xcap and VLAN actions.
6166          * If hairpin is working in explicit TX rule mode, there is no actions
6167          * splitting and the validation of hairpin ingress flow should be the
6168          * same as other standard flows.
6169          */
6170         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
6171                              MLX5_FLOW_VLAN_ACTIONS)) &&
6172             (queue_index == 0xFFFF ||
6173              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN ||
6174              ((conf = mlx5_rxq_get_hairpin_conf(dev, queue_index)) != NULL &&
6175              conf->tx_explicit != 0))) {
6176                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
6177                     MLX5_FLOW_XCAP_ACTIONS)
6178                         return rte_flow_error_set(error, ENOTSUP,
6179                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6180                                                   NULL, "encap and decap "
6181                                                   "combination aren't supported");
6182                 if (!attr->transfer && attr->ingress) {
6183                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
6184                                 return rte_flow_error_set
6185                                                 (error, ENOTSUP,
6186                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6187                                                  NULL, "encap is not supported"
6188                                                  " for ingress traffic");
6189                         else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
6190                                 return rte_flow_error_set
6191                                                 (error, ENOTSUP,
6192                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6193                                                  NULL, "push VLAN action not "
6194                                                  "supported for ingress");
6195                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
6196                                         MLX5_FLOW_VLAN_ACTIONS)
6197                                 return rte_flow_error_set
6198                                                 (error, ENOTSUP,
6199                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6200                                                  NULL, "no support for "
6201                                                  "multiple VLAN actions");
6202                 }
6203         }
6204         /*
6205          * Hairpin flow will add one more TAG action in TX implicit mode.
6206          * In TX explicit mode, there will be no hairpin flow ID.
6207          */
6208         if (hairpin > 0)
6209                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6210         /* extra metadata enabled: one more TAG action will be add. */
6211         if (dev_conf->dv_flow_en &&
6212             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
6213             mlx5_flow_ext_mreg_supported(dev))
6214                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6215         if ((uint32_t)rw_act_num >
6216                         flow_dv_modify_hdr_action_max(dev, is_root)) {
6217                 return rte_flow_error_set(error, ENOTSUP,
6218                                           RTE_FLOW_ERROR_TYPE_ACTION,
6219                                           NULL, "too many header modify"
6220                                           " actions to support");
6221         }
6222         return 0;
6223 }
6224
6225 /**
6226  * Internal preparation function. Allocates the DV flow size,
6227  * this size is constant.
6228  *
6229  * @param[in] dev
6230  *   Pointer to the rte_eth_dev structure.
6231  * @param[in] attr
6232  *   Pointer to the flow attributes.
6233  * @param[in] items
6234  *   Pointer to the list of items.
6235  * @param[in] actions
6236  *   Pointer to the list of actions.
6237  * @param[out] error
6238  *   Pointer to the error structure.
6239  *
6240  * @return
6241  *   Pointer to mlx5_flow object on success,
6242  *   otherwise NULL and rte_errno is set.
6243  */
6244 static struct mlx5_flow *
6245 flow_dv_prepare(struct rte_eth_dev *dev,
6246                 const struct rte_flow_attr *attr __rte_unused,
6247                 const struct rte_flow_item items[] __rte_unused,
6248                 const struct rte_flow_action actions[] __rte_unused,
6249                 struct rte_flow_error *error)
6250 {
6251         uint32_t handle_idx = 0;
6252         struct mlx5_flow *dev_flow;
6253         struct mlx5_flow_handle *dev_handle;
6254         struct mlx5_priv *priv = dev->data->dev_private;
6255
6256         /* In case of corrupting the memory. */
6257         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
6258                 rte_flow_error_set(error, ENOSPC,
6259                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6260                                    "not free temporary device flow");
6261                 return NULL;
6262         }
6263         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
6264                                    &handle_idx);
6265         if (!dev_handle) {
6266                 rte_flow_error_set(error, ENOMEM,
6267                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6268                                    "not enough memory to create flow handle");
6269                 return NULL;
6270         }
6271         /* No multi-thread supporting. */
6272         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
6273         dev_flow->handle = dev_handle;
6274         dev_flow->handle_idx = handle_idx;
6275         /*
6276          * In some old rdma-core releases, before continuing, a check of the
6277          * length of matching parameter will be done at first. It needs to use
6278          * the length without misc4 param. If the flow has misc4 support, then
6279          * the length needs to be adjusted accordingly. Each param member is
6280          * aligned with a 64B boundary naturally.
6281          */
6282         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) -
6283                                   MLX5_ST_SZ_BYTES(fte_match_set_misc4);
6284         /*
6285          * The matching value needs to be cleared to 0 before using. In the
6286          * past, it will be automatically cleared when using rte_*alloc
6287          * API. The time consumption will be almost the same as before.
6288          */
6289         memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
6290         dev_flow->ingress = attr->ingress;
6291         dev_flow->dv.transfer = attr->transfer;
6292         return dev_flow;
6293 }
6294
6295 #ifdef RTE_LIBRTE_MLX5_DEBUG
6296 /**
6297  * Sanity check for match mask and value. Similar to check_valid_spec() in
6298  * kernel driver. If unmasked bit is present in value, it returns failure.
6299  *
6300  * @param match_mask
6301  *   pointer to match mask buffer.
6302  * @param match_value
6303  *   pointer to match value buffer.
6304  *
6305  * @return
6306  *   0 if valid, -EINVAL otherwise.
6307  */
6308 static int
6309 flow_dv_check_valid_spec(void *match_mask, void *match_value)
6310 {
6311         uint8_t *m = match_mask;
6312         uint8_t *v = match_value;
6313         unsigned int i;
6314
6315         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
6316                 if (v[i] & ~m[i]) {
6317                         DRV_LOG(ERR,
6318                                 "match_value differs from match_criteria"
6319                                 " %p[%u] != %p[%u]",
6320                                 match_value, i, match_mask, i);
6321                         return -EINVAL;
6322                 }
6323         }
6324         return 0;
6325 }
6326 #endif
6327
6328 /**
6329  * Add match of ip_version.
6330  *
6331  * @param[in] group
6332  *   Flow group.
6333  * @param[in] headers_v
6334  *   Values header pointer.
6335  * @param[in] headers_m
6336  *   Masks header pointer.
6337  * @param[in] ip_version
6338  *   The IP version to set.
6339  */
6340 static inline void
6341 flow_dv_set_match_ip_version(uint32_t group,
6342                              void *headers_v,
6343                              void *headers_m,
6344                              uint8_t ip_version)
6345 {
6346         if (group == 0)
6347                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
6348         else
6349                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
6350                          ip_version);
6351         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
6352         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
6353         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
6354 }
6355
6356 /**
6357  * Add Ethernet item to matcher and to the value.
6358  *
6359  * @param[in, out] matcher
6360  *   Flow matcher.
6361  * @param[in, out] key
6362  *   Flow matcher value.
6363  * @param[in] item
6364  *   Flow pattern to translate.
6365  * @param[in] inner
6366  *   Item is inner pattern.
6367  */
6368 static void
6369 flow_dv_translate_item_eth(void *matcher, void *key,
6370                            const struct rte_flow_item *item, int inner,
6371                            uint32_t group)
6372 {
6373         const struct rte_flow_item_eth *eth_m = item->mask;
6374         const struct rte_flow_item_eth *eth_v = item->spec;
6375         const struct rte_flow_item_eth nic_mask = {
6376                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6377                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6378                 .type = RTE_BE16(0xffff),
6379                 .has_vlan = 0,
6380         };
6381         void *hdrs_m;
6382         void *hdrs_v;
6383         char *l24_v;
6384         unsigned int i;
6385
6386         if (!eth_v)
6387                 return;
6388         if (!eth_m)
6389                 eth_m = &nic_mask;
6390         if (inner) {
6391                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
6392                                          inner_headers);
6393                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6394         } else {
6395                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
6396                                          outer_headers);
6397                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6398         }
6399         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, dmac_47_16),
6400                &eth_m->dst, sizeof(eth_m->dst));
6401         /* The value must be in the range of the mask. */
6402         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, dmac_47_16);
6403         for (i = 0; i < sizeof(eth_m->dst); ++i)
6404                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
6405         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_m, smac_47_16),
6406                &eth_m->src, sizeof(eth_m->src));
6407         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, smac_47_16);
6408         /* The value must be in the range of the mask. */
6409         for (i = 0; i < sizeof(eth_m->dst); ++i)
6410                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
6411         /*
6412          * HW supports match on one Ethertype, the Ethertype following the last
6413          * VLAN tag of the packet (see PRM).
6414          * Set match on ethertype only if ETH header is not followed by VLAN.
6415          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6416          * ethertype, and use ip_version field instead.
6417          * eCPRI over Ether layer will use type value 0xAEFE.
6418          */
6419         if (eth_m->type == 0xFFFF) {
6420                 /* Set cvlan_tag mask for any single\multi\un-tagged case. */
6421                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
6422                 switch (eth_v->type) {
6423                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
6424                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
6425                         return;
6426                 case RTE_BE16(RTE_ETHER_TYPE_QINQ):
6427                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
6428                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
6429                         return;
6430                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
6431                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
6432                         return;
6433                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
6434                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
6435                         return;
6436                 default:
6437                         break;
6438                 }
6439         }
6440         if (eth_m->has_vlan) {
6441                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
6442                 if (eth_v->has_vlan) {
6443                         /*
6444                          * Here, when also has_more_vlan field in VLAN item is
6445                          * not set, only single-tagged packets will be matched.
6446                          */
6447                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
6448                         return;
6449                 }
6450         }
6451         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
6452                  rte_be_to_cpu_16(eth_m->type));
6453         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, hdrs_v, ethertype);
6454         *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
6455 }
6456
6457 /**
6458  * Add VLAN item to matcher and to the value.
6459  *
6460  * @param[in, out] dev_flow
6461  *   Flow descriptor.
6462  * @param[in, out] matcher
6463  *   Flow matcher.
6464  * @param[in, out] key
6465  *   Flow matcher value.
6466  * @param[in] item
6467  *   Flow pattern to translate.
6468  * @param[in] inner
6469  *   Item is inner pattern.
6470  */
6471 static void
6472 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
6473                             void *matcher, void *key,
6474                             const struct rte_flow_item *item,
6475                             int inner, uint32_t group)
6476 {
6477         const struct rte_flow_item_vlan *vlan_m = item->mask;
6478         const struct rte_flow_item_vlan *vlan_v = item->spec;
6479         void *hdrs_m;
6480         void *hdrs_v;
6481         uint16_t tci_m;
6482         uint16_t tci_v;
6483
6484         if (inner) {
6485                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
6486                                          inner_headers);
6487                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6488         } else {
6489                 hdrs_m = MLX5_ADDR_OF(fte_match_param, matcher,
6490                                          outer_headers);
6491                 hdrs_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6492                 /*
6493                  * This is workaround, masks are not supported,
6494                  * and pre-validated.
6495                  */
6496                 if (vlan_v)
6497                         dev_flow->handle->vf_vlan.tag =
6498                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
6499         }
6500         /*
6501          * When VLAN item exists in flow, mark packet as tagged,
6502          * even if TCI is not specified.
6503          */
6504         if (!MLX5_GET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag)) {
6505                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, cvlan_tag, 1);
6506                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 1);
6507         }
6508         if (!vlan_v)
6509                 return;
6510         if (!vlan_m)
6511                 vlan_m = &rte_flow_item_vlan_mask;
6512         tci_m = rte_be_to_cpu_16(vlan_m->tci);
6513         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
6514         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_vid, tci_m);
6515         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_vid, tci_v);
6516         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_cfi, tci_m >> 12);
6517         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_cfi, tci_v >> 12);
6518         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, first_prio, tci_m >> 13);
6519         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, first_prio, tci_v >> 13);
6520         /*
6521          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6522          * ethertype, and use ip_version field instead.
6523          */
6524         if (vlan_m->inner_type == 0xFFFF) {
6525                 switch (vlan_v->inner_type) {
6526                 case RTE_BE16(RTE_ETHER_TYPE_VLAN):
6527                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
6528                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
6529                         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
6530                         return;
6531                 case RTE_BE16(RTE_ETHER_TYPE_IPV4):
6532                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 4);
6533                         return;
6534                 case RTE_BE16(RTE_ETHER_TYPE_IPV6):
6535                         flow_dv_set_match_ip_version(group, hdrs_v, hdrs_m, 6);
6536                         return;
6537                 default:
6538                         break;
6539                 }
6540         }
6541         if (vlan_m->has_more_vlan && vlan_v->has_more_vlan) {
6542                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, svlan_tag, 1);
6543                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, svlan_tag, 1);
6544                 /* Only one vlan_tag bit can be set. */
6545                 MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, cvlan_tag, 0);
6546                 return;
6547         }
6548         MLX5_SET(fte_match_set_lyr_2_4, hdrs_m, ethertype,
6549                  rte_be_to_cpu_16(vlan_m->inner_type));
6550         MLX5_SET(fte_match_set_lyr_2_4, hdrs_v, ethertype,
6551                  rte_be_to_cpu_16(vlan_m->inner_type & vlan_v->inner_type));
6552 }
6553
6554 /**
6555  * Add IPV4 item to matcher and to the value.
6556  *
6557  * @param[in, out] matcher
6558  *   Flow matcher.
6559  * @param[in, out] key
6560  *   Flow matcher value.
6561  * @param[in] item
6562  *   Flow pattern to translate.
6563  * @param[in] inner
6564  *   Item is inner pattern.
6565  * @param[in] group
6566  *   The group to insert the rule.
6567  */
6568 static void
6569 flow_dv_translate_item_ipv4(void *matcher, void *key,
6570                             const struct rte_flow_item *item,
6571                             int inner, uint32_t group)
6572 {
6573         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
6574         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
6575         const struct rte_flow_item_ipv4 nic_mask = {
6576                 .hdr = {
6577                         .src_addr = RTE_BE32(0xffffffff),
6578                         .dst_addr = RTE_BE32(0xffffffff),
6579                         .type_of_service = 0xff,
6580                         .next_proto_id = 0xff,
6581                         .time_to_live = 0xff,
6582                 },
6583         };
6584         void *headers_m;
6585         void *headers_v;
6586         char *l24_m;
6587         char *l24_v;
6588         uint8_t tos;
6589
6590         if (inner) {
6591                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6592                                          inner_headers);
6593                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6594         } else {
6595                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6596                                          outer_headers);
6597                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6598         }
6599         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6600         if (!ipv4_v)
6601                 return;
6602         if (!ipv4_m)
6603                 ipv4_m = &nic_mask;
6604         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6605                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6606         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6607                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6608         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
6609         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
6610         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6611                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6612         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6613                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6614         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
6615         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
6616         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
6617         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
6618                  ipv4_m->hdr.type_of_service);
6619         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
6620         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
6621                  ipv4_m->hdr.type_of_service >> 2);
6622         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
6623         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6624                  ipv4_m->hdr.next_proto_id);
6625         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6626                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
6627         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6628                  ipv4_m->hdr.time_to_live);
6629         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6630                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
6631         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
6632                  !!(ipv4_m->hdr.fragment_offset));
6633         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
6634                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
6635 }
6636
6637 /**
6638  * Add IPV6 item to matcher and to the value.
6639  *
6640  * @param[in, out] matcher
6641  *   Flow matcher.
6642  * @param[in, out] key
6643  *   Flow matcher value.
6644  * @param[in] item
6645  *   Flow pattern to translate.
6646  * @param[in] inner
6647  *   Item is inner pattern.
6648  * @param[in] group
6649  *   The group to insert the rule.
6650  */
6651 static void
6652 flow_dv_translate_item_ipv6(void *matcher, void *key,
6653                             const struct rte_flow_item *item,
6654                             int inner, uint32_t group)
6655 {
6656         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
6657         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
6658         const struct rte_flow_item_ipv6 nic_mask = {
6659                 .hdr = {
6660                         .src_addr =
6661                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6662                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6663                         .dst_addr =
6664                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6665                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6666                         .vtc_flow = RTE_BE32(0xffffffff),
6667                         .proto = 0xff,
6668                         .hop_limits = 0xff,
6669                 },
6670         };
6671         void *headers_m;
6672         void *headers_v;
6673         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6674         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6675         char *l24_m;
6676         char *l24_v;
6677         uint32_t vtc_m;
6678         uint32_t vtc_v;
6679         int i;
6680         int size;
6681
6682         if (inner) {
6683                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6684                                          inner_headers);
6685                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6686         } else {
6687                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6688                                          outer_headers);
6689                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6690         }
6691         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6692         if (!ipv6_v)
6693                 return;
6694         if (!ipv6_m)
6695                 ipv6_m = &nic_mask;
6696         size = sizeof(ipv6_m->hdr.dst_addr);
6697         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6698                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6699         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6700                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6701         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
6702         for (i = 0; i < size; ++i)
6703                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
6704         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6705                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6706         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6707                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6708         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
6709         for (i = 0; i < size; ++i)
6710                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
6711         /* TOS. */
6712         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
6713         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
6714         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
6715         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
6716         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
6717         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
6718         /* Label. */
6719         if (inner) {
6720                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
6721                          vtc_m);
6722                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
6723                          vtc_v);
6724         } else {
6725                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
6726                          vtc_m);
6727                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
6728                          vtc_v);
6729         }
6730         /* Protocol. */
6731         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6732                  ipv6_m->hdr.proto);
6733         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6734                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
6735         /* Hop limit. */
6736         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6737                  ipv6_m->hdr.hop_limits);
6738         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6739                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
6740         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
6741                  !!(ipv6_m->has_frag_ext));
6742         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
6743                  !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext));
6744 }
6745
6746 /**
6747  * Add IPV6 fragment extension item to matcher and to the value.
6748  *
6749  * @param[in, out] matcher
6750  *   Flow matcher.
6751  * @param[in, out] key
6752  *   Flow matcher value.
6753  * @param[in] item
6754  *   Flow pattern to translate.
6755  * @param[in] inner
6756  *   Item is inner pattern.
6757  */
6758 static void
6759 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key,
6760                                      const struct rte_flow_item *item,
6761                                      int inner)
6762 {
6763         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask;
6764         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec;
6765         const struct rte_flow_item_ipv6_frag_ext nic_mask = {
6766                 .hdr = {
6767                         .next_header = 0xff,
6768                         .frag_data = RTE_BE16(0xffff),
6769                 },
6770         };
6771         void *headers_m;
6772         void *headers_v;
6773
6774         if (inner) {
6775                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6776                                          inner_headers);
6777                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6778         } else {
6779                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6780                                          outer_headers);
6781                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6782         }
6783         /* IPv6 fragment extension item exists, so packet is IP fragment. */
6784         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
6785         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1);
6786         if (!ipv6_frag_ext_v)
6787                 return;
6788         if (!ipv6_frag_ext_m)
6789                 ipv6_frag_ext_m = &nic_mask;
6790         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6791                  ipv6_frag_ext_m->hdr.next_header);
6792         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6793                  ipv6_frag_ext_v->hdr.next_header &
6794                  ipv6_frag_ext_m->hdr.next_header);
6795 }
6796
6797 /**
6798  * Add TCP item to matcher and to the value.
6799  *
6800  * @param[in, out] matcher
6801  *   Flow matcher.
6802  * @param[in, out] key
6803  *   Flow matcher value.
6804  * @param[in] item
6805  *   Flow pattern to translate.
6806  * @param[in] inner
6807  *   Item is inner pattern.
6808  */
6809 static void
6810 flow_dv_translate_item_tcp(void *matcher, void *key,
6811                            const struct rte_flow_item *item,
6812                            int inner)
6813 {
6814         const struct rte_flow_item_tcp *tcp_m = item->mask;
6815         const struct rte_flow_item_tcp *tcp_v = item->spec;
6816         void *headers_m;
6817         void *headers_v;
6818
6819         if (inner) {
6820                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6821                                          inner_headers);
6822                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6823         } else {
6824                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6825                                          outer_headers);
6826                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6827         }
6828         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6829         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
6830         if (!tcp_v)
6831                 return;
6832         if (!tcp_m)
6833                 tcp_m = &rte_flow_item_tcp_mask;
6834         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
6835                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
6836         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
6837                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
6838         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
6839                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
6840         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
6841                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
6842         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
6843                  tcp_m->hdr.tcp_flags);
6844         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
6845                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
6846 }
6847
6848 /**
6849  * Add UDP item to matcher and to the value.
6850  *
6851  * @param[in, out] matcher
6852  *   Flow matcher.
6853  * @param[in, out] key
6854  *   Flow matcher value.
6855  * @param[in] item
6856  *   Flow pattern to translate.
6857  * @param[in] inner
6858  *   Item is inner pattern.
6859  */
6860 static void
6861 flow_dv_translate_item_udp(void *matcher, void *key,
6862                            const struct rte_flow_item *item,
6863                            int inner)
6864 {
6865         const struct rte_flow_item_udp *udp_m = item->mask;
6866         const struct rte_flow_item_udp *udp_v = item->spec;
6867         void *headers_m;
6868         void *headers_v;
6869
6870         if (inner) {
6871                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6872                                          inner_headers);
6873                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6874         } else {
6875                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6876                                          outer_headers);
6877                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6878         }
6879         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6880         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
6881         if (!udp_v)
6882                 return;
6883         if (!udp_m)
6884                 udp_m = &rte_flow_item_udp_mask;
6885         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
6886                  rte_be_to_cpu_16(udp_m->hdr.src_port));
6887         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
6888                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
6889         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
6890                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
6891         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6892                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
6893 }
6894
6895 /**
6896  * Add GRE optional Key item to matcher and to the value.
6897  *
6898  * @param[in, out] matcher
6899  *   Flow matcher.
6900  * @param[in, out] key
6901  *   Flow matcher value.
6902  * @param[in] item
6903  *   Flow pattern to translate.
6904  * @param[in] inner
6905  *   Item is inner pattern.
6906  */
6907 static void
6908 flow_dv_translate_item_gre_key(void *matcher, void *key,
6909                                    const struct rte_flow_item *item)
6910 {
6911         const rte_be32_t *key_m = item->mask;
6912         const rte_be32_t *key_v = item->spec;
6913         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6914         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6915         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
6916
6917         /* GRE K bit must be on and should already be validated */
6918         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
6919         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
6920         if (!key_v)
6921                 return;
6922         if (!key_m)
6923                 key_m = &gre_key_default_mask;
6924         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
6925                  rte_be_to_cpu_32(*key_m) >> 8);
6926         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
6927                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
6928         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
6929                  rte_be_to_cpu_32(*key_m) & 0xFF);
6930         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
6931                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
6932 }
6933
6934 /**
6935  * Add GRE item to matcher and to the value.
6936  *
6937  * @param[in, out] matcher
6938  *   Flow matcher.
6939  * @param[in, out] key
6940  *   Flow matcher value.
6941  * @param[in] item
6942  *   Flow pattern to translate.
6943  * @param[in] inner
6944  *   Item is inner pattern.
6945  */
6946 static void
6947 flow_dv_translate_item_gre(void *matcher, void *key,
6948                            const struct rte_flow_item *item,
6949                            int inner)
6950 {
6951         const struct rte_flow_item_gre *gre_m = item->mask;
6952         const struct rte_flow_item_gre *gre_v = item->spec;
6953         void *headers_m;
6954         void *headers_v;
6955         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6956         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6957         struct {
6958                 union {
6959                         __extension__
6960                         struct {
6961                                 uint16_t version:3;
6962                                 uint16_t rsvd0:9;
6963                                 uint16_t s_present:1;
6964                                 uint16_t k_present:1;
6965                                 uint16_t rsvd_bit1:1;
6966                                 uint16_t c_present:1;
6967                         };
6968                         uint16_t value;
6969                 };
6970         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
6971
6972         if (inner) {
6973                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6974                                          inner_headers);
6975                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6976         } else {
6977                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6978                                          outer_headers);
6979                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6980         }
6981         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6982         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
6983         if (!gre_v)
6984                 return;
6985         if (!gre_m)
6986                 gre_m = &rte_flow_item_gre_mask;
6987         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
6988                  rte_be_to_cpu_16(gre_m->protocol));
6989         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6990                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
6991         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
6992         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
6993         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
6994                  gre_crks_rsvd0_ver_m.c_present);
6995         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
6996                  gre_crks_rsvd0_ver_v.c_present &
6997                  gre_crks_rsvd0_ver_m.c_present);
6998         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
6999                  gre_crks_rsvd0_ver_m.k_present);
7000         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
7001                  gre_crks_rsvd0_ver_v.k_present &
7002                  gre_crks_rsvd0_ver_m.k_present);
7003         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
7004                  gre_crks_rsvd0_ver_m.s_present);
7005         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
7006                  gre_crks_rsvd0_ver_v.s_present &
7007                  gre_crks_rsvd0_ver_m.s_present);
7008 }
7009
7010 /**
7011  * Add NVGRE item to matcher and to the value.
7012  *
7013  * @param[in, out] matcher
7014  *   Flow matcher.
7015  * @param[in, out] key
7016  *   Flow matcher value.
7017  * @param[in] item
7018  *   Flow pattern to translate.
7019  * @param[in] inner
7020  *   Item is inner pattern.
7021  */
7022 static void
7023 flow_dv_translate_item_nvgre(void *matcher, void *key,
7024                              const struct rte_flow_item *item,
7025                              int inner)
7026 {
7027         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
7028         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
7029         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7030         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7031         const char *tni_flow_id_m;
7032         const char *tni_flow_id_v;
7033         char *gre_key_m;
7034         char *gre_key_v;
7035         int size;
7036         int i;
7037
7038         /* For NVGRE, GRE header fields must be set with defined values. */
7039         const struct rte_flow_item_gre gre_spec = {
7040                 .c_rsvd0_ver = RTE_BE16(0x2000),
7041                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
7042         };
7043         const struct rte_flow_item_gre gre_mask = {
7044                 .c_rsvd0_ver = RTE_BE16(0xB000),
7045                 .protocol = RTE_BE16(UINT16_MAX),
7046         };
7047         const struct rte_flow_item gre_item = {
7048                 .spec = &gre_spec,
7049                 .mask = &gre_mask,
7050                 .last = NULL,
7051         };
7052         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
7053         if (!nvgre_v)
7054                 return;
7055         if (!nvgre_m)
7056                 nvgre_m = &rte_flow_item_nvgre_mask;
7057         tni_flow_id_m = (const char *)nvgre_m->tni;
7058         tni_flow_id_v = (const char *)nvgre_v->tni;
7059         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
7060         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
7061         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
7062         memcpy(gre_key_m, tni_flow_id_m, size);
7063         for (i = 0; i < size; ++i)
7064                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
7065 }
7066
7067 /**
7068  * Add VXLAN item to matcher and to the value.
7069  *
7070  * @param[in, out] matcher
7071  *   Flow matcher.
7072  * @param[in, out] key
7073  *   Flow matcher value.
7074  * @param[in] item
7075  *   Flow pattern to translate.
7076  * @param[in] inner
7077  *   Item is inner pattern.
7078  */
7079 static void
7080 flow_dv_translate_item_vxlan(void *matcher, void *key,
7081                              const struct rte_flow_item *item,
7082                              int inner)
7083 {
7084         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
7085         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
7086         void *headers_m;
7087         void *headers_v;
7088         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7089         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7090         char *vni_m;
7091         char *vni_v;
7092         uint16_t dport;
7093         int size;
7094         int i;
7095
7096         if (inner) {
7097                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7098                                          inner_headers);
7099                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7100         } else {
7101                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7102                                          outer_headers);
7103                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7104         }
7105         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
7106                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
7107         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7108                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7109                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7110         }
7111         if (!vxlan_v)
7112                 return;
7113         if (!vxlan_m)
7114                 vxlan_m = &rte_flow_item_vxlan_mask;
7115         size = sizeof(vxlan_m->vni);
7116         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
7117         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
7118         memcpy(vni_m, vxlan_m->vni, size);
7119         for (i = 0; i < size; ++i)
7120                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
7121 }
7122
7123 /**
7124  * Add VXLAN-GPE item to matcher and to the value.
7125  *
7126  * @param[in, out] matcher
7127  *   Flow matcher.
7128  * @param[in, out] key
7129  *   Flow matcher value.
7130  * @param[in] item
7131  *   Flow pattern to translate.
7132  * @param[in] inner
7133  *   Item is inner pattern.
7134  */
7135
7136 static void
7137 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
7138                                  const struct rte_flow_item *item, int inner)
7139 {
7140         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
7141         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
7142         void *headers_m;
7143         void *headers_v;
7144         void *misc_m =
7145                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
7146         void *misc_v =
7147                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7148         char *vni_m;
7149         char *vni_v;
7150         uint16_t dport;
7151         int size;
7152         int i;
7153         uint8_t flags_m = 0xff;
7154         uint8_t flags_v = 0xc;
7155
7156         if (inner) {
7157                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7158                                          inner_headers);
7159                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7160         } else {
7161                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7162                                          outer_headers);
7163                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7164         }
7165         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
7166                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
7167         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7168                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7169                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7170         }
7171         if (!vxlan_v)
7172                 return;
7173         if (!vxlan_m)
7174                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
7175         size = sizeof(vxlan_m->vni);
7176         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
7177         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
7178         memcpy(vni_m, vxlan_m->vni, size);
7179         for (i = 0; i < size; ++i)
7180                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
7181         if (vxlan_m->flags) {
7182                 flags_m = vxlan_m->flags;
7183                 flags_v = vxlan_v->flags;
7184         }
7185         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
7186         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
7187         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
7188                  vxlan_m->protocol);
7189         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
7190                  vxlan_v->protocol);
7191 }
7192
7193 /**
7194  * Add Geneve item to matcher and to the value.
7195  *
7196  * @param[in, out] matcher
7197  *   Flow matcher.
7198  * @param[in, out] key
7199  *   Flow matcher value.
7200  * @param[in] item
7201  *   Flow pattern to translate.
7202  * @param[in] inner
7203  *   Item is inner pattern.
7204  */
7205
7206 static void
7207 flow_dv_translate_item_geneve(void *matcher, void *key,
7208                               const struct rte_flow_item *item, int inner)
7209 {
7210         const struct rte_flow_item_geneve *geneve_m = item->mask;
7211         const struct rte_flow_item_geneve *geneve_v = item->spec;
7212         void *headers_m;
7213         void *headers_v;
7214         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7215         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7216         uint16_t dport;
7217         uint16_t gbhdr_m;
7218         uint16_t gbhdr_v;
7219         char *vni_m;
7220         char *vni_v;
7221         size_t size, i;
7222
7223         if (inner) {
7224                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7225                                          inner_headers);
7226                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7227         } else {
7228                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7229                                          outer_headers);
7230                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7231         }
7232         dport = MLX5_UDP_PORT_GENEVE;
7233         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7234                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7235                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7236         }
7237         if (!geneve_v)
7238                 return;
7239         if (!geneve_m)
7240                 geneve_m = &rte_flow_item_geneve_mask;
7241         size = sizeof(geneve_m->vni);
7242         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
7243         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
7244         memcpy(vni_m, geneve_m->vni, size);
7245         for (i = 0; i < size; ++i)
7246                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
7247         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
7248                  rte_be_to_cpu_16(geneve_m->protocol));
7249         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
7250                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
7251         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
7252         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
7253         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
7254                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7255         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
7256                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7257         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
7258                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7259         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
7260                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
7261                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7262 }
7263
7264 /**
7265  * Add MPLS item to matcher and to the value.
7266  *
7267  * @param[in, out] matcher
7268  *   Flow matcher.
7269  * @param[in, out] key
7270  *   Flow matcher value.
7271  * @param[in] item
7272  *   Flow pattern to translate.
7273  * @param[in] prev_layer
7274  *   The protocol layer indicated in previous item.
7275  * @param[in] inner
7276  *   Item is inner pattern.
7277  */
7278 static void
7279 flow_dv_translate_item_mpls(void *matcher, void *key,
7280                             const struct rte_flow_item *item,
7281                             uint64_t prev_layer,
7282                             int inner)
7283 {
7284         const uint32_t *in_mpls_m = item->mask;
7285         const uint32_t *in_mpls_v = item->spec;
7286         uint32_t *out_mpls_m = 0;
7287         uint32_t *out_mpls_v = 0;
7288         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7289         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7290         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
7291                                      misc_parameters_2);
7292         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7293         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
7294         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7295
7296         switch (prev_layer) {
7297         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7298                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
7299                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
7300                          MLX5_UDP_PORT_MPLS);
7301                 break;
7302         case MLX5_FLOW_LAYER_GRE:
7303                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
7304                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
7305                          RTE_ETHER_TYPE_MPLS);
7306                 break;
7307         default:
7308                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
7309                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
7310                          IPPROTO_MPLS);
7311                 break;
7312         }
7313         if (!in_mpls_v)
7314                 return;
7315         if (!in_mpls_m)
7316                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
7317         switch (prev_layer) {
7318         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7319                 out_mpls_m =
7320                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7321                                                  outer_first_mpls_over_udp);
7322                 out_mpls_v =
7323                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7324                                                  outer_first_mpls_over_udp);
7325                 break;
7326         case MLX5_FLOW_LAYER_GRE:
7327                 out_mpls_m =
7328                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7329                                                  outer_first_mpls_over_gre);
7330                 out_mpls_v =
7331                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7332                                                  outer_first_mpls_over_gre);
7333                 break;
7334         default:
7335                 /* Inner MPLS not over GRE is not supported. */
7336                 if (!inner) {
7337                         out_mpls_m =
7338                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7339                                                          misc2_m,
7340                                                          outer_first_mpls);
7341                         out_mpls_v =
7342                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7343                                                          misc2_v,
7344                                                          outer_first_mpls);
7345                 }
7346                 break;
7347         }
7348         if (out_mpls_m && out_mpls_v) {
7349                 *out_mpls_m = *in_mpls_m;
7350                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
7351         }
7352 }
7353
7354 /**
7355  * Add metadata register item to matcher
7356  *
7357  * @param[in, out] matcher
7358  *   Flow matcher.
7359  * @param[in, out] key
7360  *   Flow matcher value.
7361  * @param[in] reg_type
7362  *   Type of device metadata register
7363  * @param[in] value
7364  *   Register value
7365  * @param[in] mask
7366  *   Register mask
7367  */
7368 static void
7369 flow_dv_match_meta_reg(void *matcher, void *key,
7370                        enum modify_reg reg_type,
7371                        uint32_t data, uint32_t mask)
7372 {
7373         void *misc2_m =
7374                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
7375         void *misc2_v =
7376                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7377         uint32_t temp;
7378
7379         data &= mask;
7380         switch (reg_type) {
7381         case REG_A:
7382                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
7383                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
7384                 break;
7385         case REG_B:
7386                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
7387                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
7388                 break;
7389         case REG_C_0:
7390                 /*
7391                  * The metadata register C0 field might be divided into
7392                  * source vport index and META item value, we should set
7393                  * this field according to specified mask, not as whole one.
7394                  */
7395                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
7396                 temp |= mask;
7397                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
7398                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
7399                 temp &= ~mask;
7400                 temp |= data;
7401                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
7402                 break;
7403         case REG_C_1:
7404                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
7405                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
7406                 break;
7407         case REG_C_2:
7408                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
7409                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
7410                 break;
7411         case REG_C_3:
7412                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
7413                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
7414                 break;
7415         case REG_C_4:
7416                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
7417                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
7418                 break;
7419         case REG_C_5:
7420                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
7421                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
7422                 break;
7423         case REG_C_6:
7424                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
7425                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
7426                 break;
7427         case REG_C_7:
7428                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
7429                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
7430                 break;
7431         default:
7432                 MLX5_ASSERT(false);
7433                 break;
7434         }
7435 }
7436
7437 /**
7438  * Add MARK item to matcher
7439  *
7440  * @param[in] dev
7441  *   The device to configure through.
7442  * @param[in, out] matcher
7443  *   Flow matcher.
7444  * @param[in, out] key
7445  *   Flow matcher value.
7446  * @param[in] item
7447  *   Flow pattern to translate.
7448  */
7449 static void
7450 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
7451                             void *matcher, void *key,
7452                             const struct rte_flow_item *item)
7453 {
7454         struct mlx5_priv *priv = dev->data->dev_private;
7455         const struct rte_flow_item_mark *mark;
7456         uint32_t value;
7457         uint32_t mask;
7458
7459         mark = item->mask ? (const void *)item->mask :
7460                             &rte_flow_item_mark_mask;
7461         mask = mark->id & priv->sh->dv_mark_mask;
7462         mark = (const void *)item->spec;
7463         MLX5_ASSERT(mark);
7464         value = mark->id & priv->sh->dv_mark_mask & mask;
7465         if (mask) {
7466                 enum modify_reg reg;
7467
7468                 /* Get the metadata register index for the mark. */
7469                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
7470                 MLX5_ASSERT(reg > 0);
7471                 if (reg == REG_C_0) {
7472                         struct mlx5_priv *priv = dev->data->dev_private;
7473                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7474                         uint32_t shl_c0 = rte_bsf32(msk_c0);
7475
7476                         mask &= msk_c0;
7477                         mask <<= shl_c0;
7478                         value <<= shl_c0;
7479                 }
7480                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7481         }
7482 }
7483
7484 /**
7485  * Add META item to matcher
7486  *
7487  * @param[in] dev
7488  *   The devich to configure through.
7489  * @param[in, out] matcher
7490  *   Flow matcher.
7491  * @param[in, out] key
7492  *   Flow matcher value.
7493  * @param[in] attr
7494  *   Attributes of flow that includes this item.
7495  * @param[in] item
7496  *   Flow pattern to translate.
7497  */
7498 static void
7499 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
7500                             void *matcher, void *key,
7501                             const struct rte_flow_attr *attr,
7502                             const struct rte_flow_item *item)
7503 {
7504         const struct rte_flow_item_meta *meta_m;
7505         const struct rte_flow_item_meta *meta_v;
7506
7507         meta_m = (const void *)item->mask;
7508         if (!meta_m)
7509                 meta_m = &rte_flow_item_meta_mask;
7510         meta_v = (const void *)item->spec;
7511         if (meta_v) {
7512                 int reg;
7513                 uint32_t value = meta_v->data;
7514                 uint32_t mask = meta_m->data;
7515
7516                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
7517                 if (reg < 0)
7518                         return;
7519                 /*
7520                  * In datapath code there is no endianness
7521                  * coversions for perfromance reasons, all
7522                  * pattern conversions are done in rte_flow.
7523                  */
7524                 value = rte_cpu_to_be_32(value);
7525                 mask = rte_cpu_to_be_32(mask);
7526                 if (reg == REG_C_0) {
7527                         struct mlx5_priv *priv = dev->data->dev_private;
7528                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7529                         uint32_t shl_c0 = rte_bsf32(msk_c0);
7530 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
7531                         uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
7532
7533                         value >>= shr_c0;
7534                         mask >>= shr_c0;
7535 #endif
7536                         value <<= shl_c0;
7537                         mask <<= shl_c0;
7538                         MLX5_ASSERT(msk_c0);
7539                         MLX5_ASSERT(!(~msk_c0 & mask));
7540                 }
7541                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7542         }
7543 }
7544
7545 /**
7546  * Add vport metadata Reg C0 item to matcher
7547  *
7548  * @param[in, out] matcher
7549  *   Flow matcher.
7550  * @param[in, out] key
7551  *   Flow matcher value.
7552  * @param[in] reg
7553  *   Flow pattern to translate.
7554  */
7555 static void
7556 flow_dv_translate_item_meta_vport(void *matcher, void *key,
7557                                   uint32_t value, uint32_t mask)
7558 {
7559         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
7560 }
7561
7562 /**
7563  * Add tag item to matcher
7564  *
7565  * @param[in] dev
7566  *   The devich to configure through.
7567  * @param[in, out] matcher
7568  *   Flow matcher.
7569  * @param[in, out] key
7570  *   Flow matcher value.
7571  * @param[in] item
7572  *   Flow pattern to translate.
7573  */
7574 static void
7575 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
7576                                 void *matcher, void *key,
7577                                 const struct rte_flow_item *item)
7578 {
7579         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
7580         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
7581         uint32_t mask, value;
7582
7583         MLX5_ASSERT(tag_v);
7584         value = tag_v->data;
7585         mask = tag_m ? tag_m->data : UINT32_MAX;
7586         if (tag_v->id == REG_C_0) {
7587                 struct mlx5_priv *priv = dev->data->dev_private;
7588                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7589                 uint32_t shl_c0 = rte_bsf32(msk_c0);
7590
7591                 mask &= msk_c0;
7592                 mask <<= shl_c0;
7593                 value <<= shl_c0;
7594         }
7595         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
7596 }
7597
7598 /**
7599  * Add TAG item to matcher
7600  *
7601  * @param[in] dev
7602  *   The devich to configure through.
7603  * @param[in, out] matcher
7604  *   Flow matcher.
7605  * @param[in, out] key
7606  *   Flow matcher value.
7607  * @param[in] item
7608  *   Flow pattern to translate.
7609  */
7610 static void
7611 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
7612                            void *matcher, void *key,
7613                            const struct rte_flow_item *item)
7614 {
7615         const struct rte_flow_item_tag *tag_v = item->spec;
7616         const struct rte_flow_item_tag *tag_m = item->mask;
7617         enum modify_reg reg;
7618
7619         MLX5_ASSERT(tag_v);
7620         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
7621         /* Get the metadata register index for the tag. */
7622         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
7623         MLX5_ASSERT(reg > 0);
7624         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
7625 }
7626
7627 /**
7628  * Add source vport match to the specified matcher.
7629  *
7630  * @param[in, out] matcher
7631  *   Flow matcher.
7632  * @param[in, out] key
7633  *   Flow matcher value.
7634  * @param[in] port
7635  *   Source vport value to match
7636  * @param[in] mask
7637  *   Mask
7638  */
7639 static void
7640 flow_dv_translate_item_source_vport(void *matcher, void *key,
7641                                     int16_t port, uint16_t mask)
7642 {
7643         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7644         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7645
7646         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
7647         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
7648 }
7649
7650 /**
7651  * Translate port-id item to eswitch match on  port-id.
7652  *
7653  * @param[in] dev
7654  *   The devich to configure through.
7655  * @param[in, out] matcher
7656  *   Flow matcher.
7657  * @param[in, out] key
7658  *   Flow matcher value.
7659  * @param[in] item
7660  *   Flow pattern to translate.
7661  *
7662  * @return
7663  *   0 on success, a negative errno value otherwise.
7664  */
7665 static int
7666 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
7667                                void *key, const struct rte_flow_item *item)
7668 {
7669         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
7670         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
7671         struct mlx5_priv *priv;
7672         uint16_t mask, id;
7673
7674         mask = pid_m ? pid_m->id : 0xffff;
7675         id = pid_v ? pid_v->id : dev->data->port_id;
7676         priv = mlx5_port_to_eswitch_info(id, item == NULL);
7677         if (!priv)
7678                 return -rte_errno;
7679         /* Translate to vport field or to metadata, depending on mode. */
7680         if (priv->vport_meta_mask)
7681                 flow_dv_translate_item_meta_vport(matcher, key,
7682                                                   priv->vport_meta_tag,
7683                                                   priv->vport_meta_mask);
7684         else
7685                 flow_dv_translate_item_source_vport(matcher, key,
7686                                                     priv->vport_id, mask);
7687         return 0;
7688 }
7689
7690 /**
7691  * Add ICMP6 item to matcher and to the value.
7692  *
7693  * @param[in, out] matcher
7694  *   Flow matcher.
7695  * @param[in, out] key
7696  *   Flow matcher value.
7697  * @param[in] item
7698  *   Flow pattern to translate.
7699  * @param[in] inner
7700  *   Item is inner pattern.
7701  */
7702 static void
7703 flow_dv_translate_item_icmp6(void *matcher, void *key,
7704                               const struct rte_flow_item *item,
7705                               int inner)
7706 {
7707         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
7708         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
7709         void *headers_m;
7710         void *headers_v;
7711         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7712                                      misc_parameters_3);
7713         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7714         if (inner) {
7715                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7716                                          inner_headers);
7717                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7718         } else {
7719                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7720                                          outer_headers);
7721                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7722         }
7723         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7724         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
7725         if (!icmp6_v)
7726                 return;
7727         if (!icmp6_m)
7728                 icmp6_m = &rte_flow_item_icmp6_mask;
7729         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
7730         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
7731                  icmp6_v->type & icmp6_m->type);
7732         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
7733         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
7734                  icmp6_v->code & icmp6_m->code);
7735 }
7736
7737 /**
7738  * Add ICMP item to matcher and to the value.
7739  *
7740  * @param[in, out] matcher
7741  *   Flow matcher.
7742  * @param[in, out] key
7743  *   Flow matcher value.
7744  * @param[in] item
7745  *   Flow pattern to translate.
7746  * @param[in] inner
7747  *   Item is inner pattern.
7748  */
7749 static void
7750 flow_dv_translate_item_icmp(void *matcher, void *key,
7751                             const struct rte_flow_item *item,
7752                             int inner)
7753 {
7754         const struct rte_flow_item_icmp *icmp_m = item->mask;
7755         const struct rte_flow_item_icmp *icmp_v = item->spec;
7756         uint32_t icmp_header_data_m = 0;
7757         uint32_t icmp_header_data_v = 0;
7758         void *headers_m;
7759         void *headers_v;
7760         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7761                                      misc_parameters_3);
7762         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7763         if (inner) {
7764                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7765                                          inner_headers);
7766                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7767         } else {
7768                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7769                                          outer_headers);
7770                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7771         }
7772         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7773         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
7774         if (!icmp_v)
7775                 return;
7776         if (!icmp_m)
7777                 icmp_m = &rte_flow_item_icmp_mask;
7778         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
7779                  icmp_m->hdr.icmp_type);
7780         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
7781                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
7782         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
7783                  icmp_m->hdr.icmp_code);
7784         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
7785                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
7786         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
7787         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
7788         if (icmp_header_data_m) {
7789                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
7790                 icmp_header_data_v |=
7791                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
7792                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
7793                          icmp_header_data_m);
7794                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
7795                          icmp_header_data_v & icmp_header_data_m);
7796         }
7797 }
7798
7799 /**
7800  * Add GTP item to matcher and to the value.
7801  *
7802  * @param[in, out] matcher
7803  *   Flow matcher.
7804  * @param[in, out] key
7805  *   Flow matcher value.
7806  * @param[in] item
7807  *   Flow pattern to translate.
7808  * @param[in] inner
7809  *   Item is inner pattern.
7810  */
7811 static void
7812 flow_dv_translate_item_gtp(void *matcher, void *key,
7813                            const struct rte_flow_item *item, int inner)
7814 {
7815         const struct rte_flow_item_gtp *gtp_m = item->mask;
7816         const struct rte_flow_item_gtp *gtp_v = item->spec;
7817         void *headers_m;
7818         void *headers_v;
7819         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7820                                      misc_parameters_3);
7821         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7822         uint16_t dport = RTE_GTPU_UDP_PORT;
7823
7824         if (inner) {
7825                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7826                                          inner_headers);
7827                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7828         } else {
7829                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7830                                          outer_headers);
7831                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7832         }
7833         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7834                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7835                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7836         }
7837         if (!gtp_v)
7838                 return;
7839         if (!gtp_m)
7840                 gtp_m = &rte_flow_item_gtp_mask;
7841         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
7842                  gtp_m->v_pt_rsv_flags);
7843         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
7844                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
7845         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
7846         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
7847                  gtp_v->msg_type & gtp_m->msg_type);
7848         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
7849                  rte_be_to_cpu_32(gtp_m->teid));
7850         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
7851                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
7852 }
7853
7854 /**
7855  * Add eCPRI item to matcher and to the value.
7856  *
7857  * @param[in] dev
7858  *   The devich to configure through.
7859  * @param[in, out] matcher
7860  *   Flow matcher.
7861  * @param[in, out] key
7862  *   Flow matcher value.
7863  * @param[in] item
7864  *   Flow pattern to translate.
7865  * @param[in] samples
7866  *   Sample IDs to be used in the matching.
7867  */
7868 static void
7869 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
7870                              void *key, const struct rte_flow_item *item)
7871 {
7872         struct mlx5_priv *priv = dev->data->dev_private;
7873         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
7874         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
7875         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
7876                                      misc_parameters_4);
7877         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
7878         uint32_t *samples;
7879         void *dw_m;
7880         void *dw_v;
7881
7882         if (!ecpri_v)
7883                 return;
7884         if (!ecpri_m)
7885                 ecpri_m = &rte_flow_item_ecpri_mask;
7886         /*
7887          * Maximal four DW samples are supported in a single matching now.
7888          * Two are used now for a eCPRI matching:
7889          * 1. Type: one byte, mask should be 0x00ff0000 in network order
7890          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
7891          *    if any.
7892          */
7893         if (!ecpri_m->hdr.common.u32)
7894                 return;
7895         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
7896         /* Need to take the whole DW as the mask to fill the entry. */
7897         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
7898                             prog_sample_field_value_0);
7899         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
7900                             prog_sample_field_value_0);
7901         /* Already big endian (network order) in the header. */
7902         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
7903         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32;
7904         /* Sample#0, used for matching type, offset 0. */
7905         MLX5_SET(fte_match_set_misc4, misc4_m,
7906                  prog_sample_field_id_0, samples[0]);
7907         /* It makes no sense to set the sample ID in the mask field. */
7908         MLX5_SET(fte_match_set_misc4, misc4_v,
7909                  prog_sample_field_id_0, samples[0]);
7910         /*
7911          * Checking if message body part needs to be matched.
7912          * Some wildcard rules only matching type field should be supported.
7913          */
7914         if (ecpri_m->hdr.dummy[0]) {
7915                 switch (ecpri_v->hdr.common.type) {
7916                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
7917                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
7918                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
7919                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
7920                                             prog_sample_field_value_1);
7921                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
7922                                             prog_sample_field_value_1);
7923                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
7924                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0];
7925                         /* Sample#1, to match message body, offset 4. */
7926                         MLX5_SET(fte_match_set_misc4, misc4_m,
7927                                  prog_sample_field_id_1, samples[1]);
7928                         MLX5_SET(fte_match_set_misc4, misc4_v,
7929                                  prog_sample_field_id_1, samples[1]);
7930                         break;
7931                 default:
7932                         /* Others, do not match any sample ID. */
7933                         break;
7934                 }
7935         }
7936 }
7937
7938 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
7939
7940 #define HEADER_IS_ZERO(match_criteria, headers)                              \
7941         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
7942                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
7943
7944 /**
7945  * Calculate flow matcher enable bitmap.
7946  *
7947  * @param match_criteria
7948  *   Pointer to flow matcher criteria.
7949  *
7950  * @return
7951  *   Bitmap of enabled fields.
7952  */
7953 static uint8_t
7954 flow_dv_matcher_enable(uint32_t *match_criteria)
7955 {
7956         uint8_t match_criteria_enable;
7957
7958         match_criteria_enable =
7959                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
7960                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
7961         match_criteria_enable |=
7962                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
7963                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
7964         match_criteria_enable |=
7965                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
7966                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
7967         match_criteria_enable |=
7968                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
7969                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
7970         match_criteria_enable |=
7971                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
7972                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
7973         match_criteria_enable |=
7974                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
7975                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
7976         return match_criteria_enable;
7977 }
7978
7979
7980 /**
7981  * Get a flow table.
7982  *
7983  * @param[in, out] dev
7984  *   Pointer to rte_eth_dev structure.
7985  * @param[in] table_id
7986  *   Table id to use.
7987  * @param[in] egress
7988  *   Direction of the table.
7989  * @param[in] transfer
7990  *   E-Switch or NIC flow.
7991  * @param[out] error
7992  *   pointer to error structure.
7993  *
7994  * @return
7995  *   Returns tables resource based on the index, NULL in case of failed.
7996  */
7997 static struct mlx5_flow_tbl_resource *
7998 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
7999                          uint32_t table_id, uint8_t egress,
8000                          uint8_t transfer,
8001                          bool external,
8002                          const struct mlx5_flow_tunnel *tunnel,
8003                          uint32_t group_id,
8004                          struct rte_flow_error *error)
8005 {
8006         struct mlx5_priv *priv = dev->data->dev_private;
8007         struct mlx5_dev_ctx_shared *sh = priv->sh;
8008         struct mlx5_flow_tbl_resource *tbl;
8009         union mlx5_flow_tbl_key table_key = {
8010                 {
8011                         .table_id = table_id,
8012                         .reserved = 0,
8013                         .domain = !!transfer,
8014                         .direction = !!egress,
8015                 }
8016         };
8017         struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
8018                                                          table_key.v64);
8019         struct mlx5_flow_tbl_data_entry *tbl_data;
8020         uint32_t idx = 0;
8021         int ret;
8022         void *domain;
8023
8024         if (pos) {
8025                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
8026                                         entry);
8027                 tbl = &tbl_data->tbl;
8028                 __atomic_fetch_add(&tbl->refcnt, 1, __ATOMIC_RELAXED);
8029                 return tbl;
8030         }
8031         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
8032         if (!tbl_data) {
8033                 rte_flow_error_set(error, ENOMEM,
8034                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8035                                    NULL,
8036                                    "cannot allocate flow table data entry");
8037                 return NULL;
8038         }
8039         tbl_data->idx = idx;
8040         tbl_data->tunnel = tunnel;
8041         tbl_data->group_id = group_id;
8042         tbl_data->external = external;
8043         tbl = &tbl_data->tbl;
8044         pos = &tbl_data->entry;
8045         if (transfer)
8046                 domain = sh->fdb_domain;
8047         else if (egress)
8048                 domain = sh->tx_domain;
8049         else
8050                 domain = sh->rx_domain;
8051         ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj);
8052         if (ret) {
8053                 rte_flow_error_set(error, ENOMEM,
8054                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8055                                    NULL, "cannot create flow table object");
8056                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
8057                 return NULL;
8058         }
8059         /*
8060          * No multi-threads now, but still better to initialize the reference
8061          * count before insert it into the hash list.
8062          */
8063         __atomic_store_n(&tbl->refcnt, 0, __ATOMIC_RELAXED);
8064         /* Jump action reference count is initialized here. */
8065         __atomic_store_n(&tbl_data->jump.refcnt, 0, __ATOMIC_RELAXED);
8066         pos->key = table_key.v64;
8067         ret = mlx5_hlist_insert(sh->flow_tbls, pos);
8068         if (ret < 0) {
8069                 rte_flow_error_set(error, -ret,
8070                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8071                                    "cannot insert flow table data entry");
8072                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
8073                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
8074         }
8075         __atomic_fetch_add(&tbl->refcnt, 1, __ATOMIC_RELAXED);
8076         return tbl;
8077 }
8078
8079 /**
8080  * Release a flow table.
8081  *
8082  * @param[in] dev
8083  *   Pointer to rte_eth_dev structure.
8084  * @param[in] tbl
8085  *   Table resource to be released.
8086  *
8087  * @return
8088  *   Returns 0 if table was released, else return 1;
8089  */
8090 static int
8091 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
8092                              struct mlx5_flow_tbl_resource *tbl)
8093 {
8094         struct mlx5_priv *priv = dev->data->dev_private;
8095         struct mlx5_dev_ctx_shared *sh = priv->sh;
8096         struct mlx5_flow_tbl_data_entry *tbl_data =
8097                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
8098
8099         if (!tbl)
8100                 return 0;
8101         if (__atomic_sub_fetch(&tbl->refcnt, 1, __ATOMIC_RELAXED) == 0) {
8102                 struct mlx5_hlist_entry *pos = &tbl_data->entry;
8103
8104                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
8105                 tbl->obj = NULL;
8106                 if (is_tunnel_offload_active(dev) && tbl_data->external) {
8107                         struct mlx5_hlist_entry *he;
8108                         struct mlx5_hlist *tunnel_grp_hash;
8109                         struct mlx5_flow_tunnel_hub *thub =
8110                                                         mlx5_tunnel_hub(dev);
8111                         union tunnel_tbl_key tunnel_key = {
8112                                 .tunnel_id = tbl_data->tunnel ?
8113                                                 tbl_data->tunnel->tunnel_id : 0,
8114                                 .group = tbl_data->group_id
8115                         };
8116                         union mlx5_flow_tbl_key table_key = {
8117                                 .v64 = pos->key
8118                         };
8119                         uint32_t table_id = table_key.table_id;
8120
8121                         tunnel_grp_hash = tbl_data->tunnel ?
8122                                                 tbl_data->tunnel->groups :
8123                                                 thub->groups;
8124                         he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val);
8125                         if (he) {
8126                                 struct tunnel_tbl_entry *tte;
8127                                 tte = container_of(he, typeof(*tte), hash);
8128                                 MLX5_ASSERT(tte->flow_table == table_id);
8129                                 mlx5_hlist_remove(tunnel_grp_hash, he);
8130                                 mlx5_free(tte);
8131                         }
8132                         mlx5_flow_id_release(mlx5_tunnel_hub(dev)->table_ids,
8133                                              tunnel_flow_tbl_to_id(table_id));
8134                         DRV_LOG(DEBUG,
8135                                 "port %u release table_id %#x tunnel %u group %u",
8136                                 dev->data->port_id, table_id,
8137                                 tbl_data->tunnel ?
8138                                 tbl_data->tunnel->tunnel_id : 0,
8139                                 tbl_data->group_id);
8140                 }
8141                 /* remove the entry from the hash list and free memory. */
8142                 mlx5_hlist_remove(sh->flow_tbls, pos);
8143                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
8144                                 tbl_data->idx);
8145                 return 0;
8146         }
8147         return 1;
8148 }
8149
8150 /**
8151  * Register the flow matcher.
8152  *
8153  * @param[in, out] dev
8154  *   Pointer to rte_eth_dev structure.
8155  * @param[in, out] matcher
8156  *   Pointer to flow matcher.
8157  * @param[in, out] key
8158  *   Pointer to flow table key.
8159  * @parm[in, out] dev_flow
8160  *   Pointer to the dev_flow.
8161  * @param[out] error
8162  *   pointer to error structure.
8163  *
8164  * @return
8165  *   0 on success otherwise -errno and errno is set.
8166  */
8167 static int
8168 flow_dv_matcher_register(struct rte_eth_dev *dev,
8169                          struct mlx5_flow_dv_matcher *matcher,
8170                          union mlx5_flow_tbl_key *key,
8171                          struct mlx5_flow *dev_flow,
8172                          struct rte_flow_error *error)
8173 {
8174         struct mlx5_priv *priv = dev->data->dev_private;
8175         struct mlx5_dev_ctx_shared *sh = priv->sh;
8176         struct mlx5_flow_dv_matcher *cache_matcher;
8177         struct mlx5dv_flow_matcher_attr dv_attr = {
8178                 .type = IBV_FLOW_ATTR_NORMAL,
8179                 .match_mask = (void *)&matcher->mask,
8180         };
8181         struct mlx5_flow_tbl_resource *tbl;
8182         struct mlx5_flow_tbl_data_entry *tbl_data;
8183         int ret;
8184
8185         tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
8186                                        key->domain, false, NULL, 0, error);
8187         if (!tbl)
8188                 return -rte_errno;      /* No need to refill the error info */
8189         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
8190         /* Lookup from cache. */
8191         LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
8192                 if (matcher->crc == cache_matcher->crc &&
8193                     matcher->priority == cache_matcher->priority &&
8194                     !memcmp((const void *)matcher->mask.buf,
8195                             (const void *)cache_matcher->mask.buf,
8196                             cache_matcher->mask.size)) {
8197                         DRV_LOG(DEBUG,
8198                                 "%s group %u priority %hd use %s "
8199                                 "matcher %p: refcnt %d++",
8200                                 key->domain ? "FDB" : "NIC", key->table_id,
8201                                 cache_matcher->priority,
8202                                 key->direction ? "tx" : "rx",
8203                                 (void *)cache_matcher,
8204                                 __atomic_load_n(&cache_matcher->refcnt,
8205                                                 __ATOMIC_RELAXED));
8206                         __atomic_fetch_add(&cache_matcher->refcnt, 1,
8207                                            __ATOMIC_RELAXED);
8208                         dev_flow->handle->dvh.matcher = cache_matcher;
8209                         /* old matcher should not make the table ref++. */
8210                         flow_dv_tbl_resource_release(dev, tbl);
8211                         return 0;
8212                 }
8213         }
8214         /* Register new matcher. */
8215         cache_matcher = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache_matcher), 0,
8216                                     SOCKET_ID_ANY);
8217         if (!cache_matcher) {
8218                 flow_dv_tbl_resource_release(dev, tbl);
8219                 return rte_flow_error_set(error, ENOMEM,
8220                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8221                                           "cannot allocate matcher memory");
8222         }
8223         *cache_matcher = *matcher;
8224         dv_attr.match_criteria_enable =
8225                 flow_dv_matcher_enable(cache_matcher->mask.buf);
8226         dv_attr.priority = matcher->priority;
8227         if (key->direction)
8228                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
8229         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
8230                                                &cache_matcher->matcher_object);
8231         if (ret) {
8232                 mlx5_free(cache_matcher);
8233 #ifdef HAVE_MLX5DV_DR
8234                 flow_dv_tbl_resource_release(dev, tbl);
8235 #endif
8236                 return rte_flow_error_set(error, ENOMEM,
8237                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8238                                           NULL, "cannot create matcher");
8239         }
8240         /* Save the table information */
8241         cache_matcher->tbl = tbl;
8242         /* only matcher ref++, table ref++ already done above in get API. */
8243         __atomic_store_n(&cache_matcher->refcnt, 1, __ATOMIC_RELAXED);
8244         LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
8245         dev_flow->handle->dvh.matcher = cache_matcher;
8246         DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
8247                 key->domain ? "FDB" : "NIC", key->table_id,
8248                 cache_matcher->priority,
8249                 key->direction ? "tx" : "rx", (void *)cache_matcher,
8250                 __atomic_load_n(&cache_matcher->refcnt, __ATOMIC_RELAXED));
8251         return 0;
8252 }
8253
8254 /**
8255  * Find existing tag resource or create and register a new one.
8256  *
8257  * @param dev[in, out]
8258  *   Pointer to rte_eth_dev structure.
8259  * @param[in, out] tag_be24
8260  *   Tag value in big endian then R-shift 8.
8261  * @parm[in, out] dev_flow
8262  *   Pointer to the dev_flow.
8263  * @param[out] error
8264  *   pointer to error structure.
8265  *
8266  * @return
8267  *   0 on success otherwise -errno and errno is set.
8268  */
8269 static int
8270 flow_dv_tag_resource_register
8271                         (struct rte_eth_dev *dev,
8272                          uint32_t tag_be24,
8273                          struct mlx5_flow *dev_flow,
8274                          struct rte_flow_error *error)
8275 {
8276         struct mlx5_priv *priv = dev->data->dev_private;
8277         struct mlx5_dev_ctx_shared *sh = priv->sh;
8278         struct mlx5_flow_dv_tag_resource *cache_resource;
8279         struct mlx5_hlist_entry *entry;
8280         int ret;
8281
8282         /* Lookup a matching resource from cache. */
8283         entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
8284         if (entry) {
8285                 cache_resource = container_of
8286                         (entry, struct mlx5_flow_dv_tag_resource, entry);
8287                 __atomic_fetch_add(&cache_resource->refcnt, 1,
8288                                    __ATOMIC_RELAXED);
8289                 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
8290                 dev_flow->dv.tag_resource = cache_resource;
8291                 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
8292                         (void *)cache_resource,
8293                         __atomic_load_n(&cache_resource->refcnt,
8294                                         __ATOMIC_RELAXED));
8295                 return 0;
8296         }
8297         /* Register new resource. */
8298         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG],
8299                                        &dev_flow->handle->dvh.rix_tag);
8300         if (!cache_resource)
8301                 return rte_flow_error_set(error, ENOMEM,
8302                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8303                                           "cannot allocate resource memory");
8304         cache_resource->entry.key = (uint64_t)tag_be24;
8305         ret = mlx5_flow_os_create_flow_action_tag(tag_be24,
8306                                                   &cache_resource->action);
8307         if (ret) {
8308                 mlx5_free(cache_resource);
8309                 return rte_flow_error_set(error, ENOMEM,
8310                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8311                                           NULL, "cannot create action");
8312         }
8313         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8314         if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
8315                 mlx5_flow_os_destroy_flow_action(cache_resource->action);
8316                 mlx5_free(cache_resource);
8317                 return rte_flow_error_set(error, EEXIST,
8318                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8319                                           NULL, "cannot insert tag");
8320         }
8321         dev_flow->dv.tag_resource = cache_resource;
8322         DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
8323                 (void *)cache_resource,
8324                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8325         return 0;
8326 }
8327
8328 /**
8329  * Release the tag.
8330  *
8331  * @param dev
8332  *   Pointer to Ethernet device.
8333  * @param tag_idx
8334  *   Tag index.
8335  *
8336  * @return
8337  *   1 while a reference on it exists, 0 when freed.
8338  */
8339 static int
8340 flow_dv_tag_release(struct rte_eth_dev *dev,
8341                     uint32_t tag_idx)
8342 {
8343         struct mlx5_priv *priv = dev->data->dev_private;
8344         struct mlx5_dev_ctx_shared *sh = priv->sh;
8345         struct mlx5_flow_dv_tag_resource *tag;
8346
8347         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8348         if (!tag)
8349                 return 0;
8350         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
8351                 dev->data->port_id, (void *)tag,
8352                 __atomic_load_n(&tag->refcnt, __ATOMIC_RELAXED));
8353         if (__atomic_sub_fetch(&tag->refcnt, 1, __ATOMIC_RELAXED) == 0) {
8354                 claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
8355                 mlx5_hlist_remove(sh->tag_table, &tag->entry);
8356                 DRV_LOG(DEBUG, "port %u tag %p: removed",
8357                         dev->data->port_id, (void *)tag);
8358                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8359                 return 0;
8360         }
8361         return 1;
8362 }
8363
8364 /**
8365  * Translate port ID action to vport.
8366  *
8367  * @param[in] dev
8368  *   Pointer to rte_eth_dev structure.
8369  * @param[in] action
8370  *   Pointer to the port ID action.
8371  * @param[out] dst_port_id
8372  *   The target port ID.
8373  * @param[out] error
8374  *   Pointer to the error structure.
8375  *
8376  * @return
8377  *   0 on success, a negative errno value otherwise and rte_errno is set.
8378  */
8379 static int
8380 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
8381                                  const struct rte_flow_action *action,
8382                                  uint32_t *dst_port_id,
8383                                  struct rte_flow_error *error)
8384 {
8385         uint32_t port;
8386         struct mlx5_priv *priv;
8387         const struct rte_flow_action_port_id *conf =
8388                         (const struct rte_flow_action_port_id *)action->conf;
8389
8390         port = conf->original ? dev->data->port_id : conf->id;
8391         priv = mlx5_port_to_eswitch_info(port, false);
8392         if (!priv)
8393                 return rte_flow_error_set(error, -rte_errno,
8394                                           RTE_FLOW_ERROR_TYPE_ACTION,
8395                                           NULL,
8396                                           "No eswitch info was found for port");
8397 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
8398         /*
8399          * This parameter is transferred to
8400          * mlx5dv_dr_action_create_dest_ib_port().
8401          */
8402         *dst_port_id = priv->dev_port;
8403 #else
8404         /*
8405          * Legacy mode, no LAG configurations is supported.
8406          * This parameter is transferred to
8407          * mlx5dv_dr_action_create_dest_vport().
8408          */
8409         *dst_port_id = priv->vport_id;
8410 #endif
8411         return 0;
8412 }
8413
8414 /**
8415  * Create a counter with aging configuration.
8416  *
8417  * @param[in] dev
8418  *   Pointer to rte_eth_dev structure.
8419  * @param[out] count
8420  *   Pointer to the counter action configuration.
8421  * @param[in] age
8422  *   Pointer to the aging action configuration.
8423  *
8424  * @return
8425  *   Index to flow counter on success, 0 otherwise.
8426  */
8427 static uint32_t
8428 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
8429                                 struct mlx5_flow *dev_flow,
8430                                 const struct rte_flow_action_count *count,
8431                                 const struct rte_flow_action_age *age)
8432 {
8433         uint32_t counter;
8434         struct mlx5_age_param *age_param;
8435
8436         if (count && count->shared)
8437                 counter = flow_dv_counter_get_shared(dev, count->id);
8438         else
8439                 counter = flow_dv_counter_alloc(dev, !!age);
8440         if (!counter || age == NULL)
8441                 return counter;
8442         age_param  = flow_dv_counter_idx_get_age(dev, counter);
8443         age_param->context = age->context ? age->context :
8444                 (void *)(uintptr_t)(dev_flow->flow_idx);
8445         age_param->timeout = age->timeout;
8446         age_param->port_id = dev->data->port_id;
8447         __atomic_store_n(&age_param->sec_since_last_hit, 0, __ATOMIC_RELAXED);
8448         __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED);
8449         return counter;
8450 }
8451 /**
8452  * Add Tx queue matcher
8453  *
8454  * @param[in] dev
8455  *   Pointer to the dev struct.
8456  * @param[in, out] matcher
8457  *   Flow matcher.
8458  * @param[in, out] key
8459  *   Flow matcher value.
8460  * @param[in] item
8461  *   Flow pattern to translate.
8462  * @param[in] inner
8463  *   Item is inner pattern.
8464  */
8465 static void
8466 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
8467                                 void *matcher, void *key,
8468                                 const struct rte_flow_item *item)
8469 {
8470         const struct mlx5_rte_flow_item_tx_queue *queue_m;
8471         const struct mlx5_rte_flow_item_tx_queue *queue_v;
8472         void *misc_m =
8473                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8474         void *misc_v =
8475                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8476         struct mlx5_txq_ctrl *txq;
8477         uint32_t queue;
8478
8479
8480         queue_m = (const void *)item->mask;
8481         if (!queue_m)
8482                 return;
8483         queue_v = (const void *)item->spec;
8484         if (!queue_v)
8485                 return;
8486         txq = mlx5_txq_get(dev, queue_v->queue);
8487         if (!txq)
8488                 return;
8489         queue = txq->obj->sq->id;
8490         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
8491         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
8492                  queue & queue_m->queue);
8493         mlx5_txq_release(dev, queue_v->queue);
8494 }
8495
8496 /**
8497  * Set the hash fields according to the @p flow information.
8498  *
8499  * @param[in] dev_flow
8500  *   Pointer to the mlx5_flow.
8501  * @param[in] rss_desc
8502  *   Pointer to the mlx5_flow_rss_desc.
8503  */
8504 static void
8505 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
8506                        struct mlx5_flow_rss_desc *rss_desc)
8507 {
8508         uint64_t items = dev_flow->handle->layers;
8509         int rss_inner = 0;
8510         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
8511
8512         dev_flow->hash_fields = 0;
8513 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
8514         if (rss_desc->level >= 2) {
8515                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
8516                 rss_inner = 1;
8517         }
8518 #endif
8519         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
8520             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
8521                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
8522                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
8523                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
8524                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
8525                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
8526                         else
8527                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
8528                 }
8529         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
8530                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
8531                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
8532                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
8533                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
8534                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
8535                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
8536                         else
8537                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
8538                 }
8539         }
8540         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
8541             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
8542                 if (rss_types & ETH_RSS_UDP) {
8543                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
8544                                 dev_flow->hash_fields |=
8545                                                 IBV_RX_HASH_SRC_PORT_UDP;
8546                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
8547                                 dev_flow->hash_fields |=
8548                                                 IBV_RX_HASH_DST_PORT_UDP;
8549                         else
8550                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
8551                 }
8552         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
8553                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
8554                 if (rss_types & ETH_RSS_TCP) {
8555                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
8556                                 dev_flow->hash_fields |=
8557                                                 IBV_RX_HASH_SRC_PORT_TCP;
8558                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
8559                                 dev_flow->hash_fields |=
8560                                                 IBV_RX_HASH_DST_PORT_TCP;
8561                         else
8562                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
8563                 }
8564         }
8565 }
8566
8567 /**
8568  * Create an Rx Hash queue.
8569  *
8570  * @param dev
8571  *   Pointer to Ethernet device.
8572  * @param[in] dev_flow
8573  *   Pointer to the mlx5_flow.
8574  * @param[in] rss_desc
8575  *   Pointer to the mlx5_flow_rss_desc.
8576  * @param[out] hrxq_idx
8577  *   Hash Rx queue index.
8578  *
8579  * @return
8580  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
8581  */
8582 static struct mlx5_hrxq *
8583 flow_dv_handle_rx_queue(struct rte_eth_dev *dev,
8584                         struct mlx5_flow *dev_flow,
8585                         struct mlx5_flow_rss_desc *rss_desc,
8586                         uint32_t *hrxq_idx)
8587 {
8588         struct mlx5_priv *priv = dev->data->dev_private;
8589         struct mlx5_flow_handle *dh = dev_flow->handle;
8590         struct mlx5_hrxq *hrxq;
8591
8592         MLX5_ASSERT(rss_desc->queue_num);
8593         *hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key, MLX5_RSS_HASH_KEY_LEN,
8594                                   dev_flow->hash_fields,
8595                                   rss_desc->queue, rss_desc->queue_num);
8596         if (!*hrxq_idx) {
8597                 *hrxq_idx = mlx5_hrxq_new
8598                                 (dev, rss_desc->key, MLX5_RSS_HASH_KEY_LEN,
8599                                  dev_flow->hash_fields,
8600                                  rss_desc->queue, rss_desc->queue_num,
8601                                  !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL),
8602                                  false);
8603                 if (!*hrxq_idx)
8604                         return NULL;
8605         }
8606         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
8607                               *hrxq_idx);
8608         return hrxq;
8609 }
8610
8611 /**
8612  * Find existing sample resource or create and register a new one.
8613  *
8614  * @param[in, out] dev
8615  *   Pointer to rte_eth_dev structure.
8616  * @param[in] attr
8617  *   Attributes of flow that includes this item.
8618  * @param[in] resource
8619  *   Pointer to sample resource.
8620  * @parm[in, out] dev_flow
8621  *   Pointer to the dev_flow.
8622  * @param[in, out] sample_dv_actions
8623  *   Pointer to sample actions list.
8624  * @param[out] error
8625  *   pointer to error structure.
8626  *
8627  * @return
8628  *   0 on success otherwise -errno and errno is set.
8629  */
8630 static int
8631 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
8632                          const struct rte_flow_attr *attr,
8633                          struct mlx5_flow_dv_sample_resource *resource,
8634                          struct mlx5_flow *dev_flow,
8635                          void **sample_dv_actions,
8636                          struct rte_flow_error *error)
8637 {
8638         struct mlx5_flow_dv_sample_resource *cache_resource;
8639         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
8640         struct mlx5_priv *priv = dev->data->dev_private;
8641         struct mlx5_dev_ctx_shared *sh = priv->sh;
8642         struct mlx5_flow_tbl_resource *tbl;
8643         uint32_t idx = 0;
8644         const uint32_t next_ft_step = 1;
8645         uint32_t next_ft_id = resource->ft_id + next_ft_step;
8646
8647         /* Lookup a matching resource from cache. */
8648         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_SAMPLE], sh->sample_action_list,
8649                       idx, cache_resource, next) {
8650                 if (resource->ratio == cache_resource->ratio &&
8651                     resource->ft_type == cache_resource->ft_type &&
8652                     resource->ft_id == cache_resource->ft_id &&
8653                     resource->set_action == cache_resource->set_action &&
8654                     !memcmp((void *)&resource->sample_act,
8655                             (void *)&cache_resource->sample_act,
8656                             sizeof(struct mlx5_flow_sub_actions_list))) {
8657                         DRV_LOG(DEBUG, "sample resource %p: refcnt %d++",
8658                                 (void *)cache_resource,
8659                                 __atomic_load_n(&cache_resource->refcnt,
8660                                                 __ATOMIC_RELAXED));
8661                         __atomic_fetch_add(&cache_resource->refcnt, 1,
8662                                            __ATOMIC_RELAXED);
8663                         dev_flow->handle->dvh.rix_sample = idx;
8664                         dev_flow->dv.sample_res = cache_resource;
8665                         return 0;
8666                 }
8667         }
8668         /* Register new sample resource. */
8669         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE],
8670                                        &dev_flow->handle->dvh.rix_sample);
8671         if (!cache_resource)
8672                 return rte_flow_error_set(error, ENOMEM,
8673                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8674                                           NULL,
8675                                           "cannot allocate resource memory");
8676         *cache_resource = *resource;
8677         /* Create normal path table level */
8678         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
8679                                         attr->egress, attr->transfer,
8680                                         dev_flow->external, NULL, 0, error);
8681         if (!tbl) {
8682                 rte_flow_error_set(error, ENOMEM,
8683                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8684                                           NULL,
8685                                           "fail to create normal path table "
8686                                           "for sample");
8687                 goto error;
8688         }
8689         cache_resource->normal_path_tbl = tbl;
8690         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
8691                 cache_resource->default_miss =
8692                                 mlx5_glue->dr_create_flow_action_default_miss();
8693                 if (!cache_resource->default_miss) {
8694                         rte_flow_error_set(error, ENOMEM,
8695                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8696                                                 NULL,
8697                                                 "cannot create default miss "
8698                                                 "action");
8699                         goto error;
8700                 }
8701                 sample_dv_actions[resource->sample_act.actions_num++] =
8702                                                 cache_resource->default_miss;
8703         }
8704         /* Create a DR sample action */
8705         sampler_attr.sample_ratio = cache_resource->ratio;
8706         sampler_attr.default_next_table = tbl->obj;
8707         sampler_attr.num_sample_actions = resource->sample_act.actions_num;
8708         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
8709                                                         &sample_dv_actions[0];
8710         sampler_attr.action = cache_resource->set_action;
8711         cache_resource->verbs_action =
8712                 mlx5_glue->dr_create_flow_action_sampler(&sampler_attr);
8713         if (!cache_resource->verbs_action) {
8714                 rte_flow_error_set(error, ENOMEM,
8715                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8716                                         NULL, "cannot create sample action");
8717                 goto error;
8718         }
8719         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8720         ILIST_INSERT(sh->ipool[MLX5_IPOOL_SAMPLE], &sh->sample_action_list,
8721                      dev_flow->handle->dvh.rix_sample, cache_resource,
8722                      next);
8723         dev_flow->dv.sample_res = cache_resource;
8724         DRV_LOG(DEBUG, "new sample resource %p: refcnt %d++",
8725                 (void *)cache_resource,
8726                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8727         return 0;
8728 error:
8729         if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
8730                 if (cache_resource->default_miss)
8731                         claim_zero(mlx5_glue->destroy_flow_action
8732                                 (cache_resource->default_miss));
8733         } else {
8734                 if (cache_resource->sample_idx.rix_hrxq &&
8735                     !mlx5_hrxq_release(dev,
8736                                 cache_resource->sample_idx.rix_hrxq))
8737                         cache_resource->sample_idx.rix_hrxq = 0;
8738                 if (cache_resource->sample_idx.rix_tag &&
8739                     !flow_dv_tag_release(dev,
8740                                 cache_resource->sample_idx.rix_tag))
8741                         cache_resource->sample_idx.rix_tag = 0;
8742                 if (cache_resource->sample_idx.cnt) {
8743                         flow_dv_counter_release(dev,
8744                                 cache_resource->sample_idx.cnt);
8745                         cache_resource->sample_idx.cnt = 0;
8746                 }
8747         }
8748         if (cache_resource->normal_path_tbl)
8749                 flow_dv_tbl_resource_release(dev,
8750                                 cache_resource->normal_path_tbl);
8751         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE],
8752                                 dev_flow->handle->dvh.rix_sample);
8753         dev_flow->handle->dvh.rix_sample = 0;
8754         return -rte_errno;
8755 }
8756
8757 /**
8758  * Find existing destination array resource or create and register a new one.
8759  *
8760  * @param[in, out] dev
8761  *   Pointer to rte_eth_dev structure.
8762  * @param[in] attr
8763  *   Attributes of flow that includes this item.
8764  * @param[in] resource
8765  *   Pointer to destination array resource.
8766  * @parm[in, out] dev_flow
8767  *   Pointer to the dev_flow.
8768  * @param[out] error
8769  *   pointer to error structure.
8770  *
8771  * @return
8772  *   0 on success otherwise -errno and errno is set.
8773  */
8774 static int
8775 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
8776                          const struct rte_flow_attr *attr,
8777                          struct mlx5_flow_dv_dest_array_resource *resource,
8778                          struct mlx5_flow *dev_flow,
8779                          struct rte_flow_error *error)
8780 {
8781         struct mlx5_flow_dv_dest_array_resource *cache_resource;
8782         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
8783         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
8784         struct mlx5_priv *priv = dev->data->dev_private;
8785         struct mlx5_dev_ctx_shared *sh = priv->sh;
8786         struct mlx5_flow_sub_actions_list *sample_act;
8787         struct mlx5dv_dr_domain *domain;
8788         uint32_t idx = 0;
8789
8790         /* Lookup a matching resource from cache. */
8791         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8792                       sh->dest_array_list,
8793                       idx, cache_resource, next) {
8794                 if (resource->num_of_dest == cache_resource->num_of_dest &&
8795                     resource->ft_type == cache_resource->ft_type &&
8796                     !memcmp((void *)cache_resource->sample_act,
8797                             (void *)resource->sample_act,
8798                            (resource->num_of_dest *
8799                            sizeof(struct mlx5_flow_sub_actions_list)))) {
8800                         DRV_LOG(DEBUG, "dest array resource %p: refcnt %d++",
8801                                 (void *)cache_resource,
8802                                 __atomic_load_n(&cache_resource->refcnt,
8803                                                 __ATOMIC_RELAXED));
8804                         __atomic_fetch_add(&cache_resource->refcnt, 1,
8805                                            __ATOMIC_RELAXED);
8806                         dev_flow->handle->dvh.rix_dest_array = idx;
8807                         dev_flow->dv.dest_array_res = cache_resource;
8808                         return 0;
8809                 }
8810         }
8811         /* Register new destination array resource. */
8812         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8813                                        &dev_flow->handle->dvh.rix_dest_array);
8814         if (!cache_resource)
8815                 return rte_flow_error_set(error, ENOMEM,
8816                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8817                                           NULL,
8818                                           "cannot allocate resource memory");
8819         *cache_resource = *resource;
8820         if (attr->transfer)
8821                 domain = sh->fdb_domain;
8822         else if (attr->ingress)
8823                 domain = sh->rx_domain;
8824         else
8825                 domain = sh->tx_domain;
8826         for (idx = 0; idx < resource->num_of_dest; idx++) {
8827                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
8828                                  mlx5_malloc(MLX5_MEM_ZERO,
8829                                  sizeof(struct mlx5dv_dr_action_dest_attr),
8830                                  0, SOCKET_ID_ANY);
8831                 if (!dest_attr[idx]) {
8832                         rte_flow_error_set(error, ENOMEM,
8833                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8834                                            NULL,
8835                                            "cannot allocate resource memory");
8836                         goto error;
8837                 }
8838                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
8839                 sample_act = &resource->sample_act[idx];
8840                 if (sample_act->action_flags == MLX5_FLOW_ACTION_QUEUE) {
8841                         dest_attr[idx]->dest = sample_act->dr_queue_action;
8842                 } else if (sample_act->action_flags ==
8843                           (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP)) {
8844                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
8845                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
8846                         dest_attr[idx]->dest_reformat->reformat =
8847                                         sample_act->dr_encap_action;
8848                         dest_attr[idx]->dest_reformat->dest =
8849                                         sample_act->dr_port_id_action;
8850                 } else if (sample_act->action_flags ==
8851                            MLX5_FLOW_ACTION_PORT_ID) {
8852                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
8853                 }
8854         }
8855         /* create a dest array actioin */
8856         cache_resource->action = mlx5_glue->dr_create_flow_action_dest_array
8857                                                 (domain,
8858                                                  cache_resource->num_of_dest,
8859                                                  dest_attr);
8860         if (!cache_resource->action) {
8861                 rte_flow_error_set(error, ENOMEM,
8862                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8863                                    NULL,
8864                                    "cannot create destination array action");
8865                 goto error;
8866         }
8867         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8868         ILIST_INSERT(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8869                      &sh->dest_array_list,
8870                      dev_flow->handle->dvh.rix_dest_array, cache_resource,
8871                      next);
8872         dev_flow->dv.dest_array_res = cache_resource;
8873         DRV_LOG(DEBUG, "new destination array resource %p: refcnt %d++",
8874                 (void *)cache_resource,
8875                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8876         for (idx = 0; idx < resource->num_of_dest; idx++)
8877                 mlx5_free(dest_attr[idx]);
8878         return 0;
8879 error:
8880         for (idx = 0; idx < resource->num_of_dest; idx++) {
8881                 struct mlx5_flow_sub_actions_idx *act_res =
8882                                         &cache_resource->sample_idx[idx];
8883                 if (act_res->rix_hrxq &&
8884                     !mlx5_hrxq_release(dev,
8885                                 act_res->rix_hrxq))
8886                         act_res->rix_hrxq = 0;
8887                 if (act_res->rix_encap_decap &&
8888                         !flow_dv_encap_decap_resource_release(dev,
8889                                 act_res->rix_encap_decap))
8890                         act_res->rix_encap_decap = 0;
8891                 if (act_res->rix_port_id_action &&
8892                         !flow_dv_port_id_action_resource_release(dev,
8893                                 act_res->rix_port_id_action))
8894                         act_res->rix_port_id_action = 0;
8895                 if (dest_attr[idx])
8896                         mlx5_free(dest_attr[idx]);
8897         }
8898
8899         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8900                                 dev_flow->handle->dvh.rix_dest_array);
8901         dev_flow->handle->dvh.rix_dest_array = 0;
8902         return -rte_errno;
8903 }
8904
8905 /**
8906  * Convert Sample action to DV specification.
8907  *
8908  * @param[in] dev
8909  *   Pointer to rte_eth_dev structure.
8910  * @param[in] action
8911  *   Pointer to action structure.
8912  * @param[in, out] dev_flow
8913  *   Pointer to the mlx5_flow.
8914  * @param[in] attr
8915  *   Pointer to the flow attributes.
8916  * @param[in, out] num_of_dest
8917  *   Pointer to the num of destination.
8918  * @param[in, out] sample_actions
8919  *   Pointer to sample actions list.
8920  * @param[in, out] res
8921  *   Pointer to sample resource.
8922  * @param[out] error
8923  *   Pointer to the error structure.
8924  *
8925  * @return
8926  *   0 on success, a negative errno value otherwise and rte_errno is set.
8927  */
8928 static int
8929 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
8930                                 const struct rte_flow_action *action,
8931                                 struct mlx5_flow *dev_flow,
8932                                 const struct rte_flow_attr *attr,
8933                                 uint32_t *num_of_dest,
8934                                 void **sample_actions,
8935                                 struct mlx5_flow_dv_sample_resource *res,
8936                                 struct rte_flow_error *error)
8937 {
8938         struct mlx5_priv *priv = dev->data->dev_private;
8939         const struct rte_flow_action_sample *sample_action;
8940         const struct rte_flow_action *sub_actions;
8941         const struct rte_flow_action_queue *queue;
8942         struct mlx5_flow_sub_actions_list *sample_act;
8943         struct mlx5_flow_sub_actions_idx *sample_idx;
8944         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
8945                                               priv->rss_desc)
8946                                               [!!priv->flow_nested_idx];
8947         uint64_t action_flags = 0;
8948
8949         sample_act = &res->sample_act;
8950         sample_idx = &res->sample_idx;
8951         sample_action = (const struct rte_flow_action_sample *)action->conf;
8952         res->ratio = sample_action->ratio;
8953         sub_actions = sample_action->actions;
8954         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
8955                 int type = sub_actions->type;
8956                 uint32_t pre_rix = 0;
8957                 void *pre_r;
8958                 switch (type) {
8959                 case RTE_FLOW_ACTION_TYPE_QUEUE:
8960                 {
8961                         struct mlx5_hrxq *hrxq;
8962                         uint32_t hrxq_idx;
8963
8964                         queue = sub_actions->conf;
8965                         rss_desc->queue_num = 1;
8966                         rss_desc->queue[0] = queue->index;
8967                         hrxq = flow_dv_handle_rx_queue(dev, dev_flow,
8968                                         rss_desc, &hrxq_idx);
8969                         if (!hrxq)
8970                                 return rte_flow_error_set
8971                                         (error, rte_errno,
8972                                          RTE_FLOW_ERROR_TYPE_ACTION,
8973                                          NULL,
8974                                          "cannot create fate queue");
8975                         sample_act->dr_queue_action = hrxq->action;
8976                         sample_idx->rix_hrxq = hrxq_idx;
8977                         sample_actions[sample_act->actions_num++] =
8978                                                 hrxq->action;
8979                         (*num_of_dest)++;
8980                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
8981                         if (action_flags & MLX5_FLOW_ACTION_MARK)
8982                                 dev_flow->handle->rix_hrxq = hrxq_idx;
8983                         dev_flow->handle->fate_action =
8984                                         MLX5_FLOW_FATE_QUEUE;
8985                         break;
8986                 }
8987                 case RTE_FLOW_ACTION_TYPE_MARK:
8988                 {
8989                         uint32_t tag_be = mlx5_flow_mark_set
8990                                 (((const struct rte_flow_action_mark *)
8991                                 (sub_actions->conf))->id);
8992
8993                         dev_flow->handle->mark = 1;
8994                         pre_rix = dev_flow->handle->dvh.rix_tag;
8995                         /* Save the mark resource before sample */
8996                         pre_r = dev_flow->dv.tag_resource;
8997                         if (flow_dv_tag_resource_register(dev, tag_be,
8998                                                   dev_flow, error))
8999                                 return -rte_errno;
9000                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9001                         sample_act->dr_tag_action =
9002                                 dev_flow->dv.tag_resource->action;
9003                         sample_idx->rix_tag =
9004                                 dev_flow->handle->dvh.rix_tag;
9005                         sample_actions[sample_act->actions_num++] =
9006                                                 sample_act->dr_tag_action;
9007                         /* Recover the mark resource after sample */
9008                         dev_flow->dv.tag_resource = pre_r;
9009                         dev_flow->handle->dvh.rix_tag = pre_rix;
9010                         action_flags |= MLX5_FLOW_ACTION_MARK;
9011                         break;
9012                 }
9013                 case RTE_FLOW_ACTION_TYPE_COUNT:
9014                 {
9015                         uint32_t counter;
9016
9017                         counter = flow_dv_translate_create_counter(dev,
9018                                         dev_flow, sub_actions->conf, 0);
9019                         if (!counter)
9020                                 return rte_flow_error_set
9021                                                 (error, rte_errno,
9022                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9023                                                  NULL,
9024                                                  "cannot create counter"
9025                                                  " object.");
9026                         sample_idx->cnt = counter;
9027                         sample_act->dr_cnt_action =
9028                                   (flow_dv_counter_get_by_idx(dev,
9029                                   counter, NULL))->action;
9030                         sample_actions[sample_act->actions_num++] =
9031                                                 sample_act->dr_cnt_action;
9032                         action_flags |= MLX5_FLOW_ACTION_COUNT;
9033                         break;
9034                 }
9035                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
9036                 {
9037                         struct mlx5_flow_dv_port_id_action_resource
9038                                         port_id_resource;
9039                         uint32_t port_id = 0;
9040
9041                         memset(&port_id_resource, 0, sizeof(port_id_resource));
9042                         /* Save the port id resource before sample */
9043                         pre_rix = dev_flow->handle->rix_port_id_action;
9044                         pre_r = dev_flow->dv.port_id_action;
9045                         if (flow_dv_translate_action_port_id(dev, sub_actions,
9046                                                              &port_id, error))
9047                                 return -rte_errno;
9048                         port_id_resource.port_id = port_id;
9049                         if (flow_dv_port_id_action_resource_register
9050                             (dev, &port_id_resource, dev_flow, error))
9051                                 return -rte_errno;
9052                         sample_act->dr_port_id_action =
9053                                 dev_flow->dv.port_id_action->action;
9054                         sample_idx->rix_port_id_action =
9055                                 dev_flow->handle->rix_port_id_action;
9056                         sample_actions[sample_act->actions_num++] =
9057                                                 sample_act->dr_port_id_action;
9058                         /* Recover the port id resource after sample */
9059                         dev_flow->dv.port_id_action = pre_r;
9060                         dev_flow->handle->rix_port_id_action = pre_rix;
9061                         (*num_of_dest)++;
9062                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9063                         break;
9064                 }
9065                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
9066                         /* Save the encap resource before sample */
9067                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
9068                         pre_r = dev_flow->dv.encap_decap;
9069                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
9070                                                            dev_flow,
9071                                                            attr->transfer,
9072                                                            error))
9073                                 return -rte_errno;
9074                         sample_act->dr_encap_action =
9075                                 dev_flow->dv.encap_decap->action;
9076                         sample_idx->rix_encap_decap =
9077                                 dev_flow->handle->dvh.rix_encap_decap;
9078                         sample_actions[sample_act->actions_num++] =
9079                                                 sample_act->dr_encap_action;
9080                         /* Recover the encap resource after sample */
9081                         dev_flow->dv.encap_decap = pre_r;
9082                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
9083                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9084                         break;
9085                 default:
9086                         return rte_flow_error_set(error, EINVAL,
9087                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9088                                 NULL,
9089                                 "Not support for sampler action");
9090                 }
9091         }
9092         sample_act->action_flags = action_flags;
9093         res->ft_id = dev_flow->dv.group;
9094         if (attr->transfer) {
9095                 union {
9096                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
9097                         uint64_t set_action;
9098                 } action_ctx = { .set_action = 0 };
9099
9100                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
9101                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
9102                          MLX5_MODIFICATION_TYPE_SET);
9103                 MLX5_SET(set_action_in, action_ctx.action_in, field,
9104                          MLX5_MODI_META_REG_C_0);
9105                 MLX5_SET(set_action_in, action_ctx.action_in, data,
9106                          priv->vport_meta_tag);
9107                 res->set_action = action_ctx.set_action;
9108         } else if (attr->ingress) {
9109                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
9110         }
9111         return 0;
9112 }
9113
9114 /**
9115  * Convert Sample action to DV specification.
9116  *
9117  * @param[in] dev
9118  *   Pointer to rte_eth_dev structure.
9119  * @param[in, out] dev_flow
9120  *   Pointer to the mlx5_flow.
9121  * @param[in] attr
9122  *   Pointer to the flow attributes.
9123  * @param[in] num_of_dest
9124  *   The num of destination.
9125  * @param[in, out] res
9126  *   Pointer to sample resource.
9127  * @param[in, out] mdest_res
9128  *   Pointer to destination array resource.
9129  * @param[in] sample_actions
9130  *   Pointer to sample path actions list.
9131  * @param[in] action_flags
9132  *   Holds the actions detected until now.
9133  * @param[out] error
9134  *   Pointer to the error structure.
9135  *
9136  * @return
9137  *   0 on success, a negative errno value otherwise and rte_errno is set.
9138  */
9139 static int
9140 flow_dv_create_action_sample(struct rte_eth_dev *dev,
9141                              struct mlx5_flow *dev_flow,
9142                              const struct rte_flow_attr *attr,
9143                              uint32_t num_of_dest,
9144                              struct mlx5_flow_dv_sample_resource *res,
9145                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
9146                              void **sample_actions,
9147                              uint64_t action_flags,
9148                              struct rte_flow_error *error)
9149 {
9150         struct mlx5_priv *priv = dev->data->dev_private;
9151         /* update normal path action resource into last index of array */
9152         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
9153         struct mlx5_flow_sub_actions_list *sample_act =
9154                                         &mdest_res->sample_act[dest_index];
9155         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
9156                                               priv->rss_desc)
9157                                               [!!priv->flow_nested_idx];
9158         uint32_t normal_idx = 0;
9159         struct mlx5_hrxq *hrxq;
9160         uint32_t hrxq_idx;
9161
9162         if (num_of_dest > 1) {
9163                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
9164                         /* Handle QP action for mirroring */
9165                         hrxq = flow_dv_handle_rx_queue(dev, dev_flow,
9166                                                        rss_desc, &hrxq_idx);
9167                         if (!hrxq)
9168                                 return rte_flow_error_set
9169                                      (error, rte_errno,
9170                                       RTE_FLOW_ERROR_TYPE_ACTION,
9171                                       NULL,
9172                                       "cannot create rx queue");
9173                         normal_idx++;
9174                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
9175                         sample_act->dr_queue_action = hrxq->action;
9176                         if (action_flags & MLX5_FLOW_ACTION_MARK)
9177                                 dev_flow->handle->rix_hrxq = hrxq_idx;
9178                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9179                 }
9180                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
9181                         normal_idx++;
9182                         mdest_res->sample_idx[dest_index].rix_encap_decap =
9183                                 dev_flow->handle->dvh.rix_encap_decap;
9184                         sample_act->dr_encap_action =
9185                                 dev_flow->dv.encap_decap->action;
9186                 }
9187                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
9188                         normal_idx++;
9189                         mdest_res->sample_idx[dest_index].rix_port_id_action =
9190                                 dev_flow->handle->rix_port_id_action;
9191                         sample_act->dr_port_id_action =
9192                                 dev_flow->dv.port_id_action->action;
9193                 }
9194                 sample_act->actions_num = normal_idx;
9195                 /* update sample action resource into first index of array */
9196                 mdest_res->ft_type = res->ft_type;
9197                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
9198                                 sizeof(struct mlx5_flow_sub_actions_idx));
9199                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
9200                                 sizeof(struct mlx5_flow_sub_actions_list));
9201                 mdest_res->num_of_dest = num_of_dest;
9202                 if (flow_dv_dest_array_resource_register(dev, attr, mdest_res,
9203                                                          dev_flow, error))
9204                         return rte_flow_error_set(error, EINVAL,
9205                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9206                                                   NULL, "can't create sample "
9207                                                   "action");
9208         } else {
9209                 if (flow_dv_sample_resource_register(dev, attr, res, dev_flow,
9210                                                      sample_actions, error))
9211                         return rte_flow_error_set(error, EINVAL,
9212                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9213                                                   NULL,
9214                                                   "can't create sample action");
9215         }
9216         return 0;
9217 }
9218
9219 /**
9220  * Fill the flow with DV spec, lock free
9221  * (mutex should be acquired by caller).
9222  *
9223  * @param[in] dev
9224  *   Pointer to rte_eth_dev structure.
9225  * @param[in, out] dev_flow
9226  *   Pointer to the sub flow.
9227  * @param[in] attr
9228  *   Pointer to the flow attributes.
9229  * @param[in] items
9230  *   Pointer to the list of items.
9231  * @param[in] actions
9232  *   Pointer to the list of actions.
9233  * @param[out] error
9234  *   Pointer to the error structure.
9235  *
9236  * @return
9237  *   0 on success, a negative errno value otherwise and rte_errno is set.
9238  */
9239 static int
9240 __flow_dv_translate(struct rte_eth_dev *dev,
9241                     struct mlx5_flow *dev_flow,
9242                     const struct rte_flow_attr *attr,
9243                     const struct rte_flow_item items[],
9244                     const struct rte_flow_action actions[],
9245                     struct rte_flow_error *error)
9246 {
9247         struct mlx5_priv *priv = dev->data->dev_private;
9248         struct mlx5_dev_config *dev_conf = &priv->config;
9249         struct rte_flow *flow = dev_flow->flow;
9250         struct mlx5_flow_handle *handle = dev_flow->handle;
9251         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
9252                                               priv->rss_desc)
9253                                               [!!priv->flow_nested_idx];
9254         uint64_t item_flags = 0;
9255         uint64_t last_item = 0;
9256         uint64_t action_flags = 0;
9257         uint64_t priority = attr->priority;
9258         struct mlx5_flow_dv_matcher matcher = {
9259                 .mask = {
9260                         .size = sizeof(matcher.mask.buf) -
9261                                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
9262                 },
9263         };
9264         int actions_n = 0;
9265         bool actions_end = false;
9266         union {
9267                 struct mlx5_flow_dv_modify_hdr_resource res;
9268                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
9269                             sizeof(struct mlx5_modification_cmd) *
9270                             (MLX5_MAX_MODIFY_NUM + 1)];
9271         } mhdr_dummy;
9272         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
9273         const struct rte_flow_action_count *count = NULL;
9274         const struct rte_flow_action_age *age = NULL;
9275         union flow_dv_attr flow_attr = { .attr = 0 };
9276         uint32_t tag_be;
9277         union mlx5_flow_tbl_key tbl_key;
9278         uint32_t modify_action_position = UINT32_MAX;
9279         void *match_mask = matcher.mask.buf;
9280         void *match_value = dev_flow->dv.value.buf;
9281         uint8_t next_protocol = 0xff;
9282         struct rte_vlan_hdr vlan = { 0 };
9283         struct mlx5_flow_dv_dest_array_resource mdest_res;
9284         struct mlx5_flow_dv_sample_resource sample_res;
9285         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
9286         struct mlx5_flow_sub_actions_list *sample_act;
9287         uint32_t sample_act_pos = UINT32_MAX;
9288         uint32_t num_of_dest = 0;
9289         int tmp_actions_n = 0;
9290         uint32_t table;
9291         int ret = 0;
9292         const struct mlx5_flow_tunnel *tunnel;
9293         struct flow_grp_info grp_info = {
9294                 .external = !!dev_flow->external,
9295                 .transfer = !!attr->transfer,
9296                 .fdb_def_rule = !!priv->fdb_def_rule,
9297         };
9298
9299         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
9300         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
9301         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
9302                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
9303         /* update normal path action resource into last index of array */
9304         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
9305         tunnel = is_flow_tunnel_match_rule(dev, attr, items, actions) ?
9306                  flow_items_to_tunnel(items) :
9307                  is_flow_tunnel_steer_rule(dev, attr, items, actions) ?
9308                  flow_actions_to_tunnel(actions) :
9309                  dev_flow->tunnel ? dev_flow->tunnel : NULL;
9310         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
9311                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
9312         grp_info.std_tbl_fix = tunnel_use_standard_attr_group_translate
9313                                 (dev, tunnel, attr, items, actions);
9314         ret = mlx5_flow_group_to_table(dev, tunnel, attr->group, &table,
9315                                        grp_info, error);
9316         if (ret)
9317                 return ret;
9318         dev_flow->dv.group = table;
9319         if (attr->transfer)
9320                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
9321         if (priority == MLX5_FLOW_PRIO_RSVD)
9322                 priority = dev_conf->flow_prio - 1;
9323         /* number of actions must be set to 0 in case of dirty stack. */
9324         mhdr_res->actions_num = 0;
9325         if (is_flow_tunnel_match_rule(dev, attr, items, actions)) {
9326                 /*
9327                  * do not add decap action if match rule drops packet
9328                  * HW rejects rules with decap & drop
9329                  */
9330                 bool add_decap = true;
9331                 const struct rte_flow_action *ptr = actions;
9332                 struct mlx5_flow_tbl_resource *tbl;
9333
9334                 for (; ptr->type != RTE_FLOW_ACTION_TYPE_END; ptr++) {
9335                         if (ptr->type == RTE_FLOW_ACTION_TYPE_DROP) {
9336                                 add_decap = false;
9337                                 break;
9338                         }
9339                 }
9340                 if (add_decap) {
9341                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
9342                                                            attr->transfer,
9343                                                            error))
9344                                 return -rte_errno;
9345                         dev_flow->dv.actions[actions_n++] =
9346                                         dev_flow->dv.encap_decap->action;
9347                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9348                 }
9349                 /*
9350                  * bind table_id with <group, table> for tunnel match rule.
9351                  * Tunnel set rule establishes that bind in JUMP action handler.
9352                  * Required for scenario when application creates tunnel match
9353                  * rule before tunnel set rule.
9354                  */
9355                 tbl = flow_dv_tbl_resource_get(dev, table, attr->egress,
9356                                                attr->transfer,
9357                                                !!dev_flow->external, tunnel,
9358                                                attr->group, error);
9359                 if (!tbl)
9360                         return rte_flow_error_set
9361                                (error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
9362                                actions, "cannot register tunnel group");
9363         }
9364         for (; !actions_end ; actions++) {
9365                 const struct rte_flow_action_queue *queue;
9366                 const struct rte_flow_action_rss *rss;
9367                 const struct rte_flow_action *action = actions;
9368                 const uint8_t *rss_key;
9369                 const struct rte_flow_action_meter *mtr;
9370                 struct mlx5_flow_tbl_resource *tbl;
9371                 uint32_t port_id = 0;
9372                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
9373                 int action_type = actions->type;
9374                 const struct rte_flow_action *found_action = NULL;
9375                 struct mlx5_flow_meter *fm = NULL;
9376                 uint32_t jump_group = 0;
9377
9378                 if (!mlx5_flow_os_action_supported(action_type))
9379                         return rte_flow_error_set(error, ENOTSUP,
9380                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9381                                                   actions,
9382                                                   "action not supported");
9383                 switch (action_type) {
9384                 case MLX5_RTE_FLOW_ACTION_TYPE_TUNNEL_SET:
9385                         action_flags |= MLX5_FLOW_ACTION_TUNNEL_SET;
9386                         break;
9387                 case RTE_FLOW_ACTION_TYPE_VOID:
9388                         break;
9389                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
9390                         if (flow_dv_translate_action_port_id(dev, action,
9391                                                              &port_id, error))
9392                                 return -rte_errno;
9393                         port_id_resource.port_id = port_id;
9394                         MLX5_ASSERT(!handle->rix_port_id_action);
9395                         if (flow_dv_port_id_action_resource_register
9396                             (dev, &port_id_resource, dev_flow, error))
9397                                 return -rte_errno;
9398                         dev_flow->dv.actions[actions_n++] =
9399                                         dev_flow->dv.port_id_action->action;
9400                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9401                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
9402                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9403                         num_of_dest++;
9404                         break;
9405                 case RTE_FLOW_ACTION_TYPE_FLAG:
9406                         action_flags |= MLX5_FLOW_ACTION_FLAG;
9407                         dev_flow->handle->mark = 1;
9408                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
9409                                 struct rte_flow_action_mark mark = {
9410                                         .id = MLX5_FLOW_MARK_DEFAULT,
9411                                 };
9412
9413                                 if (flow_dv_convert_action_mark(dev, &mark,
9414                                                                 mhdr_res,
9415                                                                 error))
9416                                         return -rte_errno;
9417                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
9418                                 break;
9419                         }
9420                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
9421                         /*
9422                          * Only one FLAG or MARK is supported per device flow
9423                          * right now. So the pointer to the tag resource must be
9424                          * zero before the register process.
9425                          */
9426                         MLX5_ASSERT(!handle->dvh.rix_tag);
9427                         if (flow_dv_tag_resource_register(dev, tag_be,
9428                                                           dev_flow, error))
9429                                 return -rte_errno;
9430                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9431                         dev_flow->dv.actions[actions_n++] =
9432                                         dev_flow->dv.tag_resource->action;
9433                         break;
9434                 case RTE_FLOW_ACTION_TYPE_MARK:
9435                         action_flags |= MLX5_FLOW_ACTION_MARK;
9436                         dev_flow->handle->mark = 1;
9437                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
9438                                 const struct rte_flow_action_mark *mark =
9439                                         (const struct rte_flow_action_mark *)
9440                                                 actions->conf;
9441
9442                                 if (flow_dv_convert_action_mark(dev, mark,
9443                                                                 mhdr_res,
9444                                                                 error))
9445                                         return -rte_errno;
9446                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
9447                                 break;
9448                         }
9449                         /* Fall-through */
9450                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
9451                         /* Legacy (non-extensive) MARK action. */
9452                         tag_be = mlx5_flow_mark_set
9453                               (((const struct rte_flow_action_mark *)
9454                                (actions->conf))->id);
9455                         MLX5_ASSERT(!handle->dvh.rix_tag);
9456                         if (flow_dv_tag_resource_register(dev, tag_be,
9457                                                           dev_flow, error))
9458                                 return -rte_errno;
9459                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9460                         dev_flow->dv.actions[actions_n++] =
9461                                         dev_flow->dv.tag_resource->action;
9462                         break;
9463                 case RTE_FLOW_ACTION_TYPE_SET_META:
9464                         if (flow_dv_convert_action_set_meta
9465                                 (dev, mhdr_res, attr,
9466                                  (const struct rte_flow_action_set_meta *)
9467                                   actions->conf, error))
9468                                 return -rte_errno;
9469                         action_flags |= MLX5_FLOW_ACTION_SET_META;
9470                         break;
9471                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
9472                         if (flow_dv_convert_action_set_tag
9473                                 (dev, mhdr_res,
9474                                  (const struct rte_flow_action_set_tag *)
9475                                   actions->conf, error))
9476                                 return -rte_errno;
9477                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9478                         break;
9479                 case RTE_FLOW_ACTION_TYPE_DROP:
9480                         action_flags |= MLX5_FLOW_ACTION_DROP;
9481                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
9482                         break;
9483                 case RTE_FLOW_ACTION_TYPE_QUEUE:
9484                         queue = actions->conf;
9485                         rss_desc->queue_num = 1;
9486                         rss_desc->queue[0] = queue->index;
9487                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
9488                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9489                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
9490                         num_of_dest++;
9491                         break;
9492                 case RTE_FLOW_ACTION_TYPE_RSS:
9493                         rss = actions->conf;
9494                         memcpy(rss_desc->queue, rss->queue,
9495                                rss->queue_num * sizeof(uint16_t));
9496                         rss_desc->queue_num = rss->queue_num;
9497                         /* NULL RSS key indicates default RSS key. */
9498                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
9499                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
9500                         /*
9501                          * rss->level and rss.types should be set in advance
9502                          * when expanding items for RSS.
9503                          */
9504                         action_flags |= MLX5_FLOW_ACTION_RSS;
9505                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9506                         break;
9507                 case RTE_FLOW_ACTION_TYPE_AGE:
9508                 case RTE_FLOW_ACTION_TYPE_COUNT:
9509                         if (!dev_conf->devx) {
9510                                 return rte_flow_error_set
9511                                               (error, ENOTSUP,
9512                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9513                                                NULL,
9514                                                "count action not supported");
9515                         }
9516                         /* Save information first, will apply later. */
9517                         if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT)
9518                                 count = action->conf;
9519                         else
9520                                 age = action->conf;
9521                         action_flags |= MLX5_FLOW_ACTION_COUNT;
9522                         break;
9523                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
9524                         dev_flow->dv.actions[actions_n++] =
9525                                                 priv->sh->pop_vlan_action;
9526                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
9527                         break;
9528                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
9529                         if (!(action_flags &
9530                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
9531                                 flow_dev_get_vlan_info_from_items(items, &vlan);
9532                         vlan.eth_proto = rte_be_to_cpu_16
9533                              ((((const struct rte_flow_action_of_push_vlan *)
9534                                                    actions->conf)->ethertype));
9535                         found_action = mlx5_flow_find_action
9536                                         (actions + 1,
9537                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
9538                         if (found_action)
9539                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
9540                         found_action = mlx5_flow_find_action
9541                                         (actions + 1,
9542                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
9543                         if (found_action)
9544                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
9545                         if (flow_dv_create_action_push_vlan
9546                                             (dev, attr, &vlan, dev_flow, error))
9547                                 return -rte_errno;
9548                         dev_flow->dv.actions[actions_n++] =
9549                                         dev_flow->dv.push_vlan_res->action;
9550                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
9551                         break;
9552                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
9553                         /* of_vlan_push action handled this action */
9554                         MLX5_ASSERT(action_flags &
9555                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
9556                         break;
9557                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
9558                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
9559                                 break;
9560                         flow_dev_get_vlan_info_from_items(items, &vlan);
9561                         mlx5_update_vlan_vid_pcp(actions, &vlan);
9562                         /* If no VLAN push - this is a modify header action */
9563                         if (flow_dv_convert_action_modify_vlan_vid
9564                                                 (mhdr_res, actions, error))
9565                                 return -rte_errno;
9566                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
9567                         break;
9568                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
9569                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
9570                         if (flow_dv_create_action_l2_encap(dev, actions,
9571                                                            dev_flow,
9572                                                            attr->transfer,
9573                                                            error))
9574                                 return -rte_errno;
9575                         dev_flow->dv.actions[actions_n++] =
9576                                         dev_flow->dv.encap_decap->action;
9577                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9578                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
9579                                 sample_act->action_flags |=
9580                                                         MLX5_FLOW_ACTION_ENCAP;
9581                         break;
9582                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
9583                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
9584                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
9585                                                            attr->transfer,
9586                                                            error))
9587                                 return -rte_errno;
9588                         dev_flow->dv.actions[actions_n++] =
9589                                         dev_flow->dv.encap_decap->action;
9590                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9591                         break;
9592                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
9593                         /* Handle encap with preceding decap. */
9594                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
9595                                 if (flow_dv_create_action_raw_encap
9596                                         (dev, actions, dev_flow, attr, error))
9597                                         return -rte_errno;
9598                                 dev_flow->dv.actions[actions_n++] =
9599                                         dev_flow->dv.encap_decap->action;
9600                         } else {
9601                                 /* Handle encap without preceding decap. */
9602                                 if (flow_dv_create_action_l2_encap
9603                                     (dev, actions, dev_flow, attr->transfer,
9604                                      error))
9605                                         return -rte_errno;
9606                                 dev_flow->dv.actions[actions_n++] =
9607                                         dev_flow->dv.encap_decap->action;
9608                         }
9609                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9610                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
9611                                 sample_act->action_flags |=
9612                                                         MLX5_FLOW_ACTION_ENCAP;
9613                         break;
9614                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
9615                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
9616                                 ;
9617                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
9618                                 if (flow_dv_create_action_l2_decap
9619                                     (dev, dev_flow, attr->transfer, error))
9620                                         return -rte_errno;
9621                                 dev_flow->dv.actions[actions_n++] =
9622                                         dev_flow->dv.encap_decap->action;
9623                         }
9624                         /* If decap is followed by encap, handle it at encap. */
9625                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9626                         break;
9627                 case RTE_FLOW_ACTION_TYPE_JUMP:
9628                         jump_group = ((const struct rte_flow_action_jump *)
9629                                                         action->conf)->group;
9630                         grp_info.std_tbl_fix = 0;
9631                         ret = mlx5_flow_group_to_table(dev, tunnel,
9632                                                        jump_group,
9633                                                        &table,
9634                                                        grp_info, error);
9635                         if (ret)
9636                                 return ret;
9637                         tbl = flow_dv_tbl_resource_get(dev, table, attr->egress,
9638                                                        attr->transfer,
9639                                                        !!dev_flow->external,
9640                                                        tunnel, jump_group,
9641                                                        error);
9642                         if (!tbl)
9643                                 return rte_flow_error_set
9644                                                 (error, errno,
9645                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9646                                                  NULL,
9647                                                  "cannot create jump action.");
9648                         if (flow_dv_jump_tbl_resource_register
9649                             (dev, tbl, dev_flow, error)) {
9650                                 flow_dv_tbl_resource_release(dev, tbl);
9651                                 return rte_flow_error_set
9652                                                 (error, errno,
9653                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9654                                                  NULL,
9655                                                  "cannot create jump action.");
9656                         }
9657                         dev_flow->dv.actions[actions_n++] =
9658                                         dev_flow->dv.jump->action;
9659                         action_flags |= MLX5_FLOW_ACTION_JUMP;
9660                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
9661                         break;
9662                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
9663                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
9664                         if (flow_dv_convert_action_modify_mac
9665                                         (mhdr_res, actions, error))
9666                                 return -rte_errno;
9667                         action_flags |= actions->type ==
9668                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
9669                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
9670                                         MLX5_FLOW_ACTION_SET_MAC_DST;
9671                         break;
9672                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
9673                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
9674                         if (flow_dv_convert_action_modify_ipv4
9675                                         (mhdr_res, actions, error))
9676                                 return -rte_errno;
9677                         action_flags |= actions->type ==
9678                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
9679                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
9680                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
9681                         break;
9682                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
9683                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
9684                         if (flow_dv_convert_action_modify_ipv6
9685                                         (mhdr_res, actions, error))
9686                                 return -rte_errno;
9687                         action_flags |= actions->type ==
9688                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
9689                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
9690                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
9691                         break;
9692                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
9693                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
9694                         if (flow_dv_convert_action_modify_tp
9695                                         (mhdr_res, actions, items,
9696                                          &flow_attr, dev_flow, !!(action_flags &
9697                                          MLX5_FLOW_ACTION_DECAP), error))
9698                                 return -rte_errno;
9699                         action_flags |= actions->type ==
9700                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
9701                                         MLX5_FLOW_ACTION_SET_TP_SRC :
9702                                         MLX5_FLOW_ACTION_SET_TP_DST;
9703                         break;
9704                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
9705                         if (flow_dv_convert_action_modify_dec_ttl
9706                                         (mhdr_res, items, &flow_attr, dev_flow,
9707                                          !!(action_flags &
9708                                          MLX5_FLOW_ACTION_DECAP), error))
9709                                 return -rte_errno;
9710                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
9711                         break;
9712                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
9713                         if (flow_dv_convert_action_modify_ttl
9714                                         (mhdr_res, actions, items, &flow_attr,
9715                                          dev_flow, !!(action_flags &
9716                                          MLX5_FLOW_ACTION_DECAP), error))
9717                                 return -rte_errno;
9718                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
9719                         break;
9720                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
9721                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
9722                         if (flow_dv_convert_action_modify_tcp_seq
9723                                         (mhdr_res, actions, error))
9724                                 return -rte_errno;
9725                         action_flags |= actions->type ==
9726                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
9727                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
9728                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
9729                         break;
9730
9731                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
9732                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
9733                         if (flow_dv_convert_action_modify_tcp_ack
9734                                         (mhdr_res, actions, error))
9735                                 return -rte_errno;
9736                         action_flags |= actions->type ==
9737                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
9738                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
9739                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
9740                         break;
9741                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
9742                         if (flow_dv_convert_action_set_reg
9743                                         (mhdr_res, actions, error))
9744                                 return -rte_errno;
9745                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9746                         break;
9747                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
9748                         if (flow_dv_convert_action_copy_mreg
9749                                         (dev, mhdr_res, actions, error))
9750                                 return -rte_errno;
9751                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9752                         break;
9753                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
9754                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
9755                         dev_flow->handle->fate_action =
9756                                         MLX5_FLOW_FATE_DEFAULT_MISS;
9757                         break;
9758                 case RTE_FLOW_ACTION_TYPE_METER:
9759                         mtr = actions->conf;
9760                         if (!flow->meter) {
9761                                 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
9762                                                             attr, error);
9763                                 if (!fm)
9764                                         return rte_flow_error_set(error,
9765                                                 rte_errno,
9766                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9767                                                 NULL,
9768                                                 "meter not found "
9769                                                 "or invalid parameters");
9770                                 flow->meter = fm->idx;
9771                         }
9772                         /* Set the meter action. */
9773                         if (!fm) {
9774                                 fm = mlx5_ipool_get(priv->sh->ipool
9775                                                 [MLX5_IPOOL_MTR], flow->meter);
9776                                 if (!fm)
9777                                         return rte_flow_error_set(error,
9778                                                 rte_errno,
9779                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9780                                                 NULL,
9781                                                 "meter not found "
9782                                                 "or invalid parameters");
9783                         }
9784                         dev_flow->dv.actions[actions_n++] =
9785                                 fm->mfts->meter_action;
9786                         action_flags |= MLX5_FLOW_ACTION_METER;
9787                         break;
9788                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
9789                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
9790                                                               actions, error))
9791                                 return -rte_errno;
9792                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
9793                         break;
9794                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
9795                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
9796                                                               actions, error))
9797                                 return -rte_errno;
9798                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
9799                         break;
9800                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
9801                         sample_act_pos = actions_n;
9802                         ret = flow_dv_translate_action_sample(dev,
9803                                                               actions,
9804                                                               dev_flow, attr,
9805                                                               &num_of_dest,
9806                                                               sample_actions,
9807                                                               &sample_res,
9808                                                               error);
9809                         if (ret < 0)
9810                                 return ret;
9811                         actions_n++;
9812                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
9813                         /* put encap action into group if work with port id */
9814                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
9815                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
9816                                 sample_act->action_flags |=
9817                                                         MLX5_FLOW_ACTION_ENCAP;
9818                         break;
9819                 case RTE_FLOW_ACTION_TYPE_END:
9820                         actions_end = true;
9821                         if (mhdr_res->actions_num) {
9822                                 /* create modify action if needed. */
9823                                 if (flow_dv_modify_hdr_resource_register
9824                                         (dev, mhdr_res, dev_flow, error))
9825                                         return -rte_errno;
9826                                 dev_flow->dv.actions[modify_action_position] =
9827                                         handle->dvh.modify_hdr->action;
9828                         }
9829                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
9830                                 flow->counter =
9831                                         flow_dv_translate_create_counter(dev,
9832                                                 dev_flow, count, age);
9833
9834                                 if (!flow->counter)
9835                                         return rte_flow_error_set
9836                                                 (error, rte_errno,
9837                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9838                                                 NULL,
9839                                                 "cannot create counter"
9840                                                 " object.");
9841                                 dev_flow->dv.actions[actions_n] =
9842                                           (flow_dv_counter_get_by_idx(dev,
9843                                           flow->counter, NULL))->action;
9844                                 actions_n++;
9845                         }
9846                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
9847                                 ret = flow_dv_create_action_sample(dev,
9848                                                           dev_flow, attr,
9849                                                           num_of_dest,
9850                                                           &sample_res,
9851                                                           &mdest_res,
9852                                                           sample_actions,
9853                                                           action_flags,
9854                                                           error);
9855                                 if (ret < 0)
9856                                         return rte_flow_error_set
9857                                                 (error, rte_errno,
9858                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9859                                                 NULL,
9860                                                 "cannot create sample action");
9861                                 if (num_of_dest > 1) {
9862                                         dev_flow->dv.actions[sample_act_pos] =
9863                                         dev_flow->dv.dest_array_res->action;
9864                                 } else {
9865                                         dev_flow->dv.actions[sample_act_pos] =
9866                                         dev_flow->dv.sample_res->verbs_action;
9867                                 }
9868                         }
9869                         break;
9870                 default:
9871                         break;
9872                 }
9873                 if (mhdr_res->actions_num &&
9874                     modify_action_position == UINT32_MAX)
9875                         modify_action_position = actions_n++;
9876         }
9877         /*
9878          * For multiple destination (sample action with ratio=1), the encap
9879          * action and port id action will be combined into group action.
9880          * So need remove the original these actions in the flow and only
9881          * use the sample action instead of.
9882          */
9883         if (num_of_dest > 1 && sample_act->dr_port_id_action) {
9884                 int i;
9885                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
9886
9887                 for (i = 0; i < actions_n; i++) {
9888                         if ((sample_act->dr_encap_action &&
9889                                 sample_act->dr_encap_action ==
9890                                 dev_flow->dv.actions[i]) ||
9891                                 (sample_act->dr_port_id_action &&
9892                                 sample_act->dr_port_id_action ==
9893                                 dev_flow->dv.actions[i]))
9894                                 continue;
9895                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
9896                 }
9897                 memcpy((void *)dev_flow->dv.actions,
9898                                 (void *)temp_actions,
9899                                 tmp_actions_n * sizeof(void *));
9900                 actions_n = tmp_actions_n;
9901         }
9902         dev_flow->dv.actions_n = actions_n;
9903         dev_flow->act_flags = action_flags;
9904         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
9905                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
9906                 int item_type = items->type;
9907
9908                 if (!mlx5_flow_os_item_supported(item_type))
9909                         return rte_flow_error_set(error, ENOTSUP,
9910                                                   RTE_FLOW_ERROR_TYPE_ITEM,
9911                                                   NULL, "item not supported");
9912                 switch (item_type) {
9913                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
9914                         flow_dv_translate_item_port_id(dev, match_mask,
9915                                                        match_value, items);
9916                         last_item = MLX5_FLOW_ITEM_PORT_ID;
9917                         break;
9918                 case RTE_FLOW_ITEM_TYPE_ETH:
9919                         flow_dv_translate_item_eth(match_mask, match_value,
9920                                                    items, tunnel,
9921                                                    dev_flow->dv.group);
9922                         matcher.priority = action_flags &
9923                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
9924                                         !dev_flow->external ?
9925                                         MLX5_PRIORITY_MAP_L3 :
9926                                         MLX5_PRIORITY_MAP_L2;
9927                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
9928                                              MLX5_FLOW_LAYER_OUTER_L2;
9929                         break;
9930                 case RTE_FLOW_ITEM_TYPE_VLAN:
9931                         flow_dv_translate_item_vlan(dev_flow,
9932                                                     match_mask, match_value,
9933                                                     items, tunnel,
9934                                                     dev_flow->dv.group);
9935                         matcher.priority = MLX5_PRIORITY_MAP_L2;
9936                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
9937                                               MLX5_FLOW_LAYER_INNER_VLAN) :
9938                                              (MLX5_FLOW_LAYER_OUTER_L2 |
9939                                               MLX5_FLOW_LAYER_OUTER_VLAN);
9940                         break;
9941                 case RTE_FLOW_ITEM_TYPE_IPV4:
9942                         mlx5_flow_tunnel_ip_check(items, next_protocol,
9943                                                   &item_flags, &tunnel);
9944                         flow_dv_translate_item_ipv4(match_mask, match_value,
9945                                                     items, tunnel,
9946                                                     dev_flow->dv.group);
9947                         matcher.priority = MLX5_PRIORITY_MAP_L3;
9948                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
9949                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
9950                         if (items->mask != NULL &&
9951                             ((const struct rte_flow_item_ipv4 *)
9952                              items->mask)->hdr.next_proto_id) {
9953                                 next_protocol =
9954                                         ((const struct rte_flow_item_ipv4 *)
9955                                          (items->spec))->hdr.next_proto_id;
9956                                 next_protocol &=
9957                                         ((const struct rte_flow_item_ipv4 *)
9958                                          (items->mask))->hdr.next_proto_id;
9959                         } else {
9960                                 /* Reset for inner layer. */
9961                                 next_protocol = 0xff;
9962                         }
9963                         break;
9964                 case RTE_FLOW_ITEM_TYPE_IPV6:
9965                         mlx5_flow_tunnel_ip_check(items, next_protocol,
9966                                                   &item_flags, &tunnel);
9967                         flow_dv_translate_item_ipv6(match_mask, match_value,
9968                                                     items, tunnel,
9969                                                     dev_flow->dv.group);
9970                         matcher.priority = MLX5_PRIORITY_MAP_L3;
9971                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
9972                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
9973                         if (items->mask != NULL &&
9974                             ((const struct rte_flow_item_ipv6 *)
9975                              items->mask)->hdr.proto) {
9976                                 next_protocol =
9977                                         ((const struct rte_flow_item_ipv6 *)
9978                                          items->spec)->hdr.proto;
9979                                 next_protocol &=
9980                                         ((const struct rte_flow_item_ipv6 *)
9981                                          items->mask)->hdr.proto;
9982                         } else {
9983                                 /* Reset for inner layer. */
9984                                 next_protocol = 0xff;
9985                         }
9986                         break;
9987                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
9988                         flow_dv_translate_item_ipv6_frag_ext(match_mask,
9989                                                              match_value,
9990                                                              items, tunnel);
9991                         last_item = tunnel ?
9992                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
9993                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
9994                         if (items->mask != NULL &&
9995                             ((const struct rte_flow_item_ipv6_frag_ext *)
9996                              items->mask)->hdr.next_header) {
9997                                 next_protocol =
9998                                 ((const struct rte_flow_item_ipv6_frag_ext *)
9999                                  items->spec)->hdr.next_header;
10000                                 next_protocol &=
10001                                 ((const struct rte_flow_item_ipv6_frag_ext *)
10002                                  items->mask)->hdr.next_header;
10003                         } else {
10004                                 /* Reset for inner layer. */
10005                                 next_protocol = 0xff;
10006                         }
10007                         break;
10008                 case RTE_FLOW_ITEM_TYPE_TCP:
10009                         flow_dv_translate_item_tcp(match_mask, match_value,
10010                                                    items, tunnel);
10011                         matcher.priority = MLX5_PRIORITY_MAP_L4;
10012                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
10013                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
10014                         break;
10015                 case RTE_FLOW_ITEM_TYPE_UDP:
10016                         flow_dv_translate_item_udp(match_mask, match_value,
10017                                                    items, tunnel);
10018                         matcher.priority = MLX5_PRIORITY_MAP_L4;
10019                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
10020                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
10021                         break;
10022                 case RTE_FLOW_ITEM_TYPE_GRE:
10023                         flow_dv_translate_item_gre(match_mask, match_value,
10024                                                    items, tunnel);
10025                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10026                         last_item = MLX5_FLOW_LAYER_GRE;
10027                         break;
10028                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
10029                         flow_dv_translate_item_gre_key(match_mask,
10030                                                        match_value, items);
10031                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
10032                         break;
10033                 case RTE_FLOW_ITEM_TYPE_NVGRE:
10034                         flow_dv_translate_item_nvgre(match_mask, match_value,
10035                                                      items, tunnel);
10036                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10037                         last_item = MLX5_FLOW_LAYER_GRE;
10038                         break;
10039                 case RTE_FLOW_ITEM_TYPE_VXLAN:
10040                         flow_dv_translate_item_vxlan(match_mask, match_value,
10041                                                      items, tunnel);
10042                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10043                         last_item = MLX5_FLOW_LAYER_VXLAN;
10044                         break;
10045                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
10046                         flow_dv_translate_item_vxlan_gpe(match_mask,
10047                                                          match_value, items,
10048                                                          tunnel);
10049                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10050                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
10051                         break;
10052                 case RTE_FLOW_ITEM_TYPE_GENEVE:
10053                         flow_dv_translate_item_geneve(match_mask, match_value,
10054                                                       items, tunnel);
10055                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10056                         last_item = MLX5_FLOW_LAYER_GENEVE;
10057                         break;
10058                 case RTE_FLOW_ITEM_TYPE_MPLS:
10059                         flow_dv_translate_item_mpls(match_mask, match_value,
10060                                                     items, last_item, tunnel);
10061                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10062                         last_item = MLX5_FLOW_LAYER_MPLS;
10063                         break;
10064                 case RTE_FLOW_ITEM_TYPE_MARK:
10065                         flow_dv_translate_item_mark(dev, match_mask,
10066                                                     match_value, items);
10067                         last_item = MLX5_FLOW_ITEM_MARK;
10068                         break;
10069                 case RTE_FLOW_ITEM_TYPE_META:
10070                         flow_dv_translate_item_meta(dev, match_mask,
10071                                                     match_value, attr, items);
10072                         last_item = MLX5_FLOW_ITEM_METADATA;
10073                         break;
10074                 case RTE_FLOW_ITEM_TYPE_ICMP:
10075                         flow_dv_translate_item_icmp(match_mask, match_value,
10076                                                     items, tunnel);
10077                         last_item = MLX5_FLOW_LAYER_ICMP;
10078                         break;
10079                 case RTE_FLOW_ITEM_TYPE_ICMP6:
10080                         flow_dv_translate_item_icmp6(match_mask, match_value,
10081                                                       items, tunnel);
10082                         last_item = MLX5_FLOW_LAYER_ICMP6;
10083                         break;
10084                 case RTE_FLOW_ITEM_TYPE_TAG:
10085                         flow_dv_translate_item_tag(dev, match_mask,
10086                                                    match_value, items);
10087                         last_item = MLX5_FLOW_ITEM_TAG;
10088                         break;
10089                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
10090                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
10091                                                         match_value, items);
10092                         last_item = MLX5_FLOW_ITEM_TAG;
10093                         break;
10094                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
10095                         flow_dv_translate_item_tx_queue(dev, match_mask,
10096                                                         match_value,
10097                                                         items);
10098                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
10099                         break;
10100                 case RTE_FLOW_ITEM_TYPE_GTP:
10101                         flow_dv_translate_item_gtp(match_mask, match_value,
10102                                                    items, tunnel);
10103                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10104                         last_item = MLX5_FLOW_LAYER_GTP;
10105                         break;
10106                 case RTE_FLOW_ITEM_TYPE_ECPRI:
10107                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
10108                                 /* Create it only the first time to be used. */
10109                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
10110                                 if (ret)
10111                                         return rte_flow_error_set
10112                                                 (error, -ret,
10113                                                 RTE_FLOW_ERROR_TYPE_ITEM,
10114                                                 NULL,
10115                                                 "cannot create eCPRI parser");
10116                         }
10117                         /* Adjust the length matcher and device flow value. */
10118                         matcher.mask.size = MLX5_ST_SZ_BYTES(fte_match_param);
10119                         dev_flow->dv.value.size =
10120                                         MLX5_ST_SZ_BYTES(fte_match_param);
10121                         flow_dv_translate_item_ecpri(dev, match_mask,
10122                                                      match_value, items);
10123                         /* No other protocol should follow eCPRI layer. */
10124                         last_item = MLX5_FLOW_LAYER_ECPRI;
10125                         break;
10126                 default:
10127                         break;
10128                 }
10129                 item_flags |= last_item;
10130         }
10131         /*
10132          * When E-Switch mode is enabled, we have two cases where we need to
10133          * set the source port manually.
10134          * The first one, is in case of Nic steering rule, and the second is
10135          * E-Switch rule where no port_id item was found. In both cases
10136          * the source port is set according the current port in use.
10137          */
10138         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
10139             (priv->representor || priv->master)) {
10140                 if (flow_dv_translate_item_port_id(dev, match_mask,
10141                                                    match_value, NULL))
10142                         return -rte_errno;
10143         }
10144 #ifdef RTE_LIBRTE_MLX5_DEBUG
10145         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
10146                                               dev_flow->dv.value.buf));
10147 #endif
10148         /*
10149          * Layers may be already initialized from prefix flow if this dev_flow
10150          * is the suffix flow.
10151          */
10152         handle->layers |= item_flags;
10153         if (action_flags & MLX5_FLOW_ACTION_RSS)
10154                 flow_dv_hashfields_set(dev_flow, rss_desc);
10155         /* Register matcher. */
10156         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
10157                                     matcher.mask.size);
10158         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
10159                                                      matcher.priority);
10160         /* reserved field no needs to be set to 0 here. */
10161         tbl_key.domain = attr->transfer;
10162         tbl_key.direction = attr->egress;
10163         tbl_key.table_id = dev_flow->dv.group;
10164         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
10165                 return -rte_errno;
10166         return 0;
10167 }
10168
10169 /**
10170  * Set hash RX queue by hash fields (see enum ibv_rx_hash_fields)
10171  * and tunnel.
10172  *
10173  * @param[in, out] action
10174  *   Shred RSS action holding hash RX queue objects.
10175  * @param[in] hash_fields
10176  *   Defines combination of packet fields to participate in RX hash.
10177  * @param[in] tunnel
10178  *   Tunnel type
10179  * @param[in] hrxq_idx
10180  *   Hash RX queue index to set.
10181  *
10182  * @return
10183  *   0 on success, otherwise negative errno value.
10184  */
10185 static int
10186 __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action,
10187                               const uint64_t hash_fields,
10188                               const int tunnel,
10189                               uint32_t hrxq_idx)
10190 {
10191         uint32_t *hrxqs = tunnel ? action->hrxq : action->hrxq_tunnel;
10192
10193         switch (hash_fields & ~IBV_RX_HASH_INNER) {
10194         case MLX5_RSS_HASH_IPV4:
10195                 hrxqs[0] = hrxq_idx;
10196                 return 0;
10197         case MLX5_RSS_HASH_IPV4_TCP:
10198                 hrxqs[1] = hrxq_idx;
10199                 return 0;
10200         case MLX5_RSS_HASH_IPV4_UDP:
10201                 hrxqs[2] = hrxq_idx;
10202                 return 0;
10203         case MLX5_RSS_HASH_IPV6:
10204                 hrxqs[3] = hrxq_idx;
10205                 return 0;
10206         case MLX5_RSS_HASH_IPV6_TCP:
10207                 hrxqs[4] = hrxq_idx;
10208                 return 0;
10209         case MLX5_RSS_HASH_IPV6_UDP:
10210                 hrxqs[5] = hrxq_idx;
10211                 return 0;
10212         case MLX5_RSS_HASH_NONE:
10213                 hrxqs[6] = hrxq_idx;
10214                 return 0;
10215         default:
10216                 return -1;
10217         }
10218 }
10219
10220 /**
10221  * Look up for hash RX queue by hash fields (see enum ibv_rx_hash_fields)
10222  * and tunnel.
10223  *
10224  * @param[in] action
10225  *   Shred RSS action holding hash RX queue objects.
10226  * @param[in] hash_fields
10227  *   Defines combination of packet fields to participate in RX hash.
10228  * @param[in] tunnel
10229  *   Tunnel type
10230  *
10231  * @return
10232  *   Valid hash RX queue index, otherwise 0.
10233  */
10234 static uint32_t
10235 __flow_dv_action_rss_hrxq_lookup(const struct mlx5_shared_action_rss *action,
10236                                  const uint64_t hash_fields,
10237                                  const int tunnel)
10238 {
10239         const uint32_t *hrxqs = tunnel ? action->hrxq : action->hrxq_tunnel;
10240
10241         switch (hash_fields & ~IBV_RX_HASH_INNER) {
10242         case MLX5_RSS_HASH_IPV4:
10243                 return hrxqs[0];
10244         case MLX5_RSS_HASH_IPV4_TCP:
10245                 return hrxqs[1];
10246         case MLX5_RSS_HASH_IPV4_UDP:
10247                 return hrxqs[2];
10248         case MLX5_RSS_HASH_IPV6:
10249                 return hrxqs[3];
10250         case MLX5_RSS_HASH_IPV6_TCP:
10251                 return hrxqs[4];
10252         case MLX5_RSS_HASH_IPV6_UDP:
10253                 return hrxqs[5];
10254         case MLX5_RSS_HASH_NONE:
10255                 return hrxqs[6];
10256         default:
10257                 return 0;
10258         }
10259 }
10260
10261 /**
10262  * Retrieves hash RX queue suitable for the *flow*.
10263  * If shared action configured for *flow* suitable hash RX queue will be
10264  * retrieved from attached shared action.
10265  *
10266  * @param[in] flow
10267  *   Shred RSS action holding hash RX queue objects.
10268  * @param[in] dev_flow
10269  *   Pointer to the sub flow.
10270  * @param[out] hrxq
10271  *   Pointer to retrieved hash RX queue object.
10272  *
10273  * @return
10274  *   Valid hash RX queue index, otherwise 0 and rte_errno is set.
10275  */
10276 static uint32_t
10277 __flow_dv_rss_get_hrxq(struct rte_eth_dev *dev, struct rte_flow *flow,
10278                            struct mlx5_flow *dev_flow,
10279                            struct mlx5_hrxq **hrxq)
10280 {
10281         struct mlx5_priv *priv = dev->data->dev_private;
10282         uint32_t hrxq_idx;
10283
10284         if (flow->shared_rss) {
10285                 hrxq_idx = __flow_dv_action_rss_hrxq_lookup
10286                                 (flow->shared_rss, dev_flow->hash_fields,
10287                                  !!(dev_flow->handle->layers &
10288                                     MLX5_FLOW_LAYER_TUNNEL));
10289                 if (hrxq_idx) {
10290                         *hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
10291                                                hrxq_idx);
10292                         __atomic_fetch_add(&(*hrxq)->refcnt, 1,
10293                                            __ATOMIC_RELAXED);
10294                 }
10295         } else {
10296                 struct mlx5_flow_rss_desc *rss_desc =
10297                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
10298                                 [!!priv->flow_nested_idx];
10299
10300                 MLX5_ASSERT(rss_desc->queue_num);
10301                 hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
10302                                          MLX5_RSS_HASH_KEY_LEN,
10303                                          dev_flow->hash_fields,
10304                                          rss_desc->queue, rss_desc->queue_num);
10305                 if (!hrxq_idx) {
10306                         hrxq_idx = mlx5_hrxq_new(dev,
10307                                                  rss_desc->key,
10308                                                  MLX5_RSS_HASH_KEY_LEN,
10309                                                  dev_flow->hash_fields,
10310                                                  rss_desc->queue,
10311                                                  rss_desc->queue_num,
10312                                                  !!(dev_flow->handle->layers &
10313                                                  MLX5_FLOW_LAYER_TUNNEL),
10314                                                  false);
10315                 }
10316                 *hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
10317                                        hrxq_idx);
10318         }
10319         return hrxq_idx;
10320 }
10321
10322 /**
10323  * Apply the flow to the NIC, lock free,
10324  * (mutex should be acquired by caller).
10325  *
10326  * @param[in] dev
10327  *   Pointer to the Ethernet device structure.
10328  * @param[in, out] flow
10329  *   Pointer to flow structure.
10330  * @param[out] error
10331  *   Pointer to error structure.
10332  *
10333  * @return
10334  *   0 on success, a negative errno value otherwise and rte_errno is set.
10335  */
10336 static int
10337 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
10338                 struct rte_flow_error *error)
10339 {
10340         struct mlx5_flow_dv_workspace *dv;
10341         struct mlx5_flow_handle *dh;
10342         struct mlx5_flow_handle_dv *dv_h;
10343         struct mlx5_flow *dev_flow;
10344         struct mlx5_priv *priv = dev->data->dev_private;
10345         uint32_t handle_idx;
10346         int n;
10347         int err;
10348         int idx;
10349
10350         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
10351                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
10352                 dv = &dev_flow->dv;
10353                 dh = dev_flow->handle;
10354                 dv_h = &dh->dvh;
10355                 n = dv->actions_n;
10356                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
10357                         if (dv->transfer) {
10358                                 dv->actions[n++] = priv->sh->esw_drop_action;
10359                         } else {
10360                                 struct mlx5_hrxq *drop_hrxq;
10361                                 drop_hrxq = mlx5_drop_action_create(dev);
10362                                 if (!drop_hrxq) {
10363                                         rte_flow_error_set
10364                                                 (error, errno,
10365                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10366                                                  NULL,
10367                                                  "cannot get drop hash queue");
10368                                         goto error;
10369                                 }
10370                                 /*
10371                                  * Drop queues will be released by the specify
10372                                  * mlx5_drop_action_destroy() function. Assign
10373                                  * the special index to hrxq to mark the queue
10374                                  * has been allocated.
10375                                  */
10376                                 dh->rix_hrxq = UINT32_MAX;
10377                                 dv->actions[n++] = drop_hrxq->action;
10378                         }
10379                 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
10380                            !dv_h->rix_sample && !dv_h->rix_dest_array) {
10381                         struct mlx5_hrxq *hrxq = NULL;
10382                         uint32_t hrxq_idx = __flow_dv_rss_get_hrxq
10383                                                 (dev, flow, dev_flow, &hrxq);
10384
10385                         if (!hrxq) {
10386                                 rte_flow_error_set
10387                                         (error, rte_errno,
10388                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10389                                          "cannot get hash queue");
10390                                 goto error;
10391                         }
10392                         dh->rix_hrxq = hrxq_idx;
10393                         dv->actions[n++] = hrxq->action;
10394                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
10395                         if (flow_dv_default_miss_resource_register
10396                                         (dev, error)) {
10397                                 rte_flow_error_set
10398                                         (error, rte_errno,
10399                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10400                                          "cannot create default miss resource");
10401                                 goto error_default_miss;
10402                         }
10403                         dh->rix_default_fate =  MLX5_FLOW_FATE_DEFAULT_MISS;
10404                         dv->actions[n++] = priv->sh->default_miss.action;
10405                 }
10406                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
10407                                                (void *)&dv->value, n,
10408                                                dv->actions, &dh->drv_flow);
10409                 if (err) {
10410                         rte_flow_error_set(error, errno,
10411                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10412                                            NULL,
10413                                            "hardware refuses to create flow");
10414                         goto error;
10415                 }
10416                 if (priv->vmwa_context &&
10417                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
10418                         /*
10419                          * The rule contains the VLAN pattern.
10420                          * For VF we are going to create VLAN
10421                          * interface to make hypervisor set correct
10422                          * e-Switch vport context.
10423                          */
10424                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
10425                 }
10426         }
10427         return 0;
10428 error:
10429         if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
10430                 flow_dv_default_miss_resource_release(dev);
10431 error_default_miss:
10432         err = rte_errno; /* Save rte_errno before cleanup. */
10433         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
10434                        handle_idx, dh, next) {
10435                 /* hrxq is union, don't clear it if the flag is not set. */
10436                 if (dh->rix_hrxq) {
10437                         if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
10438                                 mlx5_drop_action_destroy(dev);
10439                                 dh->rix_hrxq = 0;
10440                         } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
10441                                 mlx5_hrxq_release(dev, dh->rix_hrxq);
10442                                 dh->rix_hrxq = 0;
10443                         }
10444                 }
10445                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
10446                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
10447         }
10448         rte_errno = err; /* Restore rte_errno. */
10449         return -rte_errno;
10450 }
10451
10452 /**
10453  * Release the flow matcher.
10454  *
10455  * @param dev
10456  *   Pointer to Ethernet device.
10457  * @param handle
10458  *   Pointer to mlx5_flow_handle.
10459  *
10460  * @return
10461  *   1 while a reference on it exists, 0 when freed.
10462  */
10463 static int
10464 flow_dv_matcher_release(struct rte_eth_dev *dev,
10465                         struct mlx5_flow_handle *handle)
10466 {
10467         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
10468
10469         MLX5_ASSERT(matcher->matcher_object);
10470         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
10471                 dev->data->port_id, (void *)matcher,
10472                 __atomic_load_n(&matcher->refcnt, __ATOMIC_RELAXED));
10473         if (__atomic_sub_fetch(&matcher->refcnt, 1, __ATOMIC_RELAXED) == 0) {
10474                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10475                            (matcher->matcher_object));
10476                 LIST_REMOVE(matcher, next);
10477                 /* table ref-- in release interface. */
10478                 flow_dv_tbl_resource_release(dev, matcher->tbl);
10479                 mlx5_free(matcher);
10480                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
10481                         dev->data->port_id, (void *)matcher);
10482                 return 0;
10483         }
10484         return 1;
10485 }
10486
10487 /**
10488  * Release an encap/decap resource.
10489  *
10490  * @param dev
10491  *   Pointer to Ethernet device.
10492  * @param encap_decap_idx
10493  *   Index of encap decap resource.
10494  *
10495  * @return
10496  *   1 while a reference on it exists, 0 when freed.
10497  */
10498 static int
10499 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
10500                                      uint32_t encap_decap_idx)
10501 {
10502         struct mlx5_priv *priv = dev->data->dev_private;
10503         uint32_t idx = encap_decap_idx;
10504         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
10505
10506         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
10507                          idx);
10508         if (!cache_resource)
10509                 return 0;
10510         MLX5_ASSERT(cache_resource->action);
10511         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
10512                 (void *)cache_resource,
10513                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10514         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10515                                __ATOMIC_RELAXED) == 0) {
10516                 claim_zero(mlx5_flow_os_destroy_flow_action
10517                                                 (cache_resource->action));
10518                 mlx5_hlist_remove(priv->sh->encaps_decaps,
10519                                   &cache_resource->entry);
10520                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
10521                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
10522                         (void *)cache_resource);
10523                 return 0;
10524         }
10525         return 1;
10526 }
10527
10528 /**
10529  * Release an jump to table action resource.
10530  *
10531  * @param dev
10532  *   Pointer to Ethernet device.
10533  * @param handle
10534  *   Pointer to mlx5_flow_handle.
10535  *
10536  * @return
10537  *   1 while a reference on it exists, 0 when freed.
10538  */
10539 static int
10540 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
10541                                   struct mlx5_flow_handle *handle)
10542 {
10543         struct mlx5_priv *priv = dev->data->dev_private;
10544         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
10545         struct mlx5_flow_tbl_data_entry *tbl_data;
10546
10547         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
10548                              handle->rix_jump);
10549         if (!tbl_data)
10550                 return 0;
10551         cache_resource = &tbl_data->jump;
10552         MLX5_ASSERT(cache_resource->action);
10553         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
10554                 (void *)cache_resource,
10555                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10556         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10557                                __ATOMIC_RELAXED) == 0) {
10558                 claim_zero(mlx5_flow_os_destroy_flow_action
10559                                                 (cache_resource->action));
10560                 /* jump action memory free is inside the table release. */
10561                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
10562                 DRV_LOG(DEBUG, "jump table resource %p: removed",
10563                         (void *)cache_resource);
10564                 return 0;
10565         }
10566         return 1;
10567 }
10568
10569 /**
10570  * Release a default miss resource.
10571  *
10572  * @param dev
10573  *   Pointer to Ethernet device.
10574  * @return
10575  *   1 while a reference on it exists, 0 when freed.
10576  */
10577 static int
10578 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
10579 {
10580         struct mlx5_priv *priv = dev->data->dev_private;
10581         struct mlx5_dev_ctx_shared *sh = priv->sh;
10582         struct mlx5_flow_default_miss_resource *cache_resource =
10583                         &sh->default_miss;
10584
10585         MLX5_ASSERT(cache_resource->action);
10586         DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--",
10587                         (void *)cache_resource->action,
10588                         __atomic_load_n(&cache_resource->refcnt,
10589                                         __ATOMIC_RELAXED));
10590         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10591                                __ATOMIC_RELAXED) == 0) {
10592                 claim_zero(mlx5_glue->destroy_flow_action
10593                                 (cache_resource->action));
10594                 DRV_LOG(DEBUG, "default miss resource %p: removed",
10595                                 (void *)cache_resource->action);
10596                 return 0;
10597         }
10598         return 1;
10599 }
10600
10601 /**
10602  * Release a modify-header resource.
10603  *
10604  * @param dev
10605  *   Pointer to Ethernet device.
10606  * @param handle
10607  *   Pointer to mlx5_flow_handle.
10608  *
10609  * @return
10610  *   1 while a reference on it exists, 0 when freed.
10611  */
10612 static int
10613 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
10614                                     struct mlx5_flow_handle *handle)
10615 {
10616         struct mlx5_priv *priv = dev->data->dev_private;
10617         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
10618                                                         handle->dvh.modify_hdr;
10619
10620         MLX5_ASSERT(cache_resource->action);
10621         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
10622                 (void *)cache_resource,
10623                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10624         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10625                                 __ATOMIC_RELAXED) == 0) {
10626                 claim_zero(mlx5_flow_os_destroy_flow_action
10627                                                 (cache_resource->action));
10628                 mlx5_hlist_remove(priv->sh->modify_cmds,
10629                                   &cache_resource->entry);
10630                 mlx5_free(cache_resource);
10631                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
10632                         (void *)cache_resource);
10633                 return 0;
10634         }
10635         return 1;
10636 }
10637
10638 /**
10639  * Release port ID action resource.
10640  *
10641  * @param dev
10642  *   Pointer to Ethernet device.
10643  * @param handle
10644  *   Pointer to mlx5_flow_handle.
10645  *
10646  * @return
10647  *   1 while a reference on it exists, 0 when freed.
10648  */
10649 static int
10650 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
10651                                         uint32_t port_id)
10652 {
10653         struct mlx5_priv *priv = dev->data->dev_private;
10654         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
10655         uint32_t idx = port_id;
10656
10657         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
10658                                         idx);
10659         if (!cache_resource)
10660                 return 0;
10661         MLX5_ASSERT(cache_resource->action);
10662         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
10663                 (void *)cache_resource,
10664                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10665         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10666                                __ATOMIC_RELAXED) == 0) {
10667                 claim_zero(mlx5_flow_os_destroy_flow_action
10668                                                 (cache_resource->action));
10669                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
10670                              &priv->sh->port_id_action_list, idx,
10671                              cache_resource, next);
10672                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx);
10673                 DRV_LOG(DEBUG, "port id action resource %p: removed",
10674                         (void *)cache_resource);
10675                 return 0;
10676         }
10677         return 1;
10678 }
10679
10680 /**
10681  * Release push vlan action resource.
10682  *
10683  * @param dev
10684  *   Pointer to Ethernet device.
10685  * @param handle
10686  *   Pointer to mlx5_flow_handle.
10687  *
10688  * @return
10689  *   1 while a reference on it exists, 0 when freed.
10690  */
10691 static int
10692 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
10693                                           struct mlx5_flow_handle *handle)
10694 {
10695         struct mlx5_priv *priv = dev->data->dev_private;
10696         uint32_t idx = handle->dvh.rix_push_vlan;
10697         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
10698
10699         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
10700                                         idx);
10701         if (!cache_resource)
10702                 return 0;
10703         MLX5_ASSERT(cache_resource->action);
10704         DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
10705                 (void *)cache_resource,
10706                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10707         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10708                                __ATOMIC_RELAXED) == 0) {
10709                 claim_zero(mlx5_flow_os_destroy_flow_action
10710                                                 (cache_resource->action));
10711                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
10712                              &priv->sh->push_vlan_action_list, idx,
10713                              cache_resource, next);
10714                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
10715                 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
10716                         (void *)cache_resource);
10717                 return 0;
10718         }
10719         return 1;
10720 }
10721
10722 /**
10723  * Release the fate resource.
10724  *
10725  * @param dev
10726  *   Pointer to Ethernet device.
10727  * @param handle
10728  *   Pointer to mlx5_flow_handle.
10729  */
10730 static void
10731 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
10732                                struct mlx5_flow_handle *handle)
10733 {
10734         if (!handle->rix_fate)
10735                 return;
10736         switch (handle->fate_action) {
10737         case MLX5_FLOW_FATE_DROP:
10738                 mlx5_drop_action_destroy(dev);
10739                 break;
10740         case MLX5_FLOW_FATE_QUEUE:
10741                 mlx5_hrxq_release(dev, handle->rix_hrxq);
10742                 break;
10743         case MLX5_FLOW_FATE_JUMP:
10744                 flow_dv_jump_tbl_resource_release(dev, handle);
10745                 break;
10746         case MLX5_FLOW_FATE_PORT_ID:
10747                 flow_dv_port_id_action_resource_release(dev,
10748                                 handle->rix_port_id_action);
10749                 break;
10750         case MLX5_FLOW_FATE_DEFAULT_MISS:
10751                 flow_dv_default_miss_resource_release(dev);
10752                 break;
10753         default:
10754                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
10755                 break;
10756         }
10757         handle->rix_fate = 0;
10758 }
10759
10760 /**
10761  * Release an sample resource.
10762  *
10763  * @param dev
10764  *   Pointer to Ethernet device.
10765  * @param handle
10766  *   Pointer to mlx5_flow_handle.
10767  *
10768  * @return
10769  *   1 while a reference on it exists, 0 when freed.
10770  */
10771 static int
10772 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
10773                                      struct mlx5_flow_handle *handle)
10774 {
10775         struct mlx5_priv *priv = dev->data->dev_private;
10776         uint32_t idx = handle->dvh.rix_sample;
10777         struct mlx5_flow_dv_sample_resource *cache_resource;
10778
10779         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
10780                          idx);
10781         if (!cache_resource)
10782                 return 0;
10783         MLX5_ASSERT(cache_resource->verbs_action);
10784         DRV_LOG(DEBUG, "sample resource %p: refcnt %d--",
10785                 (void *)cache_resource,
10786                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10787         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10788                                __ATOMIC_RELAXED) == 0) {
10789                 if (cache_resource->verbs_action)
10790                         claim_zero(mlx5_glue->destroy_flow_action
10791                                         (cache_resource->verbs_action));
10792                 if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
10793                         if (cache_resource->default_miss)
10794                                 claim_zero(mlx5_glue->destroy_flow_action
10795                                   (cache_resource->default_miss));
10796                 }
10797                 if (cache_resource->normal_path_tbl)
10798                         flow_dv_tbl_resource_release(dev,
10799                                 cache_resource->normal_path_tbl);
10800         }
10801         if (cache_resource->sample_idx.rix_hrxq &&
10802                 !mlx5_hrxq_release(dev,
10803                         cache_resource->sample_idx.rix_hrxq))
10804                 cache_resource->sample_idx.rix_hrxq = 0;
10805         if (cache_resource->sample_idx.rix_tag &&
10806                 !flow_dv_tag_release(dev,
10807                         cache_resource->sample_idx.rix_tag))
10808                 cache_resource->sample_idx.rix_tag = 0;
10809         if (cache_resource->sample_idx.cnt) {
10810                 flow_dv_counter_release(dev,
10811                         cache_resource->sample_idx.cnt);
10812                 cache_resource->sample_idx.cnt = 0;
10813         }
10814         if (!__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)) {
10815                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
10816                              &priv->sh->sample_action_list, idx,
10817                              cache_resource, next);
10818                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], idx);
10819                 DRV_LOG(DEBUG, "sample resource %p: removed",
10820                         (void *)cache_resource);
10821                 return 0;
10822         }
10823         return 1;
10824 }
10825
10826 /**
10827  * Release an destination array resource.
10828  *
10829  * @param dev
10830  *   Pointer to Ethernet device.
10831  * @param handle
10832  *   Pointer to mlx5_flow_handle.
10833  *
10834  * @return
10835  *   1 while a reference on it exists, 0 when freed.
10836  */
10837 static int
10838 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
10839                                      struct mlx5_flow_handle *handle)
10840 {
10841         struct mlx5_priv *priv = dev->data->dev_private;
10842         struct mlx5_flow_dv_dest_array_resource *cache_resource;
10843         struct mlx5_flow_sub_actions_idx *mdest_act_res;
10844         uint32_t idx = handle->dvh.rix_dest_array;
10845         uint32_t i = 0;
10846
10847         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10848                          idx);
10849         if (!cache_resource)
10850                 return 0;
10851         MLX5_ASSERT(cache_resource->action);
10852         DRV_LOG(DEBUG, "destination array resource %p: refcnt %d--",
10853                 (void *)cache_resource,
10854                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10855         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10856                                __ATOMIC_RELAXED) == 0) {
10857                 if (cache_resource->action)
10858                         claim_zero(mlx5_glue->destroy_flow_action
10859                                                 (cache_resource->action));
10860                 for (; i < cache_resource->num_of_dest; i++) {
10861                         mdest_act_res = &cache_resource->sample_idx[i];
10862                         if (mdest_act_res->rix_hrxq) {
10863                                 mlx5_hrxq_release(dev,
10864                                         mdest_act_res->rix_hrxq);
10865                                 mdest_act_res->rix_hrxq = 0;
10866                         }
10867                         if (mdest_act_res->rix_encap_decap) {
10868                                 flow_dv_encap_decap_resource_release(dev,
10869                                         mdest_act_res->rix_encap_decap);
10870                                 mdest_act_res->rix_encap_decap = 0;
10871                         }
10872                         if (mdest_act_res->rix_port_id_action) {
10873                                 flow_dv_port_id_action_resource_release(dev,
10874                                         mdest_act_res->rix_port_id_action);
10875                                 mdest_act_res->rix_port_id_action = 0;
10876                         }
10877                         if (mdest_act_res->rix_tag) {
10878                                 flow_dv_tag_release(dev,
10879                                         mdest_act_res->rix_tag);
10880                                 mdest_act_res->rix_tag = 0;
10881                         }
10882                 }
10883                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10884                              &priv->sh->dest_array_list, idx,
10885                              cache_resource, next);
10886                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], idx);
10887                 DRV_LOG(DEBUG, "destination array resource %p: removed",
10888                         (void *)cache_resource);
10889                 return 0;
10890         }
10891         return 1;
10892 }
10893
10894 /**
10895  * Remove the flow from the NIC but keeps it in memory.
10896  * Lock free, (mutex should be acquired by caller).
10897  *
10898  * @param[in] dev
10899  *   Pointer to Ethernet device.
10900  * @param[in, out] flow
10901  *   Pointer to flow structure.
10902  */
10903 static void
10904 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
10905 {
10906         struct mlx5_flow_handle *dh;
10907         uint32_t handle_idx;
10908         struct mlx5_priv *priv = dev->data->dev_private;
10909
10910         if (!flow)
10911                 return;
10912         handle_idx = flow->dev_handles;
10913         while (handle_idx) {
10914                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10915                                     handle_idx);
10916                 if (!dh)
10917                         return;
10918                 if (dh->drv_flow) {
10919                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
10920                         dh->drv_flow = NULL;
10921                 }
10922                 if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
10923                     dh->fate_action == MLX5_FLOW_FATE_QUEUE ||
10924                     dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
10925                         flow_dv_fate_resource_release(dev, dh);
10926                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
10927                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
10928                 handle_idx = dh->next.next;
10929         }
10930 }
10931
10932 /**
10933  * Remove the flow from the NIC and the memory.
10934  * Lock free, (mutex should be acquired by caller).
10935  *
10936  * @param[in] dev
10937  *   Pointer to the Ethernet device structure.
10938  * @param[in, out] flow
10939  *   Pointer to flow structure.
10940  */
10941 static void
10942 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
10943 {
10944         struct rte_flow_shared_action *shared;
10945         struct mlx5_flow_handle *dev_handle;
10946         struct mlx5_priv *priv = dev->data->dev_private;
10947
10948         if (!flow)
10949                 return;
10950         __flow_dv_remove(dev, flow);
10951         shared = mlx5_flow_get_shared_rss(flow);
10952         if (shared)
10953                 __atomic_sub_fetch(&shared->refcnt, 1, __ATOMIC_RELAXED);
10954         if (flow->counter) {
10955                 flow_dv_counter_release(dev, flow->counter);
10956                 flow->counter = 0;
10957         }
10958         if (flow->meter) {
10959                 struct mlx5_flow_meter *fm;
10960
10961                 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
10962                                     flow->meter);
10963                 if (fm)
10964                         mlx5_flow_meter_detach(fm);
10965                 flow->meter = 0;
10966         }
10967         while (flow->dev_handles) {
10968                 uint32_t tmp_idx = flow->dev_handles;
10969
10970                 dev_handle = mlx5_ipool_get(priv->sh->ipool
10971                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
10972                 if (!dev_handle)
10973                         return;
10974                 flow->dev_handles = dev_handle->next.next;
10975                 if (dev_handle->dvh.matcher)
10976                         flow_dv_matcher_release(dev, dev_handle);
10977                 if (dev_handle->dvh.rix_sample)
10978                         flow_dv_sample_resource_release(dev, dev_handle);
10979                 if (dev_handle->dvh.rix_dest_array)
10980                         flow_dv_dest_array_resource_release(dev, dev_handle);
10981                 if (dev_handle->dvh.rix_encap_decap)
10982                         flow_dv_encap_decap_resource_release(dev,
10983                                 dev_handle->dvh.rix_encap_decap);
10984                 if (dev_handle->dvh.modify_hdr)
10985                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
10986                 if (dev_handle->dvh.rix_push_vlan)
10987                         flow_dv_push_vlan_action_resource_release(dev,
10988                                                                   dev_handle);
10989                 if (dev_handle->dvh.rix_tag)
10990                         flow_dv_tag_release(dev,
10991                                             dev_handle->dvh.rix_tag);
10992                 flow_dv_fate_resource_release(dev, dev_handle);
10993                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10994                            tmp_idx);
10995         }
10996 }
10997
10998 /**
10999  * Release array of hash RX queue objects.
11000  * Helper function.
11001  *
11002  * @param[in] dev
11003  *   Pointer to the Ethernet device structure.
11004  * @param[in, out] hrxqs
11005  *   Array of hash RX queue objects.
11006  *
11007  * @return
11008  *   Total number of references to hash RX queue objects in *hrxqs* array
11009  *   after this operation.
11010  */
11011 static int
11012 __flow_dv_hrxqs_release(struct rte_eth_dev *dev,
11013                         uint32_t (*hrxqs)[MLX5_RSS_HASH_FIELDS_LEN])
11014 {
11015         size_t i;
11016         int remaining = 0;
11017
11018         for (i = 0; i < RTE_DIM(*hrxqs); i++) {
11019                 int ret = mlx5_hrxq_release(dev, (*hrxqs)[i]);
11020
11021                 if (!ret)
11022                         (*hrxqs)[i] = 0;
11023                 remaining += ret;
11024         }
11025         return remaining;
11026 }
11027
11028 /**
11029  * Release all hash RX queue objects representing shared RSS action.
11030  *
11031  * @param[in] dev
11032  *   Pointer to the Ethernet device structure.
11033  * @param[in, out] action
11034  *   Shared RSS action to remove hash RX queue objects from.
11035  *
11036  * @return
11037  *   Total number of references to hash RX queue objects stored in *action*
11038  *   after this operation.
11039  *   Expected to be 0 if no external references held.
11040  */
11041 static int
11042 __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev,
11043                                  struct mlx5_shared_action_rss *action)
11044 {
11045         return __flow_dv_hrxqs_release(dev, &action->hrxq) +
11046                 __flow_dv_hrxqs_release(dev, &action->hrxq_tunnel);
11047 }
11048
11049 /**
11050  * Setup shared RSS action.
11051  * Prepare set of hash RX queue objects sufficient to handle all valid
11052  * hash_fields combinations (see enum ibv_rx_hash_fields).
11053  *
11054  * @param[in] dev
11055  *   Pointer to the Ethernet device structure.
11056  * @param[in, out] action
11057  *   Partially initialized shared RSS action.
11058  * @param[out] error
11059  *   Perform verbose error reporting if not NULL. Initialized in case of
11060  *   error only.
11061  *
11062  * @return
11063  *   0 on success, otherwise negative errno value.
11064  */
11065 static int
11066 __flow_dv_action_rss_setup(struct rte_eth_dev *dev,
11067                         struct mlx5_shared_action_rss *action,
11068                         struct rte_flow_error *error)
11069 {
11070         size_t i;
11071         int err;
11072
11073         for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) {
11074                 uint32_t hrxq_idx;
11075                 uint64_t hash_fields = mlx5_rss_hash_fields[i];
11076                 int tunnel;
11077
11078                 for (tunnel = 0; tunnel < 2; tunnel++) {
11079                         hrxq_idx = mlx5_hrxq_new(dev, action->origin.key,
11080                                         MLX5_RSS_HASH_KEY_LEN,
11081                                         hash_fields,
11082                                         action->origin.queue,
11083                                         action->origin.queue_num,
11084                                         tunnel, true);
11085                         if (!hrxq_idx) {
11086                                 rte_flow_error_set
11087                                         (error, rte_errno,
11088                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
11089                                          "cannot get hash queue");
11090                                 goto error_hrxq_new;
11091                         }
11092                         err = __flow_dv_action_rss_hrxq_set
11093                                 (action, hash_fields, tunnel, hrxq_idx);
11094                         MLX5_ASSERT(!err);
11095                 }
11096         }
11097         return 0;
11098 error_hrxq_new:
11099         err = rte_errno;
11100         __flow_dv_action_rss_hrxqs_release(dev, action);
11101         rte_errno = err;
11102         return -rte_errno;
11103 }
11104
11105 /**
11106  * Create shared RSS action.
11107  *
11108  * @param[in] dev
11109  *   Pointer to the Ethernet device structure.
11110  * @param[in] conf
11111  *   Shared action configuration.
11112  * @param[in] rss
11113  *   RSS action specification used to create shared action.
11114  * @param[out] error
11115  *   Perform verbose error reporting if not NULL. Initialized in case of
11116  *   error only.
11117  *
11118  * @return
11119  *   A valid shared action handle in case of success, NULL otherwise and
11120  *   rte_errno is set.
11121  */
11122 static struct rte_flow_shared_action *
11123 __flow_dv_action_rss_create(struct rte_eth_dev *dev,
11124                             const struct rte_flow_shared_action_conf *conf,
11125                             const struct rte_flow_action_rss *rss,
11126                             struct rte_flow_error *error)
11127 {
11128         struct rte_flow_shared_action *shared_action = NULL;
11129         void *queue = NULL;
11130         struct mlx5_shared_action_rss *shared_rss;
11131         struct rte_flow_action_rss *origin;
11132         const uint8_t *rss_key;
11133         uint32_t queue_size = rss->queue_num * sizeof(uint16_t);
11134
11135         RTE_SET_USED(conf);
11136         queue = mlx5_malloc(0, RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
11137                             0, SOCKET_ID_ANY);
11138         shared_action = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*shared_action), 0,
11139                                     SOCKET_ID_ANY);
11140         if (!shared_action || !queue) {
11141                 rte_flow_error_set(error, ENOMEM,
11142                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
11143                                    "cannot allocate resource memory");
11144                 goto error_rss_init;
11145         }
11146         shared_rss = &shared_action->rss;
11147         shared_rss->queue = queue;
11148         origin = &shared_rss->origin;
11149         origin->func = rss->func;
11150         origin->level = rss->level;
11151         /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
11152         origin->types = !rss->types ? ETH_RSS_IP : rss->types;
11153         /* NULL RSS key indicates default RSS key. */
11154         rss_key = !rss->key ? rss_hash_default_key : rss->key;
11155         memcpy(shared_rss->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
11156         origin->key = &shared_rss->key[0];
11157         origin->key_len = MLX5_RSS_HASH_KEY_LEN;
11158         memcpy(shared_rss->queue, rss->queue, queue_size);
11159         origin->queue = shared_rss->queue;
11160         origin->queue_num = rss->queue_num;
11161         if (__flow_dv_action_rss_setup(dev, shared_rss, error))
11162                 goto error_rss_init;
11163         shared_action->type = MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS;
11164         return shared_action;
11165 error_rss_init:
11166         mlx5_free(shared_action);
11167         mlx5_free(queue);
11168         return NULL;
11169 }
11170
11171 /**
11172  * Destroy the shared RSS action.
11173  * Release related hash RX queue objects.
11174  *
11175  * @param[in] dev
11176  *   Pointer to the Ethernet device structure.
11177  * @param[in] shared_rss
11178  *   The shared RSS action object to be removed.
11179  * @param[out] error
11180  *   Perform verbose error reporting if not NULL. Initialized in case of
11181  *   error only.
11182  *
11183  * @return
11184  *   0 on success, otherwise negative errno value.
11185  */
11186 static int
11187 __flow_dv_action_rss_release(struct rte_eth_dev *dev,
11188                          struct mlx5_shared_action_rss *shared_rss,
11189                          struct rte_flow_error *error)
11190 {
11191         struct rte_flow_shared_action *shared_action = NULL;
11192         uint32_t old_refcnt = 1;
11193         int remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
11194
11195         if (remaining) {
11196                 return rte_flow_error_set(error, ETOOMANYREFS,
11197                                           RTE_FLOW_ERROR_TYPE_ACTION,
11198                                           NULL,
11199                                           "shared rss hrxq has references");
11200         }
11201         shared_action = container_of(shared_rss,
11202                                      struct rte_flow_shared_action, rss);
11203         if (!__atomic_compare_exchange_n(&shared_action->refcnt, &old_refcnt,
11204                                          0, 0,
11205                                          __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
11206                 return rte_flow_error_set(error, ETOOMANYREFS,
11207                                           RTE_FLOW_ERROR_TYPE_ACTION,
11208                                           NULL,
11209                                           "shared rss has references");
11210         }
11211         rte_free(shared_rss->queue);
11212         return 0;
11213 }
11214
11215 /**
11216  * Create shared action, lock free,
11217  * (mutex should be acquired by caller).
11218  * Dispatcher for action type specific call.
11219  *
11220  * @param[in] dev
11221  *   Pointer to the Ethernet device structure.
11222  * @param[in] conf
11223  *   Shared action configuration.
11224  * @param[in] action
11225  *   Action specification used to create shared action.
11226  * @param[out] error
11227  *   Perform verbose error reporting if not NULL. Initialized in case of
11228  *   error only.
11229  *
11230  * @return
11231  *   A valid shared action handle in case of success, NULL otherwise and
11232  *   rte_errno is set.
11233  */
11234 static struct rte_flow_shared_action *
11235 __flow_dv_action_create(struct rte_eth_dev *dev,
11236                         const struct rte_flow_shared_action_conf *conf,
11237                         const struct rte_flow_action *action,
11238                         struct rte_flow_error *error)
11239 {
11240         struct rte_flow_shared_action *shared_action = NULL;
11241         struct mlx5_priv *priv = dev->data->dev_private;
11242
11243         switch (action->type) {
11244         case RTE_FLOW_ACTION_TYPE_RSS:
11245                 shared_action = __flow_dv_action_rss_create(dev, conf,
11246                                                             action->conf,
11247                                                             error);
11248                 break;
11249         default:
11250                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
11251                                    NULL, "action type not supported");
11252                 break;
11253         }
11254         if (shared_action) {
11255                 __atomic_add_fetch(&shared_action->refcnt, 1,
11256                                    __ATOMIC_RELAXED);
11257                 LIST_INSERT_HEAD(&priv->shared_actions, shared_action, next);
11258         }
11259         return shared_action;
11260 }
11261
11262 /**
11263  * Destroy the shared action.
11264  * Release action related resources on the NIC and the memory.
11265  * Lock free, (mutex should be acquired by caller).
11266  * Dispatcher for action type specific call.
11267  *
11268  * @param[in] dev
11269  *   Pointer to the Ethernet device structure.
11270  * @param[in] action
11271  *   The shared action object to be removed.
11272  * @param[out] error
11273  *   Perform verbose error reporting if not NULL. Initialized in case of
11274  *   error only.
11275  *
11276  * @return
11277  *   0 on success, otherwise negative errno value.
11278  */
11279 static int
11280 __flow_dv_action_destroy(struct rte_eth_dev *dev,
11281                          struct rte_flow_shared_action *action,
11282                          struct rte_flow_error *error)
11283 {
11284         int ret;
11285
11286         switch (action->type) {
11287         case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS:
11288                 ret = __flow_dv_action_rss_release(dev, &action->rss, error);
11289                 break;
11290         default:
11291                 return rte_flow_error_set(error, ENOTSUP,
11292                                           RTE_FLOW_ERROR_TYPE_ACTION,
11293                                           NULL,
11294                                           "action type not supported");
11295         }
11296         if (ret)
11297                 return ret;
11298         LIST_REMOVE(action, next);
11299         rte_free(action);
11300         return 0;
11301 }
11302
11303 /**
11304  * Updates in place shared RSS action configuration.
11305  *
11306  * @param[in] dev
11307  *   Pointer to the Ethernet device structure.
11308  * @param[in] shared_rss
11309  *   The shared RSS action object to be updated.
11310  * @param[in] action_conf
11311  *   RSS action specification used to modify *shared_rss*.
11312  * @param[out] error
11313  *   Perform verbose error reporting if not NULL. Initialized in case of
11314  *   error only.
11315  *
11316  * @return
11317  *   0 on success, otherwise negative errno value.
11318  * @note: currently only support update of RSS queues.
11319  */
11320 static int
11321 __flow_dv_action_rss_update(struct rte_eth_dev *dev,
11322                             struct mlx5_shared_action_rss *shared_rss,
11323                             const struct rte_flow_action_rss *action_conf,
11324                             struct rte_flow_error *error)
11325 {
11326         size_t i;
11327         int ret;
11328         void *queue = NULL;
11329         const uint8_t *rss_key;
11330         uint32_t rss_key_len;
11331         uint32_t queue_size = action_conf->queue_num * sizeof(uint16_t);
11332
11333         queue = mlx5_malloc(MLX5_MEM_ZERO,
11334                             RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
11335                             0, SOCKET_ID_ANY);
11336         if (!queue)
11337                 return rte_flow_error_set(error, ENOMEM,
11338                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11339                                           NULL,
11340                                           "cannot allocate resource memory");
11341         if (action_conf->key) {
11342                 rss_key = action_conf->key;
11343                 rss_key_len = action_conf->key_len;
11344         } else {
11345                 rss_key = rss_hash_default_key;
11346                 rss_key_len = MLX5_RSS_HASH_KEY_LEN;
11347         }
11348         for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) {
11349                 uint32_t hrxq_idx;
11350                 uint64_t hash_fields = mlx5_rss_hash_fields[i];
11351                 int tunnel;
11352
11353                 for (tunnel = 0; tunnel < 2; tunnel++) {
11354                         hrxq_idx = __flow_dv_action_rss_hrxq_lookup
11355                                         (shared_rss, hash_fields, tunnel);
11356                         MLX5_ASSERT(hrxq_idx);
11357                         ret = mlx5_hrxq_modify
11358                                 (dev, hrxq_idx,
11359                                  rss_key, rss_key_len,
11360                                  hash_fields,
11361                                  action_conf->queue, action_conf->queue_num);
11362                         if (ret) {
11363                                 mlx5_free(queue);
11364                                 return rte_flow_error_set
11365                                         (error, rte_errno,
11366                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
11367                                          "cannot update hash queue");
11368                         }
11369                 }
11370         }
11371         mlx5_free(shared_rss->queue);
11372         shared_rss->queue = queue;
11373         memcpy(shared_rss->queue, action_conf->queue, queue_size);
11374         shared_rss->origin.queue = shared_rss->queue;
11375         shared_rss->origin.queue_num = action_conf->queue_num;
11376         return 0;
11377 }
11378
11379 /**
11380  * Updates in place shared action configuration, lock free,
11381  * (mutex should be acquired by caller).
11382  *
11383  * @param[in] dev
11384  *   Pointer to the Ethernet device structure.
11385  * @param[in] action
11386  *   The shared action object to be updated.
11387  * @param[in] action_conf
11388  *   Action specification used to modify *action*.
11389  *   *action_conf* should be of type correlating with type of the *action*,
11390  *   otherwise considered as invalid.
11391  * @param[out] error
11392  *   Perform verbose error reporting if not NULL. Initialized in case of
11393  *   error only.
11394  *
11395  * @return
11396  *   0 on success, otherwise negative errno value.
11397  */
11398 static int
11399 __flow_dv_action_update(struct rte_eth_dev *dev,
11400                         struct rte_flow_shared_action *action,
11401                         const void *action_conf,
11402                         struct rte_flow_error *error)
11403 {
11404         switch (action->type) {
11405         case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS:
11406                 return __flow_dv_action_rss_update(dev, &action->rss,
11407                                                    action_conf, error);
11408         default:
11409                 return rte_flow_error_set(error, ENOTSUP,
11410                                           RTE_FLOW_ERROR_TYPE_ACTION,
11411                                           NULL,
11412                                           "action type not supported");
11413         }
11414 }
11415 /**
11416  * Query a dv flow  rule for its statistics via devx.
11417  *
11418  * @param[in] dev
11419  *   Pointer to Ethernet device.
11420  * @param[in] flow
11421  *   Pointer to the sub flow.
11422  * @param[out] data
11423  *   data retrieved by the query.
11424  * @param[out] error
11425  *   Perform verbose error reporting if not NULL.
11426  *
11427  * @return
11428  *   0 on success, a negative errno value otherwise and rte_errno is set.
11429  */
11430 static int
11431 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
11432                     void *data, struct rte_flow_error *error)
11433 {
11434         struct mlx5_priv *priv = dev->data->dev_private;
11435         struct rte_flow_query_count *qc = data;
11436
11437         if (!priv->config.devx)
11438                 return rte_flow_error_set(error, ENOTSUP,
11439                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11440                                           NULL,
11441                                           "counters are not supported");
11442         if (flow->counter) {
11443                 uint64_t pkts, bytes;
11444                 struct mlx5_flow_counter *cnt;
11445
11446                 cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
11447                                                  NULL);
11448                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
11449                                                &bytes);
11450
11451                 if (err)
11452                         return rte_flow_error_set(error, -err,
11453                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11454                                         NULL, "cannot read counters");
11455                 qc->hits_set = 1;
11456                 qc->bytes_set = 1;
11457                 qc->hits = pkts - cnt->hits;
11458                 qc->bytes = bytes - cnt->bytes;
11459                 if (qc->reset) {
11460                         cnt->hits = pkts;
11461                         cnt->bytes = bytes;
11462                 }
11463                 return 0;
11464         }
11465         return rte_flow_error_set(error, EINVAL,
11466                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11467                                   NULL,
11468                                   "counters are not available");
11469 }
11470
11471 /**
11472  * Query a flow rule AGE action for aging information.
11473  *
11474  * @param[in] dev
11475  *   Pointer to Ethernet device.
11476  * @param[in] flow
11477  *   Pointer to the sub flow.
11478  * @param[out] data
11479  *   data retrieved by the query.
11480  * @param[out] error
11481  *   Perform verbose error reporting if not NULL.
11482  *
11483  * @return
11484  *   0 on success, a negative errno value otherwise and rte_errno is set.
11485  */
11486 static int
11487 flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow,
11488                   void *data, struct rte_flow_error *error)
11489 {
11490         struct rte_flow_query_age *resp = data;
11491
11492         if (flow->counter) {
11493                 struct mlx5_age_param *age_param =
11494                                 flow_dv_counter_idx_get_age(dev, flow->counter);
11495
11496                 if (!age_param || !age_param->timeout)
11497                         return rte_flow_error_set
11498                                         (error, EINVAL,
11499                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11500                                          NULL, "cannot read age data");
11501                 resp->aged = __atomic_load_n(&age_param->state,
11502                                              __ATOMIC_RELAXED) ==
11503                                                         AGE_TMOUT ? 1 : 0;
11504                 resp->sec_since_last_hit_valid = !resp->aged;
11505                 if (resp->sec_since_last_hit_valid)
11506                         resp->sec_since_last_hit =
11507                                 __atomic_load_n(&age_param->sec_since_last_hit,
11508                                                 __ATOMIC_RELAXED);
11509                 return 0;
11510         }
11511         return rte_flow_error_set(error, EINVAL,
11512                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11513                                   NULL,
11514                                   "age data not available");
11515 }
11516
11517 /**
11518  * Query a flow.
11519  *
11520  * @see rte_flow_query()
11521  * @see rte_flow_ops
11522  */
11523 static int
11524 flow_dv_query(struct rte_eth_dev *dev,
11525               struct rte_flow *flow __rte_unused,
11526               const struct rte_flow_action *actions __rte_unused,
11527               void *data __rte_unused,
11528               struct rte_flow_error *error __rte_unused)
11529 {
11530         int ret = -EINVAL;
11531
11532         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
11533                 switch (actions->type) {
11534                 case RTE_FLOW_ACTION_TYPE_VOID:
11535                         break;
11536                 case RTE_FLOW_ACTION_TYPE_COUNT:
11537                         ret = flow_dv_query_count(dev, flow, data, error);
11538                         break;
11539                 case RTE_FLOW_ACTION_TYPE_AGE:
11540                         ret = flow_dv_query_age(dev, flow, data, error);
11541                         break;
11542                 default:
11543                         return rte_flow_error_set(error, ENOTSUP,
11544                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11545                                                   actions,
11546                                                   "action not supported");
11547                 }
11548         }
11549         return ret;
11550 }
11551
11552 /**
11553  * Destroy the meter table set.
11554  * Lock free, (mutex should be acquired by caller).
11555  *
11556  * @param[in] dev
11557  *   Pointer to Ethernet device.
11558  * @param[in] tbl
11559  *   Pointer to the meter table set.
11560  *
11561  * @return
11562  *   Always 0.
11563  */
11564 static int
11565 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
11566                         struct mlx5_meter_domains_infos *tbl)
11567 {
11568         struct mlx5_priv *priv = dev->data->dev_private;
11569         struct mlx5_meter_domains_infos *mtd =
11570                                 (struct mlx5_meter_domains_infos *)tbl;
11571
11572         if (!mtd || !priv->config.dv_flow_en)
11573                 return 0;
11574         if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
11575                 claim_zero(mlx5_flow_os_destroy_flow
11576                            (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
11577         if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
11578                 claim_zero(mlx5_flow_os_destroy_flow
11579                            (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
11580         if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
11581                 claim_zero(mlx5_flow_os_destroy_flow
11582                            (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
11583         if (mtd->egress.color_matcher)
11584                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11585                            (mtd->egress.color_matcher));
11586         if (mtd->egress.any_matcher)
11587                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11588                            (mtd->egress.any_matcher));
11589         if (mtd->egress.tbl)
11590                 flow_dv_tbl_resource_release(dev, mtd->egress.tbl);
11591         if (mtd->egress.sfx_tbl)
11592                 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl);
11593         if (mtd->ingress.color_matcher)
11594                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11595                            (mtd->ingress.color_matcher));
11596         if (mtd->ingress.any_matcher)
11597                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11598                            (mtd->ingress.any_matcher));
11599         if (mtd->ingress.tbl)
11600                 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl);
11601         if (mtd->ingress.sfx_tbl)
11602                 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl);
11603         if (mtd->transfer.color_matcher)
11604                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11605                            (mtd->transfer.color_matcher));
11606         if (mtd->transfer.any_matcher)
11607                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11608                            (mtd->transfer.any_matcher));
11609         if (mtd->transfer.tbl)
11610                 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl);
11611         if (mtd->transfer.sfx_tbl)
11612                 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl);
11613         if (mtd->drop_actn)
11614                 claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn));
11615         mlx5_free(mtd);
11616         return 0;
11617 }
11618
11619 /* Number of meter flow actions, count and jump or count and drop. */
11620 #define METER_ACTIONS 2
11621
11622 /**
11623  * Create specify domain meter table and suffix table.
11624  *
11625  * @param[in] dev
11626  *   Pointer to Ethernet device.
11627  * @param[in,out] mtb
11628  *   Pointer to DV meter table set.
11629  * @param[in] egress
11630  *   Table attribute.
11631  * @param[in] transfer
11632  *   Table attribute.
11633  * @param[in] color_reg_c_idx
11634  *   Reg C index for color match.
11635  *
11636  * @return
11637  *   0 on success, -1 otherwise and rte_errno is set.
11638  */
11639 static int
11640 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
11641                            struct mlx5_meter_domains_infos *mtb,
11642                            uint8_t egress, uint8_t transfer,
11643                            uint32_t color_reg_c_idx)
11644 {
11645         struct mlx5_priv *priv = dev->data->dev_private;
11646         struct mlx5_dev_ctx_shared *sh = priv->sh;
11647         struct mlx5_flow_dv_match_params mask = {
11648                 .size = sizeof(mask.buf),
11649         };
11650         struct mlx5_flow_dv_match_params value = {
11651                 .size = sizeof(value.buf),
11652         };
11653         struct mlx5dv_flow_matcher_attr dv_attr = {
11654                 .type = IBV_FLOW_ATTR_NORMAL,
11655                 .priority = 0,
11656                 .match_criteria_enable = 0,
11657                 .match_mask = (void *)&mask,
11658         };
11659         void *actions[METER_ACTIONS];
11660         struct mlx5_meter_domain_info *dtb;
11661         struct rte_flow_error error;
11662         int i = 0;
11663         int ret;
11664
11665         if (transfer)
11666                 dtb = &mtb->transfer;
11667         else if (egress)
11668                 dtb = &mtb->egress;
11669         else
11670                 dtb = &mtb->ingress;
11671         /* Create the meter table with METER level. */
11672         dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
11673                                             egress, transfer, false, NULL, 0,
11674                                             &error);
11675         if (!dtb->tbl) {
11676                 DRV_LOG(ERR, "Failed to create meter policer table.");
11677                 return -1;
11678         }
11679         /* Create the meter suffix table with SUFFIX level. */
11680         dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
11681                                             MLX5_FLOW_TABLE_LEVEL_SUFFIX,
11682                                             egress, transfer, false, NULL, 0,
11683                                             &error);
11684         if (!dtb->sfx_tbl) {
11685                 DRV_LOG(ERR, "Failed to create meter suffix table.");
11686                 return -1;
11687         }
11688         /* Create matchers, Any and Color. */
11689         dv_attr.priority = 3;
11690         dv_attr.match_criteria_enable = 0;
11691         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
11692                                                &dtb->any_matcher);
11693         if (ret) {
11694                 DRV_LOG(ERR, "Failed to create meter"
11695                              " policer default matcher.");
11696                 goto error_exit;
11697         }
11698         dv_attr.priority = 0;
11699         dv_attr.match_criteria_enable =
11700                                 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
11701         flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
11702                                rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
11703         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
11704                                                &dtb->color_matcher);
11705         if (ret) {
11706                 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
11707                 goto error_exit;
11708         }
11709         if (mtb->count_actns[RTE_MTR_DROPPED])
11710                 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
11711         actions[i++] = mtb->drop_actn;
11712         /* Default rule: lowest priority, match any, actions: drop. */
11713         ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i,
11714                                        actions,
11715                                        &dtb->policer_rules[RTE_MTR_DROPPED]);
11716         if (ret) {
11717                 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
11718                 goto error_exit;
11719         }
11720         return 0;
11721 error_exit:
11722         return -1;
11723 }
11724
11725 /**
11726  * Create the needed meter and suffix tables.
11727  * Lock free, (mutex should be acquired by caller).
11728  *
11729  * @param[in] dev
11730  *   Pointer to Ethernet device.
11731  * @param[in] fm
11732  *   Pointer to the flow meter.
11733  *
11734  * @return
11735  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
11736  */
11737 static struct mlx5_meter_domains_infos *
11738 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
11739                        const struct mlx5_flow_meter *fm)
11740 {
11741         struct mlx5_priv *priv = dev->data->dev_private;
11742         struct mlx5_meter_domains_infos *mtb;
11743         int ret;
11744         int i;
11745
11746         if (!priv->mtr_en) {
11747                 rte_errno = ENOTSUP;
11748                 return NULL;
11749         }
11750         mtb = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*mtb), 0, SOCKET_ID_ANY);
11751         if (!mtb) {
11752                 DRV_LOG(ERR, "Failed to allocate memory for meter.");
11753                 return NULL;
11754         }
11755         /* Create meter count actions */
11756         for (i = 0; i <= RTE_MTR_DROPPED; i++) {
11757                 struct mlx5_flow_counter *cnt;
11758                 if (!fm->policer_stats.cnt[i])
11759                         continue;
11760                 cnt = flow_dv_counter_get_by_idx(dev,
11761                       fm->policer_stats.cnt[i], NULL);
11762                 mtb->count_actns[i] = cnt->action;
11763         }
11764         /* Create drop action. */
11765         ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn);
11766         if (ret) {
11767                 DRV_LOG(ERR, "Failed to create drop action.");
11768                 goto error_exit;
11769         }
11770         /* Egress meter table. */
11771         ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
11772         if (ret) {
11773                 DRV_LOG(ERR, "Failed to prepare egress meter table.");
11774                 goto error_exit;
11775         }
11776         /* Ingress meter table. */
11777         ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
11778         if (ret) {
11779                 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
11780                 goto error_exit;
11781         }
11782         /* FDB meter table. */
11783         if (priv->config.dv_esw_en) {
11784                 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
11785                                                  priv->mtr_color_reg);
11786                 if (ret) {
11787                         DRV_LOG(ERR, "Failed to prepare fdb meter table.");
11788                         goto error_exit;
11789                 }
11790         }
11791         return mtb;
11792 error_exit:
11793         flow_dv_destroy_mtr_tbl(dev, mtb);
11794         return NULL;
11795 }
11796
11797 /**
11798  * Destroy domain policer rule.
11799  *
11800  * @param[in] dt
11801  *   Pointer to domain table.
11802  */
11803 static void
11804 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
11805 {
11806         int i;
11807
11808         for (i = 0; i < RTE_MTR_DROPPED; i++) {
11809                 if (dt->policer_rules[i]) {
11810                         claim_zero(mlx5_flow_os_destroy_flow
11811                                    (dt->policer_rules[i]));
11812                         dt->policer_rules[i] = NULL;
11813                 }
11814         }
11815         if (dt->jump_actn) {
11816                 claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn));
11817                 dt->jump_actn = NULL;
11818         }
11819 }
11820
11821 /**
11822  * Destroy policer rules.
11823  *
11824  * @param[in] dev
11825  *   Pointer to Ethernet device.
11826  * @param[in] fm
11827  *   Pointer to flow meter structure.
11828  * @param[in] attr
11829  *   Pointer to flow attributes.
11830  *
11831  * @return
11832  *   Always 0.
11833  */
11834 static int
11835 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
11836                               const struct mlx5_flow_meter *fm,
11837                               const struct rte_flow_attr *attr)
11838 {
11839         struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
11840
11841         if (!mtb)
11842                 return 0;
11843         if (attr->egress)
11844                 flow_dv_destroy_domain_policer_rule(&mtb->egress);
11845         if (attr->ingress)
11846                 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
11847         if (attr->transfer)
11848                 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
11849         return 0;
11850 }
11851
11852 /**
11853  * Create specify domain meter policer rule.
11854  *
11855  * @param[in] fm
11856  *   Pointer to flow meter structure.
11857  * @param[in] mtb
11858  *   Pointer to DV meter table set.
11859  * @param[in] mtr_reg_c
11860  *   Color match REG_C.
11861  *
11862  * @return
11863  *   0 on success, -1 otherwise.
11864  */
11865 static int
11866 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
11867                                     struct mlx5_meter_domain_info *dtb,
11868                                     uint8_t mtr_reg_c)
11869 {
11870         struct mlx5_flow_dv_match_params matcher = {
11871                 .size = sizeof(matcher.buf),
11872         };
11873         struct mlx5_flow_dv_match_params value = {
11874                 .size = sizeof(value.buf),
11875         };
11876         struct mlx5_meter_domains_infos *mtb = fm->mfts;
11877         void *actions[METER_ACTIONS];
11878         int i;
11879         int ret = 0;
11880
11881         /* Create jump action. */
11882         if (!dtb->jump_actn)
11883                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
11884                                 (dtb->sfx_tbl->obj, &dtb->jump_actn);
11885         if (ret) {
11886                 DRV_LOG(ERR, "Failed to create policer jump action.");
11887                 goto error;
11888         }
11889         for (i = 0; i < RTE_MTR_DROPPED; i++) {
11890                 int j = 0;
11891
11892                 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
11893                                        rte_col_2_mlx5_col(i), UINT8_MAX);
11894                 if (mtb->count_actns[i])
11895                         actions[j++] = mtb->count_actns[i];
11896                 if (fm->action[i] == MTR_POLICER_ACTION_DROP)
11897                         actions[j++] = mtb->drop_actn;
11898                 else
11899                         actions[j++] = dtb->jump_actn;
11900                 ret = mlx5_flow_os_create_flow(dtb->color_matcher,
11901                                                (void *)&value, j, actions,
11902                                                &dtb->policer_rules[i]);
11903                 if (ret) {
11904                         DRV_LOG(ERR, "Failed to create policer rule.");
11905                         goto error;
11906                 }
11907         }
11908         return 0;
11909 error:
11910         rte_errno = errno;
11911         return -1;
11912 }
11913
11914 /**
11915  * Create policer rules.
11916  *
11917  * @param[in] dev
11918  *   Pointer to Ethernet device.
11919  * @param[in] fm
11920  *   Pointer to flow meter structure.
11921  * @param[in] attr
11922  *   Pointer to flow attributes.
11923  *
11924  * @return
11925  *   0 on success, -1 otherwise.
11926  */
11927 static int
11928 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
11929                              struct mlx5_flow_meter *fm,
11930                              const struct rte_flow_attr *attr)
11931 {
11932         struct mlx5_priv *priv = dev->data->dev_private;
11933         struct mlx5_meter_domains_infos *mtb = fm->mfts;
11934         int ret;
11935
11936         if (attr->egress) {
11937                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
11938                                                 priv->mtr_color_reg);
11939                 if (ret) {
11940                         DRV_LOG(ERR, "Failed to create egress policer.");
11941                         goto error;
11942                 }
11943         }
11944         if (attr->ingress) {
11945                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
11946                                                 priv->mtr_color_reg);
11947                 if (ret) {
11948                         DRV_LOG(ERR, "Failed to create ingress policer.");
11949                         goto error;
11950                 }
11951         }
11952         if (attr->transfer) {
11953                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
11954                                                 priv->mtr_color_reg);
11955                 if (ret) {
11956                         DRV_LOG(ERR, "Failed to create transfer policer.");
11957                         goto error;
11958                 }
11959         }
11960         return 0;
11961 error:
11962         flow_dv_destroy_policer_rules(dev, fm, attr);
11963         return -1;
11964 }
11965
11966 /**
11967  * Validate the batch counter support in root table.
11968  *
11969  * Create a simple flow with invalid counter and drop action on root table to
11970  * validate if batch counter with offset on root table is supported or not.
11971  *
11972  * @param[in] dev
11973  *   Pointer to rte_eth_dev structure.
11974  *
11975  * @return
11976  *   0 on success, a negative errno value otherwise and rte_errno is set.
11977  */
11978 int
11979 mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev)
11980 {
11981         struct mlx5_priv *priv = dev->data->dev_private;
11982         struct mlx5_dev_ctx_shared *sh = priv->sh;
11983         struct mlx5_flow_dv_match_params mask = {
11984                 .size = sizeof(mask.buf),
11985         };
11986         struct mlx5_flow_dv_match_params value = {
11987                 .size = sizeof(value.buf),
11988         };
11989         struct mlx5dv_flow_matcher_attr dv_attr = {
11990                 .type = IBV_FLOW_ATTR_NORMAL,
11991                 .priority = 0,
11992                 .match_criteria_enable = 0,
11993                 .match_mask = (void *)&mask,
11994         };
11995         void *actions[2] = { 0 };
11996         struct mlx5_flow_tbl_resource *tbl = NULL, *dest_tbl = NULL;
11997         struct mlx5_devx_obj *dcs = NULL;
11998         void *matcher = NULL;
11999         void *flow = NULL;
12000         int i, ret = -1;
12001
12002         tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL, 0, NULL);
12003         if (!tbl)
12004                 goto err;
12005         dest_tbl = flow_dv_tbl_resource_get(dev, 1, 0, 0, false, NULL, 0, NULL);
12006         if (!dest_tbl)
12007                 goto err;
12008         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
12009         if (!dcs)
12010                 goto err;
12011         ret = mlx5_flow_os_create_flow_action_count(dcs->obj, UINT16_MAX,
12012                                                     &actions[0]);
12013         if (ret)
12014                 goto err;
12015         ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
12016                                 (dest_tbl->obj, &actions[1]);
12017         if (ret)
12018                 goto err;
12019         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
12020         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
12021                                                &matcher);
12022         if (ret)
12023                 goto err;
12024         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 2,
12025                                        actions, &flow);
12026 err:
12027         /*
12028          * If batch counter with offset is not supported, the driver will not
12029          * validate the invalid offset value, flow create should success.
12030          * In this case, it means batch counter is not supported in root table.
12031          *
12032          * Otherwise, if flow create is failed, counter offset is supported.
12033          */
12034         if (flow) {
12035                 DRV_LOG(INFO, "Batch counter is not supported in root "
12036                               "table. Switch to fallback mode.");
12037                 rte_errno = ENOTSUP;
12038                 ret = -rte_errno;
12039                 claim_zero(mlx5_flow_os_destroy_flow(flow));
12040         } else {
12041                 /* Check matcher to make sure validate fail at flow create. */
12042                 if (!matcher || (matcher && errno != EINVAL))
12043                         DRV_LOG(ERR, "Unexpected error in counter offset "
12044                                      "support detection");
12045                 ret = 0;
12046         }
12047         for (i = 0; i < 2; i++) {
12048                 if (actions[i])
12049                         claim_zero(mlx5_flow_os_destroy_flow_action
12050                                    (actions[i]));
12051         }
12052         if (matcher)
12053                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
12054         if (tbl)
12055                 flow_dv_tbl_resource_release(dev, tbl);
12056         if (dest_tbl)
12057                 flow_dv_tbl_resource_release(dev, dest_tbl);
12058         if (dcs)
12059                 claim_zero(mlx5_devx_cmd_destroy(dcs));
12060         return ret;
12061 }
12062
12063 /**
12064  * Query a devx counter.
12065  *
12066  * @param[in] dev
12067  *   Pointer to the Ethernet device structure.
12068  * @param[in] cnt
12069  *   Index to the flow counter.
12070  * @param[in] clear
12071  *   Set to clear the counter statistics.
12072  * @param[out] pkts
12073  *   The statistics value of packets.
12074  * @param[out] bytes
12075  *   The statistics value of bytes.
12076  *
12077  * @return
12078  *   0 on success, otherwise return -1.
12079  */
12080 static int
12081 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
12082                       uint64_t *pkts, uint64_t *bytes)
12083 {
12084         struct mlx5_priv *priv = dev->data->dev_private;
12085         struct mlx5_flow_counter *cnt;
12086         uint64_t inn_pkts, inn_bytes;
12087         int ret;
12088
12089         if (!priv->config.devx)
12090                 return -1;
12091
12092         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
12093         if (ret)
12094                 return -1;
12095         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
12096         *pkts = inn_pkts - cnt->hits;
12097         *bytes = inn_bytes - cnt->bytes;
12098         if (clear) {
12099                 cnt->hits = inn_pkts;
12100                 cnt->bytes = inn_bytes;
12101         }
12102         return 0;
12103 }
12104
12105 /**
12106  * Get aged-out flows.
12107  *
12108  * @param[in] dev
12109  *   Pointer to the Ethernet device structure.
12110  * @param[in] context
12111  *   The address of an array of pointers to the aged-out flows contexts.
12112  * @param[in] nb_contexts
12113  *   The length of context array pointers.
12114  * @param[out] error
12115  *   Perform verbose error reporting if not NULL. Initialized in case of
12116  *   error only.
12117  *
12118  * @return
12119  *   how many contexts get in success, otherwise negative errno value.
12120  *   if nb_contexts is 0, return the amount of all aged contexts.
12121  *   if nb_contexts is not 0 , return the amount of aged flows reported
12122  *   in the context array.
12123  * @note: only stub for now
12124  */
12125 static int
12126 flow_get_aged_flows(struct rte_eth_dev *dev,
12127                     void **context,
12128                     uint32_t nb_contexts,
12129                     struct rte_flow_error *error)
12130 {
12131         struct mlx5_priv *priv = dev->data->dev_private;
12132         struct mlx5_age_info *age_info;
12133         struct mlx5_age_param *age_param;
12134         struct mlx5_flow_counter *counter;
12135         int nb_flows = 0;
12136
12137         if (nb_contexts && !context)
12138                 return rte_flow_error_set(error, EINVAL,
12139                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
12140                                           NULL,
12141                                           "Should assign at least one flow or"
12142                                           " context to get if nb_contexts != 0");
12143         age_info = GET_PORT_AGE_INFO(priv);
12144         rte_spinlock_lock(&age_info->aged_sl);
12145         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
12146                 nb_flows++;
12147                 if (nb_contexts) {
12148                         age_param = MLX5_CNT_TO_AGE(counter);
12149                         context[nb_flows - 1] = age_param->context;
12150                         if (!(--nb_contexts))
12151                                 break;
12152                 }
12153         }
12154         rte_spinlock_unlock(&age_info->aged_sl);
12155         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
12156         return nb_flows;
12157 }
12158
12159 /*
12160  * Mutex-protected thunk to lock-free  __flow_dv_translate().
12161  */
12162 static int
12163 flow_dv_translate(struct rte_eth_dev *dev,
12164                   struct mlx5_flow *dev_flow,
12165                   const struct rte_flow_attr *attr,
12166                   const struct rte_flow_item items[],
12167                   const struct rte_flow_action actions[],
12168                   struct rte_flow_error *error)
12169 {
12170         int ret;
12171
12172         flow_dv_shared_lock(dev);
12173         ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
12174         flow_dv_shared_unlock(dev);
12175         return ret;
12176 }
12177
12178 /*
12179  * Mutex-protected thunk to lock-free  __flow_dv_apply().
12180  */
12181 static int
12182 flow_dv_apply(struct rte_eth_dev *dev,
12183               struct rte_flow *flow,
12184               struct rte_flow_error *error)
12185 {
12186         int ret;
12187
12188         flow_dv_shared_lock(dev);
12189         ret = __flow_dv_apply(dev, flow, error);
12190         flow_dv_shared_unlock(dev);
12191         return ret;
12192 }
12193
12194 /*
12195  * Mutex-protected thunk to lock-free __flow_dv_remove().
12196  */
12197 static void
12198 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
12199 {
12200         flow_dv_shared_lock(dev);
12201         __flow_dv_remove(dev, flow);
12202         flow_dv_shared_unlock(dev);
12203 }
12204
12205 /*
12206  * Mutex-protected thunk to lock-free __flow_dv_destroy().
12207  */
12208 static void
12209 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
12210 {
12211         flow_dv_shared_lock(dev);
12212         __flow_dv_destroy(dev, flow);
12213         flow_dv_shared_unlock(dev);
12214 }
12215
12216 /*
12217  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
12218  */
12219 static uint32_t
12220 flow_dv_counter_allocate(struct rte_eth_dev *dev)
12221 {
12222         uint32_t cnt;
12223
12224         flow_dv_shared_lock(dev);
12225         cnt = flow_dv_counter_alloc(dev, 0);
12226         flow_dv_shared_unlock(dev);
12227         return cnt;
12228 }
12229
12230 /*
12231  * Mutex-protected thunk to lock-free flow_dv_counter_release().
12232  */
12233 static void
12234 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
12235 {
12236         flow_dv_shared_lock(dev);
12237         flow_dv_counter_release(dev, cnt);
12238         flow_dv_shared_unlock(dev);
12239 }
12240
12241 /**
12242  * Validate shared action.
12243  * Dispatcher for action type specific validation.
12244  *
12245  * @param[in] dev
12246  *   Pointer to the Ethernet device structure.
12247  * @param[in] conf
12248  *   Shared action configuration.
12249  * @param[in] action
12250  *   The shared action object to validate.
12251  * @param[out] error
12252  *   Perform verbose error reporting if not NULL. Initialized in case of
12253  *   error only.
12254  *
12255  * @return
12256  *   0 on success, otherwise negative errno value.
12257  */
12258 static int
12259 flow_dv_action_validate(struct rte_eth_dev *dev,
12260                         const struct rte_flow_shared_action_conf *conf,
12261                         const struct rte_flow_action *action,
12262                         struct rte_flow_error *error)
12263 {
12264         RTE_SET_USED(conf);
12265         switch (action->type) {
12266         case RTE_FLOW_ACTION_TYPE_RSS:
12267                 return mlx5_validate_action_rss(dev, action, error);
12268         default:
12269                 return rte_flow_error_set(error, ENOTSUP,
12270                                           RTE_FLOW_ERROR_TYPE_ACTION,
12271                                           NULL,
12272                                           "action type not supported");
12273         }
12274 }
12275
12276 /*
12277  * Mutex-protected thunk to lock-free  __flow_dv_action_create().
12278  */
12279 static struct rte_flow_shared_action *
12280 flow_dv_action_create(struct rte_eth_dev *dev,
12281                       const struct rte_flow_shared_action_conf *conf,
12282                       const struct rte_flow_action *action,
12283                       struct rte_flow_error *error)
12284 {
12285         struct rte_flow_shared_action *shared_action = NULL;
12286
12287         flow_dv_shared_lock(dev);
12288         shared_action = __flow_dv_action_create(dev, conf, action, error);
12289         flow_dv_shared_unlock(dev);
12290         return shared_action;
12291 }
12292
12293 /*
12294  * Mutex-protected thunk to lock-free  __flow_dv_action_destroy().
12295  */
12296 static int
12297 flow_dv_action_destroy(struct rte_eth_dev *dev,
12298                        struct rte_flow_shared_action *action,
12299                        struct rte_flow_error *error)
12300 {
12301         int ret;
12302
12303         flow_dv_shared_lock(dev);
12304         ret = __flow_dv_action_destroy(dev, action, error);
12305         flow_dv_shared_unlock(dev);
12306         return ret;
12307 }
12308
12309 /*
12310  * Mutex-protected thunk to lock-free  __flow_dv_action_update().
12311  */
12312 static int
12313 flow_dv_action_update(struct rte_eth_dev *dev,
12314                       struct rte_flow_shared_action *action,
12315                       const void *action_conf,
12316                       struct rte_flow_error *error)
12317 {
12318         int ret;
12319
12320         flow_dv_shared_lock(dev);
12321         ret = __flow_dv_action_update(dev, action, action_conf,
12322                                       error);
12323         flow_dv_shared_unlock(dev);
12324         return ret;
12325 }
12326
12327 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
12328         .validate = flow_dv_validate,
12329         .prepare = flow_dv_prepare,
12330         .translate = flow_dv_translate,
12331         .apply = flow_dv_apply,
12332         .remove = flow_dv_remove,
12333         .destroy = flow_dv_destroy,
12334         .query = flow_dv_query,
12335         .create_mtr_tbls = flow_dv_create_mtr_tbl,
12336         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
12337         .create_policer_rules = flow_dv_create_policer_rules,
12338         .destroy_policer_rules = flow_dv_destroy_policer_rules,
12339         .counter_alloc = flow_dv_counter_allocate,
12340         .counter_free = flow_dv_counter_free,
12341         .counter_query = flow_dv_counter_query,
12342         .get_aged_flows = flow_get_aged_flows,
12343         .action_validate = flow_dv_action_validate,
12344         .action_create = flow_dv_action_create,
12345         .action_destroy = flow_dv_action_destroy,
12346         .action_update = flow_dv_action_update,
12347 };
12348
12349 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
12350