drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <rte_ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_ip.h>
  19 #include <rte_gre.h>
  20 #include <rte_vxlan.h>
  21 #include <rte_gtp.h>
  22 #include <rte_eal_paging.h>
  23 #include <rte_mpls.h>
  24
  25 #include <mlx5_glue.h>
  26 #include <mlx5_devx_cmds.h>
  27 #include <mlx5_prm.h>
  28 #include <mlx5_malloc.h>
  29
  30 #include "mlx5_defs.h"
  31 #include "mlx5.h"
  32 #include "mlx5_common_os.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_flow_os.h"
  35 #include "mlx5_rxtx.h"
  36
  37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  38
  39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  41 #endif
  42
  43 #ifndef HAVE_MLX5DV_DR_ESWITCH
  44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  46 #endif
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR
  50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  51 #endif
  52
  53 /* VLAN header definitions */
  54 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  55 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  56 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  57 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  58 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  59
  60 union flow_dv_attr {
  61         struct {
  62                 uint32_t valid:1;
  63                 uint32_t ipv4:1;
  64                 uint32_t ipv6:1;
  65                 uint32_t tcp:1;
  66                 uint32_t udp:1;
  67                 uint32_t reserved:27;
  68         };
  69         uint32_t attr;
  70 };
  71
  72 static int
  73 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
  74                              struct mlx5_flow_tbl_resource *tbl);
  75
  76 static int
  77 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev);
  78
  79 static int
  80 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  81                                       uint32_t encap_decap_idx);
  82
  83 static int
  84 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  85                                         uint32_t port_id);
  86
  87 /**
  88  * Initialize flow attributes structure according to flow items' types.
  89  *
  90  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  91  * mode. For tunnel mode, the items to be modified are the outermost ones.
  92  *
  93  * @param[in] item
  94  *   Pointer to item specification.
  95  * @param[out] attr
  96  *   Pointer to flow attributes structure.
  97  * @param[in] dev_flow
  98  *   Pointer to the sub flow.
  99  * @param[in] tunnel_decap
 100  *   Whether action is after tunnel decapsulation.
 101  */
 102 static void
 103 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 104                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 105 {
 106         uint64_t layers = dev_flow->handle->layers;
 107
 108         /*
 109          * If layers is already initialized, it means this dev_flow is the
 110          * suffix flow, the layers flags is set by the prefix flow. Need to
 111          * use the layer flags from prefix flow as the suffix flow may not
 112          * have the user defined items as the flow is split.
 113          */
 114         if (layers) {
 115                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 116                         attr->ipv4 = 1;
 117                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 118                         attr->ipv6 = 1;
 119                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 120                         attr->tcp = 1;
 121                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 122                         attr->udp = 1;
 123                 attr->valid = 1;
 124                 return;
 125         }
 126         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 127                 uint8_t next_protocol = 0xff;
 128                 switch (item->type) {
 129                 case RTE_FLOW_ITEM_TYPE_GRE:
 130                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 131                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 132                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 133                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 134                 case RTE_FLOW_ITEM_TYPE_MPLS:
 135                         if (tunnel_decap)
 136                                 attr->attr = 0;
 137                         break;
 138                 case RTE_FLOW_ITEM_TYPE_IPV4:
 139                         if (!attr->ipv6)
 140                                 attr->ipv4 = 1;
 141                         if (item->mask != NULL &&
 142                             ((const struct rte_flow_item_ipv4 *)
 143                             item->mask)->hdr.next_proto_id)
 144                                 next_protocol =
 145                                     ((const struct rte_flow_item_ipv4 *)
 146                                       (item->spec))->hdr.next_proto_id &
 147                                     ((const struct rte_flow_item_ipv4 *)
 148                                       (item->mask))->hdr.next_proto_id;
 149                         if ((next_protocol == IPPROTO_IPIP ||
 150                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 151                                 attr->attr = 0;
 152                         break;
 153                 case RTE_FLOW_ITEM_TYPE_IPV6:
 154                         if (!attr->ipv4)
 155                                 attr->ipv6 = 1;
 156                         if (item->mask != NULL &&
 157                             ((const struct rte_flow_item_ipv6 *)
 158                             item->mask)->hdr.proto)
 159                                 next_protocol =
 160                                     ((const struct rte_flow_item_ipv6 *)
 161                                       (item->spec))->hdr.proto &
 162                                     ((const struct rte_flow_item_ipv6 *)
 163                                       (item->mask))->hdr.proto;
 164                         if ((next_protocol == IPPROTO_IPIP ||
 165                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 166                                 attr->attr = 0;
 167                         break;
 168                 case RTE_FLOW_ITEM_TYPE_UDP:
 169                         if (!attr->tcp)
 170                                 attr->udp = 1;
 171                         break;
 172                 case RTE_FLOW_ITEM_TYPE_TCP:
 173                         if (!attr->udp)
 174                                 attr->tcp = 1;
 175                         break;
 176                 default:
 177                         break;
 178                 }
 179         }
 180         attr->valid = 1;
 181 }
 182
 183 /**
 184  * Convert rte_mtr_color to mlx5 color.
 185  *
 186  * @param[in] rcol
 187  *   rte_mtr_color.
 188  *
 189  * @return
 190  *   mlx5 color.
 191  */
 192 static int
 193 rte_col_2_mlx5_col(enum rte_color rcol)
 194 {
 195         switch (rcol) {
 196         case RTE_COLOR_GREEN:
 197                 return MLX5_FLOW_COLOR_GREEN;
 198         case RTE_COLOR_YELLOW:
 199                 return MLX5_FLOW_COLOR_YELLOW;
 200         case RTE_COLOR_RED:
 201                 return MLX5_FLOW_COLOR_RED;
 202         default:
 203                 break;
 204         }
 205         return MLX5_FLOW_COLOR_UNDEFINED;
 206 }
 207
 208 struct field_modify_info {
 209         uint32_t size; /* Size of field in protocol header, in bytes. */
 210         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 211         enum mlx5_modification_field id;
 212 };
 213
 214 struct field_modify_info modify_eth[] = {
 215         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 216         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 217         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 218         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 219         {0, 0, 0},
 220 };
 221
 222 struct field_modify_info modify_vlan_out_first_vid[] = {
 223         /* Size in bits !!! */
 224         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 225         {0, 0, 0},
 226 };
 227
 228 struct field_modify_info modify_ipv4[] = {
 229         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 230         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 231         {4, 12, MLX5_MODI_OUT_SIPV4},
 232         {4, 16, MLX5_MODI_OUT_DIPV4},
 233         {0, 0, 0},
 234 };
 235
 236 struct field_modify_info modify_ipv6[] = {
 237         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 238         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 239         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 240         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 241         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 242         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 243         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 244         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 245         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 246         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 247         {0, 0, 0},
 248 };
 249
 250 struct field_modify_info modify_udp[] = {
 251         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 252         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 253         {0, 0, 0},
 254 };
 255
 256 struct field_modify_info modify_tcp[] = {
 257         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 258         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 259         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 260         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 261         {0, 0, 0},
 262 };
 263
 264 static void
 265 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 266                           uint8_t next_protocol, uint64_t *item_flags,
 267                           int *tunnel)
 268 {
 269         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 270                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 271         if (next_protocol == IPPROTO_IPIP) {
 272                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 273                 *tunnel = 1;
 274         }
 275         if (next_protocol == IPPROTO_IPV6) {
 276                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 277                 *tunnel = 1;
 278         }
 279 }
 280
 281 /**
 282  * Acquire the synchronizing object to protect multithreaded access
 283  * to shared dv context. Lock occurs only if context is actually
 284  * shared, i.e. we have multiport IB device and representors are
 285  * created.
 286  *
 287  * @param[in] dev
 288  *   Pointer to the rte_eth_dev structure.
 289  */
 290 static void
 291 flow_dv_shared_lock(struct rte_eth_dev *dev)
 292 {
 293         struct mlx5_priv *priv = dev->data->dev_private;
 294         struct mlx5_dev_ctx_shared *sh = priv->sh;
 295
 296         if (sh->dv_refcnt > 1) {
 297                 int ret;
 298
 299                 ret = pthread_mutex_lock(&sh->dv_mutex);
 300                 MLX5_ASSERT(!ret);
 301                 (void)ret;
 302         }
 303 }
 304
 305 static void
 306 flow_dv_shared_unlock(struct rte_eth_dev *dev)
 307 {
 308         struct mlx5_priv *priv = dev->data->dev_private;
 309         struct mlx5_dev_ctx_shared *sh = priv->sh;
 310
 311         if (sh->dv_refcnt > 1) {
 312                 int ret;
 313
 314                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 315                 MLX5_ASSERT(!ret);
 316                 (void)ret;
 317         }
 318 }
 319
 320 /* Update VLAN's VID/PCP based on input rte_flow_action.
 321  *
 322  * @param[in] action
 323  *   Pointer to struct rte_flow_action.
 324  * @param[out] vlan
 325  *   Pointer to struct rte_vlan_hdr.
 326  */
 327 static void
 328 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 329                          struct rte_vlan_hdr *vlan)
 330 {
 331         uint16_t vlan_tci;
 332         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 333                 vlan_tci =
 334                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 335                                                action->conf)->vlan_pcp;
 336                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 337                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 338                 vlan->vlan_tci |= vlan_tci;
 339         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 340                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 341                 vlan->vlan_tci |= rte_be_to_cpu_16
 342                     (((const struct rte_flow_action_of_set_vlan_vid *)
 343                                              action->conf)->vlan_vid);
 344         }
 345 }
 346
 347 /**
 348  * Fetch 1, 2, 3 or 4 byte field from the byte array
 349  * and return as unsigned integer in host-endian format.
 350  *
 351  * @param[in] data
 352  *   Pointer to data array.
 353  * @param[in] size
 354  *   Size of field to extract.
 355  *
 356  * @return
 357  *   converted field in host endian format.
 358  */
 359 static inline uint32_t
 360 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 361 {
 362         uint32_t ret;
 363
 364         switch (size) {
 365         case 1:
 366                 ret = *data;
 367                 break;
 368         case 2:
 369                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 370                 break;
 371         case 3:
 372                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 373                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 374                 break;
 375         case 4:
 376                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 377                 break;
 378         default:
 379                 MLX5_ASSERT(false);
 380                 ret = 0;
 381                 break;
 382         }
 383         return ret;
 384 }
 385
 386 /**
 387  * Convert modify-header action to DV specification.
 388  *
 389  * Data length of each action is determined by provided field description
 390  * and the item mask. Data bit offset and width of each action is determined
 391  * by provided item mask.
 392  *
 393  * @param[in] item
 394  *   Pointer to item specification.
 395  * @param[in] field
 396  *   Pointer to field modification information.
 397  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 398  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 399  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 400  * @param[in] dcopy
 401  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 402  *   Negative offset value sets the same offset as source offset.
 403  *   size field is ignored, value is taken from source field.
 404  * @param[in,out] resource
 405  *   Pointer to the modify-header resource.
 406  * @param[in] type
 407  *   Type of modification.
 408  * @param[out] error
 409  *   Pointer to the error structure.
 410  *
 411  * @return
 412  *   0 on success, a negative errno value otherwise and rte_errno is set.
 413  */
 414 static int
 415 flow_dv_convert_modify_action(struct rte_flow_item *item,
 416                               struct field_modify_info *field,
 417                               struct field_modify_info *dcopy,
 418                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 419                               uint32_t type, struct rte_flow_error *error)
 420 {
 421         uint32_t i = resource->actions_num;
 422         struct mlx5_modification_cmd *actions = resource->actions;
 423
 424         /*
 425          * The item and mask are provided in big-endian format.
 426          * The fields should be presented as in big-endian format either.
 427          * Mask must be always present, it defines the actual field width.
 428          */
 429         MLX5_ASSERT(item->mask);
 430         MLX5_ASSERT(field->size);
 431         do {
 432                 unsigned int size_b;
 433                 unsigned int off_b;
 434                 uint32_t mask;
 435                 uint32_t data;
 436
 437                 if (i >= MLX5_MAX_MODIFY_NUM)
 438                         return rte_flow_error_set(error, EINVAL,
 439                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 440                                  "too many items to modify");
 441                 /* Fetch variable byte size mask from the array. */
 442                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 443                                            field->offset, field->size);
 444                 if (!mask) {
 445                         ++field;
 446                         continue;
 447                 }
 448                 /* Deduce actual data width in bits from mask value. */
 449                 off_b = rte_bsf32(mask);
 450                 size_b = sizeof(uint32_t) * CHAR_BIT -
 451                          off_b - __builtin_clz(mask);
 452                 MLX5_ASSERT(size_b);
 453                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 454                 actions[i] = (struct mlx5_modification_cmd) {
 455                         .action_type = type,
 456                         .field = field->id,
 457                         .offset = off_b,
 458                         .length = size_b,
 459                 };
 460                 /* Convert entire record to expected big-endian format. */
 461                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 462                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 463                         MLX5_ASSERT(dcopy);
 464                         actions[i].dst_field = dcopy->id;
 465                         actions[i].dst_offset =
 466                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 467                         /* Convert entire record to big-endian format. */
 468                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 469                 } else {
 470                         MLX5_ASSERT(item->spec);
 471                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 472                                                    field->offset, field->size);
 473                         /* Shift out the trailing masked bits from data. */
 474                         data = (data & mask) >> off_b;
 475                         actions[i].data1 = rte_cpu_to_be_32(data);
 476                 }
 477                 ++i;
 478                 ++field;
 479         } while (field->size);
 480         if (resource->actions_num == i)
 481                 return rte_flow_error_set(error, EINVAL,
 482                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 483                                           "invalid modification flow item");
 484         resource->actions_num = i;
 485         return 0;
 486 }
 487
 488 /**
 489  * Convert modify-header set IPv4 address action to DV specification.
 490  *
 491  * @param[in,out] resource
 492  *   Pointer to the modify-header resource.
 493  * @param[in] action
 494  *   Pointer to action specification.
 495  * @param[out] error
 496  *   Pointer to the error structure.
 497  *
 498  * @return
 499  *   0 on success, a negative errno value otherwise and rte_errno is set.
 500  */
 501 static int
 502 flow_dv_convert_action_modify_ipv4
 503                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 504                          const struct rte_flow_action *action,
 505                          struct rte_flow_error *error)
 506 {
 507         const struct rte_flow_action_set_ipv4 *conf =
 508                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 509         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 510         struct rte_flow_item_ipv4 ipv4;
 511         struct rte_flow_item_ipv4 ipv4_mask;
 512
 513         memset(&ipv4, 0, sizeof(ipv4));
 514         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 515         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 516                 ipv4.hdr.src_addr = conf->ipv4_addr;
 517                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 518         } else {
 519                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 520                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 521         }
 522         item.spec = &ipv4;
 523         item.mask = &ipv4_mask;
 524         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 525                                              MLX5_MODIFICATION_TYPE_SET, error);
 526 }
 527
 528 /**
 529  * Convert modify-header set IPv6 address action to DV specification.
 530  *
 531  * @param[in,out] resource
 532  *   Pointer to the modify-header resource.
 533  * @param[in] action
 534  *   Pointer to action specification.
 535  * @param[out] error
 536  *   Pointer to the error structure.
 537  *
 538  * @return
 539  *   0 on success, a negative errno value otherwise and rte_errno is set.
 540  */
 541 static int
 542 flow_dv_convert_action_modify_ipv6
 543                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 544                          const struct rte_flow_action *action,
 545                          struct rte_flow_error *error)
 546 {
 547         const struct rte_flow_action_set_ipv6 *conf =
 548                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 549         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 550         struct rte_flow_item_ipv6 ipv6;
 551         struct rte_flow_item_ipv6 ipv6_mask;
 552
 553         memset(&ipv6, 0, sizeof(ipv6));
 554         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 555         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 556                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 557                        sizeof(ipv6.hdr.src_addr));
 558                 memcpy(&ipv6_mask.hdr.src_addr,
 559                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 560                        sizeof(ipv6.hdr.src_addr));
 561         } else {
 562                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 563                        sizeof(ipv6.hdr.dst_addr));
 564                 memcpy(&ipv6_mask.hdr.dst_addr,
 565                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 566                        sizeof(ipv6.hdr.dst_addr));
 567         }
 568         item.spec = &ipv6;
 569         item.mask = &ipv6_mask;
 570         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 571                                              MLX5_MODIFICATION_TYPE_SET, error);
 572 }
 573
 574 /**
 575  * Convert modify-header set MAC address action to DV specification.
 576  *
 577  * @param[in,out] resource
 578  *   Pointer to the modify-header resource.
 579  * @param[in] action
 580  *   Pointer to action specification.
 581  * @param[out] error
 582  *   Pointer to the error structure.
 583  *
 584  * @return
 585  *   0 on success, a negative errno value otherwise and rte_errno is set.
 586  */
 587 static int
 588 flow_dv_convert_action_modify_mac
 589                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 590                          const struct rte_flow_action *action,
 591                          struct rte_flow_error *error)
 592 {
 593         const struct rte_flow_action_set_mac *conf =
 594                 (const struct rte_flow_action_set_mac *)(action->conf);
 595         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 596         struct rte_flow_item_eth eth;
 597         struct rte_flow_item_eth eth_mask;
 598
 599         memset(&eth, 0, sizeof(eth));
 600         memset(&eth_mask, 0, sizeof(eth_mask));
 601         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 602                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 603                        sizeof(eth.src.addr_bytes));
 604                 memcpy(&eth_mask.src.addr_bytes,
 605                        &rte_flow_item_eth_mask.src.addr_bytes,
 606                        sizeof(eth_mask.src.addr_bytes));
 607         } else {
 608                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 609                        sizeof(eth.dst.addr_bytes));
 610                 memcpy(&eth_mask.dst.addr_bytes,
 611                        &rte_flow_item_eth_mask.dst.addr_bytes,
 612                        sizeof(eth_mask.dst.addr_bytes));
 613         }
 614         item.spec = &eth;
 615         item.mask = &eth_mask;
 616         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 617                                              MLX5_MODIFICATION_TYPE_SET, error);
 618 }
 619
 620 /**
 621  * Convert modify-header set VLAN VID action to DV specification.
 622  *
 623  * @param[in,out] resource
 624  *   Pointer to the modify-header resource.
 625  * @param[in] action
 626  *   Pointer to action specification.
 627  * @param[out] error
 628  *   Pointer to the error structure.
 629  *
 630  * @return
 631  *   0 on success, a negative errno value otherwise and rte_errno is set.
 632  */
 633 static int
 634 flow_dv_convert_action_modify_vlan_vid
 635                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 636                          const struct rte_flow_action *action,
 637                          struct rte_flow_error *error)
 638 {
 639         const struct rte_flow_action_of_set_vlan_vid *conf =
 640                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 641         int i = resource->actions_num;
 642         struct mlx5_modification_cmd *actions = resource->actions;
 643         struct field_modify_info *field = modify_vlan_out_first_vid;
 644
 645         if (i >= MLX5_MAX_MODIFY_NUM)
 646                 return rte_flow_error_set(error, EINVAL,
 647                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 648                          "too many items to modify");
 649         actions[i] = (struct mlx5_modification_cmd) {
 650                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 651                 .field = field->id,
 652                 .length = field->size,
 653                 .offset = field->offset,
 654         };
 655         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 656         actions[i].data1 = conf->vlan_vid;
 657         actions[i].data1 = actions[i].data1 << 16;
 658         resource->actions_num = ++i;
 659         return 0;
 660 }
 661
 662 /**
 663  * Convert modify-header set TP action to DV specification.
 664  *
 665  * @param[in,out] resource
 666  *   Pointer to the modify-header resource.
 667  * @param[in] action
 668  *   Pointer to action specification.
 669  * @param[in] items
 670  *   Pointer to rte_flow_item objects list.
 671  * @param[in] attr
 672  *   Pointer to flow attributes structure.
 673  * @param[in] dev_flow
 674  *   Pointer to the sub flow.
 675  * @param[in] tunnel_decap
 676  *   Whether action is after tunnel decapsulation.
 677  * @param[out] error
 678  *   Pointer to the error structure.
 679  *
 680  * @return
 681  *   0 on success, a negative errno value otherwise and rte_errno is set.
 682  */
 683 static int
 684 flow_dv_convert_action_modify_tp
 685                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 686                          const struct rte_flow_action *action,
 687                          const struct rte_flow_item *items,
 688                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 689                          bool tunnel_decap, struct rte_flow_error *error)
 690 {
 691         const struct rte_flow_action_set_tp *conf =
 692                 (const struct rte_flow_action_set_tp *)(action->conf);
 693         struct rte_flow_item item;
 694         struct rte_flow_item_udp udp;
 695         struct rte_flow_item_udp udp_mask;
 696         struct rte_flow_item_tcp tcp;
 697         struct rte_flow_item_tcp tcp_mask;
 698         struct field_modify_info *field;
 699
 700         if (!attr->valid)
 701                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 702         if (attr->udp) {
 703                 memset(&udp, 0, sizeof(udp));
 704                 memset(&udp_mask, 0, sizeof(udp_mask));
 705                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 706                         udp.hdr.src_port = conf->port;
 707                         udp_mask.hdr.src_port =
 708                                         rte_flow_item_udp_mask.hdr.src_port;
 709                 } else {
 710                         udp.hdr.dst_port = conf->port;
 711                         udp_mask.hdr.dst_port =
 712                                         rte_flow_item_udp_mask.hdr.dst_port;
 713                 }
 714                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 715                 item.spec = &udp;
 716                 item.mask = &udp_mask;
 717                 field = modify_udp;
 718         } else {
 719                 MLX5_ASSERT(attr->tcp);
 720                 memset(&tcp, 0, sizeof(tcp));
 721                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 722                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 723                         tcp.hdr.src_port = conf->port;
 724                         tcp_mask.hdr.src_port =
 725                                         rte_flow_item_tcp_mask.hdr.src_port;
 726                 } else {
 727                         tcp.hdr.dst_port = conf->port;
 728                         tcp_mask.hdr.dst_port =
 729                                         rte_flow_item_tcp_mask.hdr.dst_port;
 730                 }
 731                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 732                 item.spec = &tcp;
 733                 item.mask = &tcp_mask;
 734                 field = modify_tcp;
 735         }
 736         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 737                                              MLX5_MODIFICATION_TYPE_SET, error);
 738 }
 739
 740 /**
 741  * Convert modify-header set TTL action to DV specification.
 742  *
 743  * @param[in,out] resource
 744  *   Pointer to the modify-header resource.
 745  * @param[in] action
 746  *   Pointer to action specification.
 747  * @param[in] items
 748  *   Pointer to rte_flow_item objects list.
 749  * @param[in] attr
 750  *   Pointer to flow attributes structure.
 751  * @param[in] dev_flow
 752  *   Pointer to the sub flow.
 753  * @param[in] tunnel_decap
 754  *   Whether action is after tunnel decapsulation.
 755  * @param[out] error
 756  *   Pointer to the error structure.
 757  *
 758  * @return
 759  *   0 on success, a negative errno value otherwise and rte_errno is set.
 760  */
 761 static int
 762 flow_dv_convert_action_modify_ttl
 763                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 764                          const struct rte_flow_action *action,
 765                          const struct rte_flow_item *items,
 766                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 767                          bool tunnel_decap, struct rte_flow_error *error)
 768 {
 769         const struct rte_flow_action_set_ttl *conf =
 770                 (const struct rte_flow_action_set_ttl *)(action->conf);
 771         struct rte_flow_item item;
 772         struct rte_flow_item_ipv4 ipv4;
 773         struct rte_flow_item_ipv4 ipv4_mask;
 774         struct rte_flow_item_ipv6 ipv6;
 775         struct rte_flow_item_ipv6 ipv6_mask;
 776         struct field_modify_info *field;
 777
 778         if (!attr->valid)
 779                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 780         if (attr->ipv4) {
 781                 memset(&ipv4, 0, sizeof(ipv4));
 782                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 783                 ipv4.hdr.time_to_live = conf->ttl_value;
 784                 ipv4_mask.hdr.time_to_live = 0xFF;
 785                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 786                 item.spec = &ipv4;
 787                 item.mask = &ipv4_mask;
 788                 field = modify_ipv4;
 789         } else {
 790                 MLX5_ASSERT(attr->ipv6);
 791                 memset(&ipv6, 0, sizeof(ipv6));
 792                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 793                 ipv6.hdr.hop_limits = conf->ttl_value;
 794                 ipv6_mask.hdr.hop_limits = 0xFF;
 795                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 796                 item.spec = &ipv6;
 797                 item.mask = &ipv6_mask;
 798                 field = modify_ipv6;
 799         }
 800         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 801                                              MLX5_MODIFICATION_TYPE_SET, error);
 802 }
 803
 804 /**
 805  * Convert modify-header decrement TTL action to DV specification.
 806  *
 807  * @param[in,out] resource
 808  *   Pointer to the modify-header resource.
 809  * @param[in] action
 810  *   Pointer to action specification.
 811  * @param[in] items
 812  *   Pointer to rte_flow_item objects list.
 813  * @param[in] attr
 814  *   Pointer to flow attributes structure.
 815  * @param[in] dev_flow
 816  *   Pointer to the sub flow.
 817  * @param[in] tunnel_decap
 818  *   Whether action is after tunnel decapsulation.
 819  * @param[out] error
 820  *   Pointer to the error structure.
 821  *
 822  * @return
 823  *   0 on success, a negative errno value otherwise and rte_errno is set.
 824  */
 825 static int
 826 flow_dv_convert_action_modify_dec_ttl
 827                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 828                          const struct rte_flow_item *items,
 829                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 830                          bool tunnel_decap, struct rte_flow_error *error)
 831 {
 832         struct rte_flow_item item;
 833         struct rte_flow_item_ipv4 ipv4;
 834         struct rte_flow_item_ipv4 ipv4_mask;
 835         struct rte_flow_item_ipv6 ipv6;
 836         struct rte_flow_item_ipv6 ipv6_mask;
 837         struct field_modify_info *field;
 838
 839         if (!attr->valid)
 840                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 841         if (attr->ipv4) {
 842                 memset(&ipv4, 0, sizeof(ipv4));
 843                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 844                 ipv4.hdr.time_to_live = 0xFF;
 845                 ipv4_mask.hdr.time_to_live = 0xFF;
 846                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 847                 item.spec = &ipv4;
 848                 item.mask = &ipv4_mask;
 849                 field = modify_ipv4;
 850         } else {
 851                 MLX5_ASSERT(attr->ipv6);
 852                 memset(&ipv6, 0, sizeof(ipv6));
 853                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 854                 ipv6.hdr.hop_limits = 0xFF;
 855                 ipv6_mask.hdr.hop_limits = 0xFF;
 856                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 857                 item.spec = &ipv6;
 858                 item.mask = &ipv6_mask;
 859                 field = modify_ipv6;
 860         }
 861         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 862                                              MLX5_MODIFICATION_TYPE_ADD, error);
 863 }
 864
 865 /**
 866  * Convert modify-header increment/decrement TCP Sequence number
 867  * to DV specification.
 868  *
 869  * @param[in,out] resource
 870  *   Pointer to the modify-header resource.
 871  * @param[in] action
 872  *   Pointer to action specification.
 873  * @param[out] error
 874  *   Pointer to the error structure.
 875  *
 876  * @return
 877  *   0 on success, a negative errno value otherwise and rte_errno is set.
 878  */
 879 static int
 880 flow_dv_convert_action_modify_tcp_seq
 881                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 882                          const struct rte_flow_action *action,
 883                          struct rte_flow_error *error)
 884 {
 885         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 886         uint64_t value = rte_be_to_cpu_32(*conf);
 887         struct rte_flow_item item;
 888         struct rte_flow_item_tcp tcp;
 889         struct rte_flow_item_tcp tcp_mask;
 890
 891         memset(&tcp, 0, sizeof(tcp));
 892         memset(&tcp_mask, 0, sizeof(tcp_mask));
 893         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 894                 /*
 895                  * The HW has no decrement operation, only increment operation.
 896                  * To simulate decrement X from Y using increment operation
 897                  * we need to add UINT32_MAX X times to Y.
 898                  * Each adding of UINT32_MAX decrements Y by 1.
 899                  */
 900                 value *= UINT32_MAX;
 901         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 902         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 903         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 904         item.spec = &tcp;
 905         item.mask = &tcp_mask;
 906         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 907                                              MLX5_MODIFICATION_TYPE_ADD, error);
 908 }
 909
 910 /**
 911  * Convert modify-header increment/decrement TCP Acknowledgment number
 912  * to DV specification.
 913  *
 914  * @param[in,out] resource
 915  *   Pointer to the modify-header resource.
 916  * @param[in] action
 917  *   Pointer to action specification.
 918  * @param[out] error
 919  *   Pointer to the error structure.
 920  *
 921  * @return
 922  *   0 on success, a negative errno value otherwise and rte_errno is set.
 923  */
 924 static int
 925 flow_dv_convert_action_modify_tcp_ack
 926                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 927                          const struct rte_flow_action *action,
 928                          struct rte_flow_error *error)
 929 {
 930         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 931         uint64_t value = rte_be_to_cpu_32(*conf);
 932         struct rte_flow_item item;
 933         struct rte_flow_item_tcp tcp;
 934         struct rte_flow_item_tcp tcp_mask;
 935
 936         memset(&tcp, 0, sizeof(tcp));
 937         memset(&tcp_mask, 0, sizeof(tcp_mask));
 938         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 939                 /*
 940                  * The HW has no decrement operation, only increment operation.
 941                  * To simulate decrement X from Y using increment operation
 942                  * we need to add UINT32_MAX X times to Y.
 943                  * Each adding of UINT32_MAX decrements Y by 1.
 944                  */
 945                 value *= UINT32_MAX;
 946         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 947         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 948         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 949         item.spec = &tcp;
 950         item.mask = &tcp_mask;
 951         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 952                                              MLX5_MODIFICATION_TYPE_ADD, error);
 953 }
 954
 955 static enum mlx5_modification_field reg_to_field[] = {
 956         [REG_NON] = MLX5_MODI_OUT_NONE,
 957         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 958         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 959         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 960         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 961         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 962         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 963         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 964         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 965         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 966         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 967 };
 968
 969 /**
 970  * Convert register set to DV specification.
 971  *
 972  * @param[in,out] resource
 973  *   Pointer to the modify-header resource.
 974  * @param[in] action
 975  *   Pointer to action specification.
 976  * @param[out] error
 977  *   Pointer to the error structure.
 978  *
 979  * @return
 980  *   0 on success, a negative errno value otherwise and rte_errno is set.
 981  */
 982 static int
 983 flow_dv_convert_action_set_reg
 984                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 985                          const struct rte_flow_action *action,
 986                          struct rte_flow_error *error)
 987 {
 988         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 989         struct mlx5_modification_cmd *actions = resource->actions;
 990         uint32_t i = resource->actions_num;
 991
 992         if (i >= MLX5_MAX_MODIFY_NUM)
 993                 return rte_flow_error_set(error, EINVAL,
 994                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 995                                           "too many items to modify");
 996         MLX5_ASSERT(conf->id != REG_NON);
 997         MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field));
 998         actions[i] = (struct mlx5_modification_cmd) {
 999                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1000                 .field = reg_to_field[conf->id],
1001         };
1002         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1003         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1004         ++i;
1005         resource->actions_num = i;
1006         return 0;
1007 }
1008
1009 /**
1010  * Convert SET_TAG action to DV specification.
1011  *
1012  * @param[in] dev
1013  *   Pointer to the rte_eth_dev structure.
1014  * @param[in,out] resource
1015  *   Pointer to the modify-header resource.
1016  * @param[in] conf
1017  *   Pointer to action specification.
1018  * @param[out] error
1019  *   Pointer to the error structure.
1020  *
1021  * @return
1022  *   0 on success, a negative errno value otherwise and rte_errno is set.
1023  */
1024 static int
1025 flow_dv_convert_action_set_tag
1026                         (struct rte_eth_dev *dev,
1027                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1028                          const struct rte_flow_action_set_tag *conf,
1029                          struct rte_flow_error *error)
1030 {
1031         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1032         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1033         struct rte_flow_item item = {
1034                 .spec = &data,
1035                 .mask = &mask,
1036         };
1037         struct field_modify_info reg_c_x[] = {
1038                 [1] = {0, 0, 0},
1039         };
1040         enum mlx5_modification_field reg_type;
1041         int ret;
1042
1043         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1044         if (ret < 0)
1045                 return ret;
1046         MLX5_ASSERT(ret != REG_NON);
1047         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1048         reg_type = reg_to_field[ret];
1049         MLX5_ASSERT(reg_type > 0);
1050         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1051         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1052                                              MLX5_MODIFICATION_TYPE_SET, error);
1053 }
1054
1055 /**
1056  * Convert internal COPY_REG action to DV specification.
1057  *
1058  * @param[in] dev
1059  *   Pointer to the rte_eth_dev structure.
1060  * @param[in,out] res
1061  *   Pointer to the modify-header resource.
1062  * @param[in] action
1063  *   Pointer to action specification.
1064  * @param[out] error
1065  *   Pointer to the error structure.
1066  *
1067  * @return
1068  *   0 on success, a negative errno value otherwise and rte_errno is set.
1069  */
1070 static int
1071 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1072                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1073                                  const struct rte_flow_action *action,
1074                                  struct rte_flow_error *error)
1075 {
1076         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1077         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1078         struct rte_flow_item item = {
1079                 .spec = NULL,
1080                 .mask = &mask,
1081         };
1082         struct field_modify_info reg_src[] = {
1083                 {4, 0, reg_to_field[conf->src]},
1084                 {0, 0, 0},
1085         };
1086         struct field_modify_info reg_dst = {
1087                 .offset = 0,
1088                 .id = reg_to_field[conf->dst],
1089         };
1090         /* Adjust reg_c[0] usage according to reported mask. */
1091         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1092                 struct mlx5_priv *priv = dev->data->dev_private;
1093                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1094
1095                 MLX5_ASSERT(reg_c0);
1096                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1097                 if (conf->dst == REG_C_0) {
1098                         /* Copy to reg_c[0], within mask only. */
1099                         reg_dst.offset = rte_bsf32(reg_c0);
1100                         /*
1101                          * Mask is ignoring the enianness, because
1102                          * there is no conversion in datapath.
1103                          */
1104 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1105                         /* Copy from destination lower bits to reg_c[0]. */
1106                         mask = reg_c0 >> reg_dst.offset;
1107 #else
1108                         /* Copy from destination upper bits to reg_c[0]. */
1109                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1110                                           rte_fls_u32(reg_c0));
1111 #endif
1112                 } else {
1113                         mask = rte_cpu_to_be_32(reg_c0);
1114 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1115                         /* Copy from reg_c[0] to destination lower bits. */
1116                         reg_dst.offset = 0;
1117 #else
1118                         /* Copy from reg_c[0] to destination upper bits. */
1119                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1120                                          (rte_fls_u32(reg_c0) -
1121                                           rte_bsf32(reg_c0));
1122 #endif
1123                 }
1124         }
1125         return flow_dv_convert_modify_action(&item,
1126                                              reg_src, &reg_dst, res,
1127                                              MLX5_MODIFICATION_TYPE_COPY,
1128                                              error);
1129 }
1130
1131 /**
1132  * Convert MARK action to DV specification. This routine is used
1133  * in extensive metadata only and requires metadata register to be
1134  * handled. In legacy mode hardware tag resource is engaged.
1135  *
1136  * @param[in] dev
1137  *   Pointer to the rte_eth_dev structure.
1138  * @param[in] conf
1139  *   Pointer to MARK action specification.
1140  * @param[in,out] resource
1141  *   Pointer to the modify-header resource.
1142  * @param[out] error
1143  *   Pointer to the error structure.
1144  *
1145  * @return
1146  *   0 on success, a negative errno value otherwise and rte_errno is set.
1147  */
1148 static int
1149 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1150                             const struct rte_flow_action_mark *conf,
1151                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1152                             struct rte_flow_error *error)
1153 {
1154         struct mlx5_priv *priv = dev->data->dev_private;
1155         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1156                                            priv->sh->dv_mark_mask);
1157         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1158         struct rte_flow_item item = {
1159                 .spec = &data,
1160                 .mask = &mask,
1161         };
1162         struct field_modify_info reg_c_x[] = {
1163                 [1] = {0, 0, 0},
1164         };
1165         int reg;
1166
1167         if (!mask)
1168                 return rte_flow_error_set(error, EINVAL,
1169                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1170                                           NULL, "zero mark action mask");
1171         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1172         if (reg < 0)
1173                 return reg;
1174         MLX5_ASSERT(reg > 0);
1175         if (reg == REG_C_0) {
1176                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1177                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1178
1179                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1180                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1181                 mask = rte_cpu_to_be_32(mask << shl_c0);
1182         }
1183         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1184         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1185                                              MLX5_MODIFICATION_TYPE_SET, error);
1186 }
1187
1188 /**
1189  * Get metadata register index for specified steering domain.
1190  *
1191  * @param[in] dev
1192  *   Pointer to the rte_eth_dev structure.
1193  * @param[in] attr
1194  *   Attributes of flow to determine steering domain.
1195  * @param[out] error
1196  *   Pointer to the error structure.
1197  *
1198  * @return
1199  *   positive index on success, a negative errno value otherwise
1200  *   and rte_errno is set.
1201  */
1202 static enum modify_reg
1203 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1204                          const struct rte_flow_attr *attr,
1205                          struct rte_flow_error *error)
1206 {
1207         int reg =
1208                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1209                                           MLX5_METADATA_FDB :
1210                                             attr->egress ?
1211                                             MLX5_METADATA_TX :
1212                                             MLX5_METADATA_RX, 0, error);
1213         if (reg < 0)
1214                 return rte_flow_error_set(error,
1215                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1216                                           NULL, "unavailable "
1217                                           "metadata register");
1218         return reg;
1219 }
1220
1221 /**
1222  * Convert SET_META action to DV specification.
1223  *
1224  * @param[in] dev
1225  *   Pointer to the rte_eth_dev structure.
1226  * @param[in,out] resource
1227  *   Pointer to the modify-header resource.
1228  * @param[in] attr
1229  *   Attributes of flow that includes this item.
1230  * @param[in] conf
1231  *   Pointer to action specification.
1232  * @param[out] error
1233  *   Pointer to the error structure.
1234  *
1235  * @return
1236  *   0 on success, a negative errno value otherwise and rte_errno is set.
1237  */
1238 static int
1239 flow_dv_convert_action_set_meta
1240                         (struct rte_eth_dev *dev,
1241                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1242                          const struct rte_flow_attr *attr,
1243                          const struct rte_flow_action_set_meta *conf,
1244                          struct rte_flow_error *error)
1245 {
1246         uint32_t data = conf->data;
1247         uint32_t mask = conf->mask;
1248         struct rte_flow_item item = {
1249                 .spec = &data,
1250                 .mask = &mask,
1251         };
1252         struct field_modify_info reg_c_x[] = {
1253                 [1] = {0, 0, 0},
1254         };
1255         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1256
1257         if (reg < 0)
1258                 return reg;
1259         /*
1260          * In datapath code there is no endianness
1261          * coversions for perfromance reasons, all
1262          * pattern conversions are done in rte_flow.
1263          */
1264         if (reg == REG_C_0) {
1265                 struct mlx5_priv *priv = dev->data->dev_private;
1266                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1267                 uint32_t shl_c0;
1268
1269                 MLX5_ASSERT(msk_c0);
1270 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1271                 shl_c0 = rte_bsf32(msk_c0);
1272 #else
1273                 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1274 #endif
1275                 mask <<= shl_c0;
1276                 data <<= shl_c0;
1277                 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1278         }
1279         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1280         /* The routine expects parameters in memory as big-endian ones. */
1281         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1282                                              MLX5_MODIFICATION_TYPE_SET, error);
1283 }
1284
1285 /**
1286  * Convert modify-header set IPv4 DSCP action to DV specification.
1287  *
1288  * @param[in,out] resource
1289  *   Pointer to the modify-header resource.
1290  * @param[in] action
1291  *   Pointer to action specification.
1292  * @param[out] error
1293  *   Pointer to the error structure.
1294  *
1295  * @return
1296  *   0 on success, a negative errno value otherwise and rte_errno is set.
1297  */
1298 static int
1299 flow_dv_convert_action_modify_ipv4_dscp
1300                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1301                          const struct rte_flow_action *action,
1302                          struct rte_flow_error *error)
1303 {
1304         const struct rte_flow_action_set_dscp *conf =
1305                 (const struct rte_flow_action_set_dscp *)(action->conf);
1306         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1307         struct rte_flow_item_ipv4 ipv4;
1308         struct rte_flow_item_ipv4 ipv4_mask;
1309
1310         memset(&ipv4, 0, sizeof(ipv4));
1311         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1312         ipv4.hdr.type_of_service = conf->dscp;
1313         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1314         item.spec = &ipv4;
1315         item.mask = &ipv4_mask;
1316         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1317                                              MLX5_MODIFICATION_TYPE_SET, error);
1318 }
1319
1320 /**
1321  * Convert modify-header set IPv6 DSCP action to DV specification.
1322  *
1323  * @param[in,out] resource
1324  *   Pointer to the modify-header resource.
1325  * @param[in] action
1326  *   Pointer to action specification.
1327  * @param[out] error
1328  *   Pointer to the error structure.
1329  *
1330  * @return
1331  *   0 on success, a negative errno value otherwise and rte_errno is set.
1332  */
1333 static int
1334 flow_dv_convert_action_modify_ipv6_dscp
1335                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1336                          const struct rte_flow_action *action,
1337                          struct rte_flow_error *error)
1338 {
1339         const struct rte_flow_action_set_dscp *conf =
1340                 (const struct rte_flow_action_set_dscp *)(action->conf);
1341         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1342         struct rte_flow_item_ipv6 ipv6;
1343         struct rte_flow_item_ipv6 ipv6_mask;
1344
1345         memset(&ipv6, 0, sizeof(ipv6));
1346         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1347         /*
1348          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1349          * rdma-core only accept the DSCP bits byte aligned start from
1350          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1351          * bits in IPv6 case as rdma-core requires byte aligned value.
1352          */
1353         ipv6.hdr.vtc_flow = conf->dscp;
1354         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1355         item.spec = &ipv6;
1356         item.mask = &ipv6_mask;
1357         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1358                                              MLX5_MODIFICATION_TYPE_SET, error);
1359 }
1360
1361 /**
1362  * Validate MARK item.
1363  *
1364  * @param[in] dev
1365  *   Pointer to the rte_eth_dev structure.
1366  * @param[in] item
1367  *   Item specification.
1368  * @param[in] attr
1369  *   Attributes of flow that includes this item.
1370  * @param[out] error
1371  *   Pointer to error structure.
1372  *
1373  * @return
1374  *   0 on success, a negative errno value otherwise and rte_errno is set.
1375  */
1376 static int
1377 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1378                            const struct rte_flow_item *item,
1379                            const struct rte_flow_attr *attr __rte_unused,
1380                            struct rte_flow_error *error)
1381 {
1382         struct mlx5_priv *priv = dev->data->dev_private;
1383         struct mlx5_dev_config *config = &priv->config;
1384         const struct rte_flow_item_mark *spec = item->spec;
1385         const struct rte_flow_item_mark *mask = item->mask;
1386         const struct rte_flow_item_mark nic_mask = {
1387                 .id = priv->sh->dv_mark_mask,
1388         };
1389         int ret;
1390
1391         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1392                 return rte_flow_error_set(error, ENOTSUP,
1393                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1394                                           "extended metadata feature"
1395                                           " isn't enabled");
1396         if (!mlx5_flow_ext_mreg_supported(dev))
1397                 return rte_flow_error_set(error, ENOTSUP,
1398                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1399                                           "extended metadata register"
1400                                           " isn't supported");
1401         if (!nic_mask.id)
1402                 return rte_flow_error_set(error, ENOTSUP,
1403                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1404                                           "extended metadata register"
1405                                           " isn't available");
1406         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1407         if (ret < 0)
1408                 return ret;
1409         if (!spec)
1410                 return rte_flow_error_set(error, EINVAL,
1411                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1412                                           item->spec,
1413                                           "data cannot be empty");
1414         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1415                 return rte_flow_error_set(error, EINVAL,
1416                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1417                                           &spec->id,
1418                                           "mark id exceeds the limit");
1419         if (!mask)
1420                 mask = &nic_mask;
1421         if (!mask->id)
1422                 return rte_flow_error_set(error, EINVAL,
1423                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1424                                         "mask cannot be zero");
1425
1426         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1427                                         (const uint8_t *)&nic_mask,
1428                                         sizeof(struct rte_flow_item_mark),
1429                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1430         if (ret < 0)
1431                 return ret;
1432         return 0;
1433 }
1434
1435 /**
1436  * Validate META item.
1437  *
1438  * @param[in] dev
1439  *   Pointer to the rte_eth_dev structure.
1440  * @param[in] item
1441  *   Item specification.
1442  * @param[in] attr
1443  *   Attributes of flow that includes this item.
1444  * @param[out] error
1445  *   Pointer to error structure.
1446  *
1447  * @return
1448  *   0 on success, a negative errno value otherwise and rte_errno is set.
1449  */
1450 static int
1451 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1452                            const struct rte_flow_item *item,
1453                            const struct rte_flow_attr *attr,
1454                            struct rte_flow_error *error)
1455 {
1456         struct mlx5_priv *priv = dev->data->dev_private;
1457         struct mlx5_dev_config *config = &priv->config;
1458         const struct rte_flow_item_meta *spec = item->spec;
1459         const struct rte_flow_item_meta *mask = item->mask;
1460         struct rte_flow_item_meta nic_mask = {
1461                 .data = UINT32_MAX
1462         };
1463         int reg;
1464         int ret;
1465
1466         if (!spec)
1467                 return rte_flow_error_set(error, EINVAL,
1468                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1469                                           item->spec,
1470                                           "data cannot be empty");
1471         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1472                 if (!mlx5_flow_ext_mreg_supported(dev))
1473                         return rte_flow_error_set(error, ENOTSUP,
1474                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1475                                           "extended metadata register"
1476                                           " isn't supported");
1477                 reg = flow_dv_get_metadata_reg(dev, attr, error);
1478                 if (reg < 0)
1479                         return reg;
1480                 if (reg == REG_B)
1481                         return rte_flow_error_set(error, ENOTSUP,
1482                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1483                                           "match on reg_b "
1484                                           "isn't supported");
1485                 if (reg != REG_A)
1486                         nic_mask.data = priv->sh->dv_meta_mask;
1487         } else if (attr->transfer) {
1488                 return rte_flow_error_set(error, ENOTSUP,
1489                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1490                                         "extended metadata feature "
1491                                         "should be enabled when "
1492                                         "meta item is requested "
1493                                         "with e-switch mode ");
1494         }
1495         if (!mask)
1496                 mask = &rte_flow_item_meta_mask;
1497         if (!mask->data)
1498                 return rte_flow_error_set(error, EINVAL,
1499                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1500                                         "mask cannot be zero");
1501
1502         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1503                                         (const uint8_t *)&nic_mask,
1504                                         sizeof(struct rte_flow_item_meta),
1505                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1506         return ret;
1507 }
1508
1509 /**
1510  * Validate TAG item.
1511  *
1512  * @param[in] dev
1513  *   Pointer to the rte_eth_dev structure.
1514  * @param[in] item
1515  *   Item specification.
1516  * @param[in] attr
1517  *   Attributes of flow that includes this item.
1518  * @param[out] error
1519  *   Pointer to error structure.
1520  *
1521  * @return
1522  *   0 on success, a negative errno value otherwise and rte_errno is set.
1523  */
1524 static int
1525 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1526                           const struct rte_flow_item *item,
1527                           const struct rte_flow_attr *attr __rte_unused,
1528                           struct rte_flow_error *error)
1529 {
1530         const struct rte_flow_item_tag *spec = item->spec;
1531         const struct rte_flow_item_tag *mask = item->mask;
1532         const struct rte_flow_item_tag nic_mask = {
1533                 .data = RTE_BE32(UINT32_MAX),
1534                 .index = 0xff,
1535         };
1536         int ret;
1537
1538         if (!mlx5_flow_ext_mreg_supported(dev))
1539                 return rte_flow_error_set(error, ENOTSUP,
1540                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1541                                           "extensive metadata register"
1542                                           " isn't supported");
1543         if (!spec)
1544                 return rte_flow_error_set(error, EINVAL,
1545                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1546                                           item->spec,
1547                                           "data cannot be empty");
1548         if (!mask)
1549                 mask = &rte_flow_item_tag_mask;
1550         if (!mask->data)
1551                 return rte_flow_error_set(error, EINVAL,
1552                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1553                                         "mask cannot be zero");
1554
1555         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1556                                         (const uint8_t *)&nic_mask,
1557                                         sizeof(struct rte_flow_item_tag),
1558                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1559         if (ret < 0)
1560                 return ret;
1561         if (mask->index != 0xff)
1562                 return rte_flow_error_set(error, EINVAL,
1563                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1564                                           "partial mask for tag index"
1565                                           " is not supported");
1566         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1567         if (ret < 0)
1568                 return ret;
1569         MLX5_ASSERT(ret != REG_NON);
1570         return 0;
1571 }
1572
1573 /**
1574  * Validate vport item.
1575  *
1576  * @param[in] dev
1577  *   Pointer to the rte_eth_dev structure.
1578  * @param[in] item
1579  *   Item specification.
1580  * @param[in] attr
1581  *   Attributes of flow that includes this item.
1582  * @param[in] item_flags
1583  *   Bit-fields that holds the items detected until now.
1584  * @param[out] error
1585  *   Pointer to error structure.
1586  *
1587  * @return
1588  *   0 on success, a negative errno value otherwise and rte_errno is set.
1589  */
1590 static int
1591 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1592                               const struct rte_flow_item *item,
1593                               const struct rte_flow_attr *attr,
1594                               uint64_t item_flags,
1595                               struct rte_flow_error *error)
1596 {
1597         const struct rte_flow_item_port_id *spec = item->spec;
1598         const struct rte_flow_item_port_id *mask = item->mask;
1599         const struct rte_flow_item_port_id switch_mask = {
1600                         .id = 0xffffffff,
1601         };
1602         struct mlx5_priv *esw_priv;
1603         struct mlx5_priv *dev_priv;
1604         int ret;
1605
1606         if (!attr->transfer)
1607                 return rte_flow_error_set(error, EINVAL,
1608                                           RTE_FLOW_ERROR_TYPE_ITEM,
1609                                           NULL,
1610                                           "match on port id is valid only"
1611                                           " when transfer flag is enabled");
1612         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1613                 return rte_flow_error_set(error, ENOTSUP,
1614                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1615                                           "multiple source ports are not"
1616                                           " supported");
1617         if (!mask)
1618                 mask = &switch_mask;
1619         if (mask->id != 0xffffffff)
1620                 return rte_flow_error_set(error, ENOTSUP,
1621                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1622                                            mask,
1623                                            "no support for partial mask on"
1624                                            " \"id\" field");
1625         ret = mlx5_flow_item_acceptable
1626                                 (item, (const uint8_t *)mask,
1627                                  (const uint8_t *)&rte_flow_item_port_id_mask,
1628                                  sizeof(struct rte_flow_item_port_id),
1629                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1630         if (ret)
1631                 return ret;
1632         if (!spec)
1633                 return 0;
1634         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1635         if (!esw_priv)
1636                 return rte_flow_error_set(error, rte_errno,
1637                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1638                                           "failed to obtain E-Switch info for"
1639                                           " port");
1640         dev_priv = mlx5_dev_to_eswitch_info(dev);
1641         if (!dev_priv)
1642                 return rte_flow_error_set(error, rte_errno,
1643                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1644                                           NULL,
1645                                           "failed to obtain E-Switch info");
1646         if (esw_priv->domain_id != dev_priv->domain_id)
1647                 return rte_flow_error_set(error, EINVAL,
1648                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1649                                           "cannot match on a port from a"
1650                                           " different E-Switch");
1651         return 0;
1652 }
1653
1654 /**
1655  * Validate VLAN item.
1656  *
1657  * @param[in] item
1658  *   Item specification.
1659  * @param[in] item_flags
1660  *   Bit-fields that holds the items detected until now.
1661  * @param[in] dev
1662  *   Ethernet device flow is being created on.
1663  * @param[out] error
1664  *   Pointer to error structure.
1665  *
1666  * @return
1667  *   0 on success, a negative errno value otherwise and rte_errno is set.
1668  */
1669 static int
1670 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
1671                            uint64_t item_flags,
1672                            struct rte_eth_dev *dev,
1673                            struct rte_flow_error *error)
1674 {
1675         const struct rte_flow_item_vlan *mask = item->mask;
1676         const struct rte_flow_item_vlan nic_mask = {
1677                 .tci = RTE_BE16(UINT16_MAX),
1678                 .inner_type = RTE_BE16(UINT16_MAX),
1679         };
1680         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1681         int ret;
1682         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1683                                         MLX5_FLOW_LAYER_INNER_L4) :
1684                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1685                                         MLX5_FLOW_LAYER_OUTER_L4);
1686         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1687                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1688
1689         if (item_flags & vlanm)
1690                 return rte_flow_error_set(error, EINVAL,
1691                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1692                                           "multiple VLAN layers not supported");
1693         else if ((item_flags & l34m) != 0)
1694                 return rte_flow_error_set(error, EINVAL,
1695                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1696                                           "VLAN cannot follow L3/L4 layer");
1697         if (!mask)
1698                 mask = &rte_flow_item_vlan_mask;
1699         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1700                                         (const uint8_t *)&nic_mask,
1701                                         sizeof(struct rte_flow_item_vlan),
1702                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1703         if (ret)
1704                 return ret;
1705         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1706                 struct mlx5_priv *priv = dev->data->dev_private;
1707
1708                 if (priv->vmwa_context) {
1709                         /*
1710                          * Non-NULL context means we have a virtual machine
1711                          * and SR-IOV enabled, we have to create VLAN interface
1712                          * to make hypervisor to setup E-Switch vport
1713                          * context correctly. We avoid creating the multiple
1714                          * VLAN interfaces, so we cannot support VLAN tag mask.
1715                          */
1716                         return rte_flow_error_set(error, EINVAL,
1717                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1718                                                   item,
1719                                                   "VLAN tag mask is not"
1720                                                   " supported in virtual"
1721                                                   " environment");
1722                 }
1723         }
1724         return 0;
1725 }
1726
1727 /*
1728  * GTP flags are contained in 1 byte of the format:
1729  * -------------------------------------------
1730  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
1731  * |-----------------------------------------|
1732  * | value | Version | PT | Res | E | S | PN |
1733  * -------------------------------------------
1734  *
1735  * Matching is supported only for GTP flags E, S, PN.
1736  */
1737 #define MLX5_GTP_FLAGS_MASK     0x07
1738
1739 /**
1740  * Validate GTP item.
1741  *
1742  * @param[in] dev
1743  *   Pointer to the rte_eth_dev structure.
1744  * @param[in] item
1745  *   Item specification.
1746  * @param[in] item_flags
1747  *   Bit-fields that holds the items detected until now.
1748  * @param[out] error
1749  *   Pointer to error structure.
1750  *
1751  * @return
1752  *   0 on success, a negative errno value otherwise and rte_errno is set.
1753  */
1754 static int
1755 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
1756                           const struct rte_flow_item *item,
1757                           uint64_t item_flags,
1758                           struct rte_flow_error *error)
1759 {
1760         struct mlx5_priv *priv = dev->data->dev_private;
1761         const struct rte_flow_item_gtp *spec = item->spec;
1762         const struct rte_flow_item_gtp *mask = item->mask;
1763         const struct rte_flow_item_gtp nic_mask = {
1764                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
1765                 .msg_type = 0xff,
1766                 .teid = RTE_BE32(0xffffffff),
1767         };
1768
1769         if (!priv->config.hca_attr.tunnel_stateless_gtp)
1770                 return rte_flow_error_set(error, ENOTSUP,
1771                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1772                                           "GTP support is not enabled");
1773         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1774                 return rte_flow_error_set(error, ENOTSUP,
1775                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1776                                           "multiple tunnel layers not"
1777                                           " supported");
1778         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1779                 return rte_flow_error_set(error, EINVAL,
1780                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1781                                           "no outer UDP layer found");
1782         if (!mask)
1783                 mask = &rte_flow_item_gtp_mask;
1784         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
1785                 return rte_flow_error_set(error, ENOTSUP,
1786                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1787                                           "Match is supported for GTP"
1788                                           " flags only");
1789         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1790                                          (const uint8_t *)&nic_mask,
1791                                          sizeof(struct rte_flow_item_gtp),
1792                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1793 }
1794
1795 /**
1796  * Validate IPV4 item.
1797  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
1798  * add specific validation of fragment_offset field,
1799  *
1800  * @param[in] item
1801  *   Item specification.
1802  * @param[in] item_flags
1803  *   Bit-fields that holds the items detected until now.
1804  * @param[out] error
1805  *   Pointer to error structure.
1806  *
1807  * @return
1808  *   0 on success, a negative errno value otherwise and rte_errno is set.
1809  */
1810 static int
1811 flow_dv_validate_item_ipv4(const struct rte_flow_item *item,
1812                            uint64_t item_flags,
1813                            uint64_t last_item,
1814                            uint16_t ether_type,
1815                            struct rte_flow_error *error)
1816 {
1817         int ret;
1818         const struct rte_flow_item_ipv4 *spec = item->spec;
1819         const struct rte_flow_item_ipv4 *last = item->last;
1820         const struct rte_flow_item_ipv4 *mask = item->mask;
1821         rte_be16_t fragment_offset_spec = 0;
1822         rte_be16_t fragment_offset_last = 0;
1823         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
1824                 .hdr = {
1825                         .src_addr = RTE_BE32(0xffffffff),
1826                         .dst_addr = RTE_BE32(0xffffffff),
1827                         .type_of_service = 0xff,
1828                         .fragment_offset = RTE_BE16(0xffff),
1829                         .next_proto_id = 0xff,
1830                         .time_to_live = 0xff,
1831                 },
1832         };
1833
1834         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
1835                                            ether_type, &nic_ipv4_mask,
1836                                            MLX5_ITEM_RANGE_ACCEPTED, error);
1837         if (ret < 0)
1838                 return ret;
1839         if (spec && mask)
1840                 fragment_offset_spec = spec->hdr.fragment_offset &
1841                                        mask->hdr.fragment_offset;
1842         if (!fragment_offset_spec)
1843                 return 0;
1844         /*
1845          * spec and mask are valid, enforce using full mask to make sure the
1846          * complete value is used correctly.
1847          */
1848         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1849                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1850                 return rte_flow_error_set(error, EINVAL,
1851                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1852                                           item, "must use full mask for"
1853                                           " fragment_offset");
1854         /*
1855          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
1856          * indicating this is 1st fragment of fragmented packet.
1857          * This is not yet supported in MLX5, return appropriate error message.
1858          */
1859         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
1860                 return rte_flow_error_set(error, ENOTSUP,
1861                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1862                                           "match on first fragment not "
1863                                           "supported");
1864         if (fragment_offset_spec && !last)
1865                 return rte_flow_error_set(error, ENOTSUP,
1866                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1867                                           "specified value not supported");
1868         /* spec and last are valid, validate the specified range. */
1869         fragment_offset_last = last->hdr.fragment_offset &
1870                                mask->hdr.fragment_offset;
1871         /*
1872          * Match on fragment_offset spec 0x2001 and last 0x3fff
1873          * means MF is 1 and frag-offset is > 0.
1874          * This packet is fragment 2nd and onward, excluding last.
1875          * This is not yet supported in MLX5, return appropriate
1876          * error message.
1877          */
1878         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
1879             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1880                 return rte_flow_error_set(error, ENOTSUP,
1881                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1882                                           last, "match on following "
1883                                           "fragments not supported");
1884         /*
1885          * Match on fragment_offset spec 0x0001 and last 0x1fff
1886          * means MF is 0 and frag-offset is > 0.
1887          * This packet is last fragment of fragmented packet.
1888          * This is not yet supported in MLX5, return appropriate
1889          * error message.
1890          */
1891         if (fragment_offset_spec == RTE_BE16(1) &&
1892             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
1893                 return rte_flow_error_set(error, ENOTSUP,
1894                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1895                                           last, "match on last "
1896                                           "fragment not supported");
1897         /*
1898          * Match on fragment_offset spec 0x0001 and last 0x3fff
1899          * means MF and/or frag-offset is not 0.
1900          * This is a fragmented packet.
1901          * Other range values are invalid and rejected.
1902          */
1903         if (!(fragment_offset_spec == RTE_BE16(1) &&
1904               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
1905                 return rte_flow_error_set(error, ENOTSUP,
1906                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
1907                                           "specified range not supported");
1908         return 0;
1909 }
1910
1911 /**
1912  * Validate IPV6 fragment extension item.
1913  *
1914  * @param[in] item
1915  *   Item specification.
1916  * @param[in] item_flags
1917  *   Bit-fields that holds the items detected until now.
1918  * @param[out] error
1919  *   Pointer to error structure.
1920  *
1921  * @return
1922  *   0 on success, a negative errno value otherwise and rte_errno is set.
1923  */
1924 static int
1925 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
1926                                     uint64_t item_flags,
1927                                     struct rte_flow_error *error)
1928 {
1929         const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
1930         const struct rte_flow_item_ipv6_frag_ext *last = item->last;
1931         const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
1932         rte_be16_t frag_data_spec = 0;
1933         rte_be16_t frag_data_last = 0;
1934         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1935         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1936                                       MLX5_FLOW_LAYER_OUTER_L4;
1937         int ret = 0;
1938         struct rte_flow_item_ipv6_frag_ext nic_mask = {
1939                 .hdr = {
1940                         .next_header = 0xff,
1941                         .frag_data = RTE_BE16(0xffff),
1942                 },
1943         };
1944
1945         if (item_flags & l4m)
1946                 return rte_flow_error_set(error, EINVAL,
1947                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1948                                           "ipv6 fragment extension item cannot "
1949                                           "follow L4 item.");
1950         if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
1951             (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
1952                 return rte_flow_error_set(error, EINVAL,
1953                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1954                                           "ipv6 fragment extension item must "
1955                                           "follow ipv6 item");
1956         if (spec && mask)
1957                 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
1958         if (!frag_data_spec)
1959                 return 0;
1960         /*
1961          * spec and mask are valid, enforce using full mask to make sure the
1962          * complete value is used correctly.
1963          */
1964         if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
1965                                 RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
1966                 return rte_flow_error_set(error, EINVAL,
1967                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1968                                           item, "must use full mask for"
1969                                           " frag_data");
1970         /*
1971          * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
1972          * This is 1st fragment of fragmented packet.
1973          */
1974         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
1975                 return rte_flow_error_set(error, ENOTSUP,
1976                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1977                                           "match on first fragment not "
1978                                           "supported");
1979         if (frag_data_spec && !last)
1980                 return rte_flow_error_set(error, EINVAL,
1981                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1982                                           "specified value not supported");
1983         ret = mlx5_flow_item_acceptable
1984                                 (item, (const uint8_t *)mask,
1985                                  (const uint8_t *)&nic_mask,
1986                                  sizeof(struct rte_flow_item_ipv6_frag_ext),
1987                                  MLX5_ITEM_RANGE_ACCEPTED, error);
1988         if (ret)
1989                 return ret;
1990         /* spec and last are valid, validate the specified range. */
1991         frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
1992         /*
1993          * Match on frag_data spec 0x0009 and last 0xfff9
1994          * means M is 1 and frag-offset is > 0.
1995          * This packet is fragment 2nd and onward, excluding last.
1996          * This is not yet supported in MLX5, return appropriate
1997          * error message.
1998          */
1999         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
2000                                        RTE_IPV6_EHDR_MF_MASK) &&
2001             frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2002                 return rte_flow_error_set(error, ENOTSUP,
2003                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2004                                           last, "match on following "
2005                                           "fragments not supported");
2006         /*
2007          * Match on frag_data spec 0x0008 and last 0xfff8
2008          * means M is 0 and frag-offset is > 0.
2009          * This packet is last fragment of fragmented packet.
2010          * This is not yet supported in MLX5, return appropriate
2011          * error message.
2012          */
2013         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
2014             frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
2015                 return rte_flow_error_set(error, ENOTSUP,
2016                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2017                                           last, "match on last "
2018                                           "fragment not supported");
2019         /* Other range values are invalid and rejected. */
2020         return rte_flow_error_set(error, EINVAL,
2021                                   RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2022                                   "specified range not supported");
2023 }
2024
2025 /**
2026  * Validate the pop VLAN action.
2027  *
2028  * @param[in] dev
2029  *   Pointer to the rte_eth_dev structure.
2030  * @param[in] action_flags
2031  *   Holds the actions detected until now.
2032  * @param[in] action
2033  *   Pointer to the pop vlan action.
2034  * @param[in] item_flags
2035  *   The items found in this flow rule.
2036  * @param[in] attr
2037  *   Pointer to flow attributes.
2038  * @param[out] error
2039  *   Pointer to error structure.
2040  *
2041  * @return
2042  *   0 on success, a negative errno value otherwise and rte_errno is set.
2043  */
2044 static int
2045 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
2046                                  uint64_t action_flags,
2047                                  const struct rte_flow_action *action,
2048                                  uint64_t item_flags,
2049                                  const struct rte_flow_attr *attr,
2050                                  struct rte_flow_error *error)
2051 {
2052         const struct mlx5_priv *priv = dev->data->dev_private;
2053
2054         (void)action;
2055         (void)attr;
2056         if (!priv->sh->pop_vlan_action)
2057                 return rte_flow_error_set(error, ENOTSUP,
2058                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2059                                           NULL,
2060                                           "pop vlan action is not supported");
2061         if (attr->egress)
2062                 return rte_flow_error_set(error, ENOTSUP,
2063                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2064                                           NULL,
2065                                           "pop vlan action not supported for "
2066                                           "egress");
2067         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
2068                 return rte_flow_error_set(error, ENOTSUP,
2069                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2070                                           "no support for multiple VLAN "
2071                                           "actions");
2072         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
2073         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
2074             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
2075                 return rte_flow_error_set(error, ENOTSUP,
2076                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2077                                           NULL,
2078                                           "cannot pop vlan after decap without "
2079                                           "match on inner vlan in the flow");
2080         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
2081         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
2082             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2083                 return rte_flow_error_set(error, ENOTSUP,
2084                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2085                                           NULL,
2086                                           "cannot pop vlan without a "
2087                                           "match on (outer) vlan in the flow");
2088         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2089                 return rte_flow_error_set(error, EINVAL,
2090                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2091                                           "wrong action order, port_id should "
2092                                           "be after pop VLAN action");
2093         if (!attr->transfer && priv->representor)
2094                 return rte_flow_error_set(error, ENOTSUP,
2095                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2096                                           "pop vlan action for VF representor "
2097                                           "not supported on NIC table");
2098         return 0;
2099 }
2100
2101 /**
2102  * Get VLAN default info from vlan match info.
2103  *
2104  * @param[in] items
2105  *   the list of item specifications.
2106  * @param[out] vlan
2107  *   pointer VLAN info to fill to.
2108  *
2109  * @return
2110  *   0 on success, a negative errno value otherwise and rte_errno is set.
2111  */
2112 static void
2113 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2114                                   struct rte_vlan_hdr *vlan)
2115 {
2116         const struct rte_flow_item_vlan nic_mask = {
2117                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2118                                 MLX5DV_FLOW_VLAN_VID_MASK),
2119                 .inner_type = RTE_BE16(0xffff),
2120         };
2121
2122         if (items == NULL)
2123                 return;
2124         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2125                 int type = items->type;
2126
2127                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2128                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2129                         break;
2130         }
2131         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2132                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2133                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2134
2135                 /* If VLAN item in pattern doesn't contain data, return here. */
2136                 if (!vlan_v)
2137                         return;
2138                 if (!vlan_m)
2139                         vlan_m = &nic_mask;
2140                 /* Only full match values are accepted */
2141                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2142                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2143                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2144                         vlan->vlan_tci |=
2145                                 rte_be_to_cpu_16(vlan_v->tci &
2146                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2147                 }
2148                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2149                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2150                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2151                         vlan->vlan_tci |=
2152                                 rte_be_to_cpu_16(vlan_v->tci &
2153                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2154                 }
2155                 if (vlan_m->inner_type == nic_mask.inner_type)
2156                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2157                                                            vlan_m->inner_type);
2158         }
2159 }
2160
2161 /**
2162  * Validate the push VLAN action.
2163  *
2164  * @param[in] dev
2165  *   Pointer to the rte_eth_dev structure.
2166  * @param[in] action_flags
2167  *   Holds the actions detected until now.
2168  * @param[in] item_flags
2169  *   The items found in this flow rule.
2170  * @param[in] action
2171  *   Pointer to the action structure.
2172  * @param[in] attr
2173  *   Pointer to flow attributes
2174  * @param[out] error
2175  *   Pointer to error structure.
2176  *
2177  * @return
2178  *   0 on success, a negative errno value otherwise and rte_errno is set.
2179  */
2180 static int
2181 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2182                                   uint64_t action_flags,
2183                                   const struct rte_flow_item_vlan *vlan_m,
2184                                   const struct rte_flow_action *action,
2185                                   const struct rte_flow_attr *attr,
2186                                   struct rte_flow_error *error)
2187 {
2188         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2189         const struct mlx5_priv *priv = dev->data->dev_private;
2190
2191         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2192             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2193                 return rte_flow_error_set(error, EINVAL,
2194                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2195                                           "invalid vlan ethertype");
2196         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2197                 return rte_flow_error_set(error, EINVAL,
2198                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2199                                           "wrong action order, port_id should "
2200                                           "be after push VLAN");
2201         if (!attr->transfer && priv->representor)
2202                 return rte_flow_error_set(error, ENOTSUP,
2203                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2204                                           "push vlan action for VF representor "
2205                                           "not supported on NIC table");
2206         if (vlan_m &&
2207             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2208             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2209                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2210             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2211             !(mlx5_flow_find_action
2212                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2213                 return rte_flow_error_set(error, EINVAL,
2214                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2215                                           "not full match mask on VLAN PCP and "
2216                                           "there is no of_set_vlan_pcp action, "
2217                                           "push VLAN action cannot figure out "
2218                                           "PCP value");
2219         if (vlan_m &&
2220             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2221             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2222                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2223             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2224             !(mlx5_flow_find_action
2225                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2226                 return rte_flow_error_set(error, EINVAL,
2227                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2228                                           "not full match mask on VLAN VID and "
2229                                           "there is no of_set_vlan_vid action, "
2230                                           "push VLAN action cannot figure out "
2231                                           "VID value");
2232         (void)attr;
2233         return 0;
2234 }
2235
2236 /**
2237  * Validate the set VLAN PCP.
2238  *
2239  * @param[in] action_flags
2240  *   Holds the actions detected until now.
2241  * @param[in] actions
2242  *   Pointer to the list of actions remaining in the flow rule.
2243  * @param[out] error
2244  *   Pointer to error structure.
2245  *
2246  * @return
2247  *   0 on success, a negative errno value otherwise and rte_errno is set.
2248  */
2249 static int
2250 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2251                                      const struct rte_flow_action actions[],
2252                                      struct rte_flow_error *error)
2253 {
2254         const struct rte_flow_action *action = actions;
2255         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2256
2257         if (conf->vlan_pcp > 7)
2258                 return rte_flow_error_set(error, EINVAL,
2259                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2260                                           "VLAN PCP value is too big");
2261         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2262                 return rte_flow_error_set(error, ENOTSUP,
2263                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2264                                           "set VLAN PCP action must follow "
2265                                           "the push VLAN action");
2266         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2267                 return rte_flow_error_set(error, ENOTSUP,
2268                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2269                                           "Multiple VLAN PCP modification are "
2270                                           "not supported");
2271         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2272                 return rte_flow_error_set(error, EINVAL,
2273                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2274                                           "wrong action order, port_id should "
2275                                           "be after set VLAN PCP");
2276         return 0;
2277 }
2278
2279 /**
2280  * Validate the set VLAN VID.
2281  *
2282  * @param[in] item_flags
2283  *   Holds the items detected in this rule.
2284  * @param[in] action_flags
2285  *   Holds the actions detected until now.
2286  * @param[in] actions
2287  *   Pointer to the list of actions remaining in the flow rule.
2288  * @param[out] error
2289  *   Pointer to error structure.
2290  *
2291  * @return
2292  *   0 on success, a negative errno value otherwise and rte_errno is set.
2293  */
2294 static int
2295 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2296                                      uint64_t action_flags,
2297                                      const struct rte_flow_action actions[],
2298                                      struct rte_flow_error *error)
2299 {
2300         const struct rte_flow_action *action = actions;
2301         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2302
2303         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2304                 return rte_flow_error_set(error, EINVAL,
2305                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2306                                           "VLAN VID value is too big");
2307         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2308             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2309                 return rte_flow_error_set(error, ENOTSUP,
2310                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2311                                           "set VLAN VID action must follow push"
2312                                           " VLAN action or match on VLAN item");
2313         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2314                 return rte_flow_error_set(error, ENOTSUP,
2315                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2316                                           "Multiple VLAN VID modifications are "
2317                                           "not supported");
2318         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2319                 return rte_flow_error_set(error, EINVAL,
2320                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2321                                           "wrong action order, port_id should "
2322                                           "be after set VLAN VID");
2323         return 0;
2324 }
2325
2326 /*
2327  * Validate the FLAG action.
2328  *
2329  * @param[in] dev
2330  *   Pointer to the rte_eth_dev structure.
2331  * @param[in] action_flags
2332  *   Holds the actions detected until now.
2333  * @param[in] attr
2334  *   Pointer to flow attributes
2335  * @param[out] error
2336  *   Pointer to error structure.
2337  *
2338  * @return
2339  *   0 on success, a negative errno value otherwise and rte_errno is set.
2340  */
2341 static int
2342 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
2343                              uint64_t action_flags,
2344                              const struct rte_flow_attr *attr,
2345                              struct rte_flow_error *error)
2346 {
2347         struct mlx5_priv *priv = dev->data->dev_private;
2348         struct mlx5_dev_config *config = &priv->config;
2349         int ret;
2350
2351         /* Fall back if no extended metadata register support. */
2352         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2353                 return mlx5_flow_validate_action_flag(action_flags, attr,
2354                                                       error);
2355         /* Extensive metadata mode requires registers. */
2356         if (!mlx5_flow_ext_mreg_supported(dev))
2357                 return rte_flow_error_set(error, ENOTSUP,
2358                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2359                                           "no metadata registers "
2360                                           "to support flag action");
2361         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
2362                 return rte_flow_error_set(error, ENOTSUP,
2363                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2364                                           "extended metadata register"
2365                                           " isn't available");
2366         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2367         if (ret < 0)
2368                 return ret;
2369         MLX5_ASSERT(ret > 0);
2370         if (action_flags & MLX5_FLOW_ACTION_MARK)
2371                 return rte_flow_error_set(error, EINVAL,
2372                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2373                                           "can't mark and flag in same flow");
2374         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2375                 return rte_flow_error_set(error, EINVAL,
2376                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2377                                           "can't have 2 flag"
2378                                           " actions in same flow");
2379         return 0;
2380 }
2381
2382 /**
2383  * Validate MARK action.
2384  *
2385  * @param[in] dev
2386  *   Pointer to the rte_eth_dev structure.
2387  * @param[in] action
2388  *   Pointer to action.
2389  * @param[in] action_flags
2390  *   Holds the actions detected until now.
2391  * @param[in] attr
2392  *   Pointer to flow attributes
2393  * @param[out] error
2394  *   Pointer to error structure.
2395  *
2396  * @return
2397  *   0 on success, a negative errno value otherwise and rte_errno is set.
2398  */
2399 static int
2400 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
2401                              const struct rte_flow_action *action,
2402                              uint64_t action_flags,
2403                              const struct rte_flow_attr *attr,
2404                              struct rte_flow_error *error)
2405 {
2406         struct mlx5_priv *priv = dev->data->dev_private;
2407         struct mlx5_dev_config *config = &priv->config;
2408         const struct rte_flow_action_mark *mark = action->conf;
2409         int ret;
2410
2411         /* Fall back if no extended metadata register support. */
2412         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2413                 return mlx5_flow_validate_action_mark(action, action_flags,
2414                                                       attr, error);
2415         /* Extensive metadata mode requires registers. */
2416         if (!mlx5_flow_ext_mreg_supported(dev))
2417                 return rte_flow_error_set(error, ENOTSUP,
2418                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2419                                           "no metadata registers "
2420                                           "to support mark action");
2421         if (!priv->sh->dv_mark_mask)
2422                 return rte_flow_error_set(error, ENOTSUP,
2423                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2424                                           "extended metadata register"
2425                                           " isn't available");
2426         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2427         if (ret < 0)
2428                 return ret;
2429         MLX5_ASSERT(ret > 0);
2430         if (!mark)
2431                 return rte_flow_error_set(error, EINVAL,
2432                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2433                                           "configuration cannot be null");
2434         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
2435                 return rte_flow_error_set(error, EINVAL,
2436                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2437                                           &mark->id,
2438                                           "mark id exceeds the limit");
2439         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2440                 return rte_flow_error_set(error, EINVAL,
2441                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2442                                           "can't flag and mark in same flow");
2443         if (action_flags & MLX5_FLOW_ACTION_MARK)
2444                 return rte_flow_error_set(error, EINVAL,
2445                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2446                                           "can't have 2 mark actions in same"
2447                                           " flow");
2448         return 0;
2449 }
2450
2451 /**
2452  * Validate SET_META action.
2453  *
2454  * @param[in] dev
2455  *   Pointer to the rte_eth_dev structure.
2456  * @param[in] action
2457  *   Pointer to the action structure.
2458  * @param[in] action_flags
2459  *   Holds the actions detected until now.
2460  * @param[in] attr
2461  *   Pointer to flow attributes
2462  * @param[out] error
2463  *   Pointer to error structure.
2464  *
2465  * @return
2466  *   0 on success, a negative errno value otherwise and rte_errno is set.
2467  */
2468 static int
2469 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
2470                                  const struct rte_flow_action *action,
2471                                  uint64_t action_flags __rte_unused,
2472                                  const struct rte_flow_attr *attr,
2473                                  struct rte_flow_error *error)
2474 {
2475         const struct rte_flow_action_set_meta *conf;
2476         uint32_t nic_mask = UINT32_MAX;
2477         int reg;
2478
2479         if (!mlx5_flow_ext_mreg_supported(dev))
2480                 return rte_flow_error_set(error, ENOTSUP,
2481                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2482                                           "extended metadata register"
2483                                           " isn't supported");
2484         reg = flow_dv_get_metadata_reg(dev, attr, error);
2485         if (reg < 0)
2486                 return reg;
2487         if (reg != REG_A && reg != REG_B) {
2488                 struct mlx5_priv *priv = dev->data->dev_private;
2489
2490                 nic_mask = priv->sh->dv_meta_mask;
2491         }
2492         if (!(action->conf))
2493                 return rte_flow_error_set(error, EINVAL,
2494                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2495                                           "configuration cannot be null");
2496         conf = (const struct rte_flow_action_set_meta *)action->conf;
2497         if (!conf->mask)
2498                 return rte_flow_error_set(error, EINVAL,
2499                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2500                                           "zero mask doesn't have any effect");
2501         if (conf->mask & ~nic_mask)
2502                 return rte_flow_error_set(error, EINVAL,
2503                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2504                                           "meta data must be within reg C0");
2505         return 0;
2506 }
2507
2508 /**
2509  * Validate SET_TAG action.
2510  *
2511  * @param[in] dev
2512  *   Pointer to the rte_eth_dev structure.
2513  * @param[in] action
2514  *   Pointer to the action structure.
2515  * @param[in] action_flags
2516  *   Holds the actions detected until now.
2517  * @param[in] attr
2518  *   Pointer to flow attributes
2519  * @param[out] error
2520  *   Pointer to error structure.
2521  *
2522  * @return
2523  *   0 on success, a negative errno value otherwise and rte_errno is set.
2524  */
2525 static int
2526 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
2527                                 const struct rte_flow_action *action,
2528                                 uint64_t action_flags,
2529                                 const struct rte_flow_attr *attr,
2530                                 struct rte_flow_error *error)
2531 {
2532         const struct rte_flow_action_set_tag *conf;
2533         const uint64_t terminal_action_flags =
2534                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
2535                 MLX5_FLOW_ACTION_RSS;
2536         int ret;
2537
2538         if (!mlx5_flow_ext_mreg_supported(dev))
2539                 return rte_flow_error_set(error, ENOTSUP,
2540                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2541                                           "extensive metadata register"
2542                                           " isn't supported");
2543         if (!(action->conf))
2544                 return rte_flow_error_set(error, EINVAL,
2545                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2546                                           "configuration cannot be null");
2547         conf = (const struct rte_flow_action_set_tag *)action->conf;
2548         if (!conf->mask)
2549                 return rte_flow_error_set(error, EINVAL,
2550                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2551                                           "zero mask doesn't have any effect");
2552         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
2553         if (ret < 0)
2554                 return ret;
2555         if (!attr->transfer && attr->ingress &&
2556             (action_flags & terminal_action_flags))
2557                 return rte_flow_error_set(error, EINVAL,
2558                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2559                                           "set_tag has no effect"
2560                                           " with terminal actions");
2561         return 0;
2562 }
2563
2564 /**
2565  * Validate count action.
2566  *
2567  * @param[in] dev
2568  *   Pointer to rte_eth_dev structure.
2569  * @param[out] error
2570  *   Pointer to error structure.
2571  *
2572  * @return
2573  *   0 on success, a negative errno value otherwise and rte_errno is set.
2574  */
2575 static int
2576 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2577                               struct rte_flow_error *error)
2578 {
2579         struct mlx5_priv *priv = dev->data->dev_private;
2580
2581         if (!priv->config.devx)
2582                 goto notsup_err;
2583 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2584         return 0;
2585 #endif
2586 notsup_err:
2587         return rte_flow_error_set
2588                       (error, ENOTSUP,
2589                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2590                        NULL,
2591                        "count action not supported");
2592 }
2593
2594 /**
2595  * Validate the L2 encap action.
2596  *
2597  * @param[in] dev
2598  *   Pointer to the rte_eth_dev structure.
2599  * @param[in] action_flags
2600  *   Holds the actions detected until now.
2601  * @param[in] action
2602  *   Pointer to the action structure.
2603  * @param[in] attr
2604  *   Pointer to flow attributes.
2605  * @param[out] error
2606  *   Pointer to error structure.
2607  *
2608  * @return
2609  *   0 on success, a negative errno value otherwise and rte_errno is set.
2610  */
2611 static int
2612 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
2613                                  uint64_t action_flags,
2614                                  const struct rte_flow_action *action,
2615                                  const struct rte_flow_attr *attr,
2616                                  struct rte_flow_error *error)
2617 {
2618         const struct mlx5_priv *priv = dev->data->dev_private;
2619
2620         if (!(action->conf))
2621                 return rte_flow_error_set(error, EINVAL,
2622                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2623                                           "configuration cannot be null");
2624         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
2625                 return rte_flow_error_set(error, EINVAL,
2626                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2627                                           "can only have a single encap action "
2628                                           "in a flow");
2629         if (!attr->transfer && priv->representor)
2630                 return rte_flow_error_set(error, ENOTSUP,
2631                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2632                                           "encap action for VF representor "
2633                                           "not supported on NIC table");
2634         return 0;
2635 }
2636
2637 /**
2638  * Validate a decap action.
2639  *
2640  * @param[in] dev
2641  *   Pointer to the rte_eth_dev structure.
2642  * @param[in] action_flags
2643  *   Holds the actions detected until now.
2644  * @param[in] attr
2645  *   Pointer to flow attributes
2646  * @param[out] error
2647  *   Pointer to error structure.
2648  *
2649  * @return
2650  *   0 on success, a negative errno value otherwise and rte_errno is set.
2651  */
2652 static int
2653 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
2654                               uint64_t action_flags,
2655                               const struct rte_flow_attr *attr,
2656                               struct rte_flow_error *error)
2657 {
2658         const struct mlx5_priv *priv = dev->data->dev_private;
2659
2660         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
2661             !priv->config.decap_en)
2662                 return rte_flow_error_set(error, ENOTSUP,
2663                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2664                                           "decap is not enabled");
2665         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
2666                 return rte_flow_error_set(error, ENOTSUP,
2667                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2668                                           action_flags &
2669                                           MLX5_FLOW_ACTION_DECAP ? "can only "
2670                                           "have a single decap action" : "decap "
2671                                           "after encap is not supported");
2672         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2673                 return rte_flow_error_set(error, EINVAL,
2674                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2675                                           "can't have decap action after"
2676                                           " modify action");
2677         if (attr->egress)
2678                 return rte_flow_error_set(error, ENOTSUP,
2679                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2680                                           NULL,
2681                                           "decap action not supported for "
2682                                           "egress");
2683         if (!attr->transfer && priv->representor)
2684                 return rte_flow_error_set(error, ENOTSUP,
2685                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2686                                           "decap action for VF representor "
2687                                           "not supported on NIC table");
2688         return 0;
2689 }
2690
2691 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
2692
2693 /**
2694  * Validate the raw encap and decap actions.
2695  *
2696  * @param[in] dev
2697  *   Pointer to the rte_eth_dev structure.
2698  * @param[in] decap
2699  *   Pointer to the decap action.
2700  * @param[in] encap
2701  *   Pointer to the encap action.
2702  * @param[in] attr
2703  *   Pointer to flow attributes
2704  * @param[in/out] action_flags
2705  *   Holds the actions detected until now.
2706  * @param[out] actions_n
2707  *   pointer to the number of actions counter.
2708  * @param[out] error
2709  *   Pointer to error structure.
2710  *
2711  * @return
2712  *   0 on success, a negative errno value otherwise and rte_errno is set.
2713  */
2714 static int
2715 flow_dv_validate_action_raw_encap_decap
2716         (struct rte_eth_dev *dev,
2717          const struct rte_flow_action_raw_decap *decap,
2718          const struct rte_flow_action_raw_encap *encap,
2719          const struct rte_flow_attr *attr, uint64_t *action_flags,
2720          int *actions_n, struct rte_flow_error *error)
2721 {
2722         const struct mlx5_priv *priv = dev->data->dev_private;
2723         int ret;
2724
2725         if (encap && (!encap->size || !encap->data))
2726                 return rte_flow_error_set(error, EINVAL,
2727                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2728                                           "raw encap data cannot be empty");
2729         if (decap && encap) {
2730                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
2731                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2732                         /* L3 encap. */
2733                         decap = NULL;
2734                 else if (encap->size <=
2735                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2736                            decap->size >
2737                            MLX5_ENCAPSULATION_DECISION_SIZE)
2738                         /* L3 decap. */
2739                         encap = NULL;
2740                 else if (encap->size >
2741                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2742                            decap->size >
2743                            MLX5_ENCAPSULATION_DECISION_SIZE)
2744                         /* 2 L2 actions: encap and decap. */
2745                         ;
2746                 else
2747                         return rte_flow_error_set(error,
2748                                 ENOTSUP,
2749                                 RTE_FLOW_ERROR_TYPE_ACTION,
2750                                 NULL, "unsupported too small "
2751                                 "raw decap and too small raw "
2752                                 "encap combination");
2753         }
2754         if (decap) {
2755                 ret = flow_dv_validate_action_decap(dev, *action_flags, attr,
2756                                                     error);
2757                 if (ret < 0)
2758                         return ret;
2759                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
2760                 ++(*actions_n);
2761         }
2762         if (encap) {
2763                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
2764                         return rte_flow_error_set(error, ENOTSUP,
2765                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2766                                                   NULL,
2767                                                   "small raw encap size");
2768                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
2769                         return rte_flow_error_set(error, EINVAL,
2770                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2771                                                   NULL,
2772                                                   "more than one encap action");
2773                 if (!attr->transfer && priv->representor)
2774                         return rte_flow_error_set
2775                                         (error, ENOTSUP,
2776                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2777                                          "encap action for VF representor "
2778                                          "not supported on NIC table");
2779                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
2780                 ++(*actions_n);
2781         }
2782         return 0;
2783 }
2784
2785 /**
2786  * Match encap_decap resource.
2787  *
2788  * @param entry
2789  *   Pointer to exist resource entry object.
2790  * @param ctx
2791  *   Pointer to new encap_decap resource.
2792  *
2793  * @return
2794  *   0 on matching, -1 otherwise.
2795  */
2796 static int
2797 flow_dv_encap_decap_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
2798 {
2799         struct mlx5_flow_dv_encap_decap_resource *resource;
2800         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2801
2802         resource = (struct mlx5_flow_dv_encap_decap_resource *)ctx;
2803         cache_resource = container_of(entry,
2804                                       struct mlx5_flow_dv_encap_decap_resource,
2805                                       entry);
2806         if (resource->entry.key == cache_resource->entry.key &&
2807             resource->reformat_type == cache_resource->reformat_type &&
2808             resource->ft_type == cache_resource->ft_type &&
2809             resource->flags == cache_resource->flags &&
2810             resource->size == cache_resource->size &&
2811             !memcmp((const void *)resource->buf,
2812                     (const void *)cache_resource->buf,
2813                     resource->size))
2814                 return 0;
2815         return -1;
2816 }
2817
2818 /**
2819  * Find existing encap/decap resource or create and register a new one.
2820  *
2821  * @param[in, out] dev
2822  *   Pointer to rte_eth_dev structure.
2823  * @param[in, out] resource
2824  *   Pointer to encap/decap resource.
2825  * @parm[in, out] dev_flow
2826  *   Pointer to the dev_flow.
2827  * @param[out] error
2828  *   pointer to error structure.
2829  *
2830  * @return
2831  *   0 on success otherwise -errno and errno is set.
2832  */
2833 static int
2834 flow_dv_encap_decap_resource_register
2835                         (struct rte_eth_dev *dev,
2836                          struct mlx5_flow_dv_encap_decap_resource *resource,
2837                          struct mlx5_flow *dev_flow,
2838                          struct rte_flow_error *error)
2839 {
2840         struct mlx5_priv *priv = dev->data->dev_private;
2841         struct mlx5_dev_ctx_shared *sh = priv->sh;
2842         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2843         struct mlx5dv_dr_domain *domain;
2844         struct mlx5_hlist_entry *entry;
2845         union mlx5_flow_encap_decap_key encap_decap_key = {
2846                 {
2847                         .ft_type = resource->ft_type,
2848                         .refmt_type = resource->reformat_type,
2849                         .buf_size = resource->size,
2850                         .table_level = !!dev_flow->dv.group,
2851                         .cksum = 0,
2852                 }
2853         };
2854         int ret;
2855
2856         resource->flags = dev_flow->dv.group ? 0 : 1;
2857         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2858                 domain = sh->fdb_domain;
2859         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2860                 domain = sh->rx_domain;
2861         else
2862                 domain = sh->tx_domain;
2863         encap_decap_key.cksum = __rte_raw_cksum(resource->buf,
2864                                                 resource->size, 0);
2865         resource->entry.key = encap_decap_key.v64;
2866         /* Lookup a matching resource from cache. */
2867         entry = mlx5_hlist_lookup_ex(sh->encaps_decaps, resource->entry.key,
2868                                      flow_dv_encap_decap_resource_match,
2869                                      (void *)resource);
2870         if (entry) {
2871                 cache_resource = container_of(entry,
2872                         struct mlx5_flow_dv_encap_decap_resource, entry);
2873                 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2874                         (void *)cache_resource,
2875                         rte_atomic32_read(&cache_resource->refcnt));
2876                 rte_atomic32_inc(&cache_resource->refcnt);
2877                 dev_flow->handle->dvh.rix_encap_decap = cache_resource->idx;
2878                 dev_flow->dv.encap_decap = cache_resource;
2879                 return 0;
2880         }
2881         /* Register new encap/decap resource. */
2882         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2883                                        &dev_flow->handle->dvh.rix_encap_decap);
2884         if (!cache_resource)
2885                 return rte_flow_error_set(error, ENOMEM,
2886                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2887                                           "cannot allocate resource memory");
2888         *cache_resource = *resource;
2889         cache_resource->idx = dev_flow->handle->dvh.rix_encap_decap;
2890         ret = mlx5_flow_os_create_flow_action_packet_reformat
2891                                         (sh->ctx, domain, cache_resource,
2892                                          &cache_resource->action);
2893         if (ret) {
2894                 mlx5_free(cache_resource);
2895                 return rte_flow_error_set(error, ENOMEM,
2896                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2897                                           NULL, "cannot create action");
2898         }
2899         rte_atomic32_init(&cache_resource->refcnt);
2900         rte_atomic32_inc(&cache_resource->refcnt);
2901         if (mlx5_hlist_insert_ex(sh->encaps_decaps, &cache_resource->entry,
2902                                  flow_dv_encap_decap_resource_match,
2903                                  (void *)cache_resource)) {
2904                 claim_zero(mlx5_flow_os_destroy_flow_action
2905                                                 (cache_resource->action));
2906                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2907                                 cache_resource->idx);
2908                 return rte_flow_error_set(error, EEXIST,
2909                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2910                                           NULL, "action exist");
2911         }
2912         dev_flow->dv.encap_decap = cache_resource;
2913         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2914                 (void *)cache_resource,
2915                 rte_atomic32_read(&cache_resource->refcnt));
2916         return 0;
2917 }
2918
2919 /**
2920  * Find existing table jump resource or create and register a new one.
2921  *
2922  * @param[in, out] dev
2923  *   Pointer to rte_eth_dev structure.
2924  * @param[in, out] tbl
2925  *   Pointer to flow table resource.
2926  * @parm[in, out] dev_flow
2927  *   Pointer to the dev_flow.
2928  * @param[out] error
2929  *   pointer to error structure.
2930  *
2931  * @return
2932  *   0 on success otherwise -errno and errno is set.
2933  */
2934 static int
2935 flow_dv_jump_tbl_resource_register
2936                         (struct rte_eth_dev *dev __rte_unused,
2937                          struct mlx5_flow_tbl_resource *tbl,
2938                          struct mlx5_flow *dev_flow,
2939                          struct rte_flow_error *error)
2940 {
2941         struct mlx5_flow_tbl_data_entry *tbl_data =
2942                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2943         int cnt, ret;
2944
2945         MLX5_ASSERT(tbl);
2946         cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
2947         if (!cnt) {
2948                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
2949                                 (tbl->obj, &tbl_data->jump.action);
2950                 if (ret)
2951                         return rte_flow_error_set(error, ENOMEM,
2952                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2953                                         NULL, "cannot create jump action");
2954                 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2955                         (void *)&tbl_data->jump, cnt);
2956         } else {
2957                 /* old jump should not make the table ref++. */
2958                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
2959                 MLX5_ASSERT(tbl_data->jump.action);
2960                 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2961                         (void *)&tbl_data->jump, cnt);
2962         }
2963         rte_atomic32_inc(&tbl_data->jump.refcnt);
2964         dev_flow->handle->rix_jump = tbl_data->idx;
2965         dev_flow->dv.jump = &tbl_data->jump;
2966         return 0;
2967 }
2968
2969 /**
2970  * Find existing default miss resource or create and register a new one.
2971  *
2972  * @param[in, out] dev
2973  *   Pointer to rte_eth_dev structure.
2974  * @param[out] error
2975  *   pointer to error structure.
2976  *
2977  * @return
2978  *   0 on success otherwise -errno and errno is set.
2979  */
2980 static int
2981 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
2982                 struct rte_flow_error *error)
2983 {
2984         struct mlx5_priv *priv = dev->data->dev_private;
2985         struct mlx5_dev_ctx_shared *sh = priv->sh;
2986         struct mlx5_flow_default_miss_resource *cache_resource =
2987                         &sh->default_miss;
2988         int cnt = rte_atomic32_read(&cache_resource->refcnt);
2989
2990         if (!cnt) {
2991                 MLX5_ASSERT(cache_resource->action);
2992                 cache_resource->action =
2993                 mlx5_glue->dr_create_flow_action_default_miss();
2994                 if (!cache_resource->action)
2995                         return rte_flow_error_set(error, ENOMEM,
2996                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2997                                         "cannot create default miss action");
2998                 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++",
2999                                 (void *)cache_resource->action, cnt);
3000         }
3001         rte_atomic32_inc(&cache_resource->refcnt);
3002         return 0;
3003 }
3004
3005 /**
3006  * Find existing table port ID resource or create and register a new one.
3007  *
3008  * @param[in, out] dev
3009  *   Pointer to rte_eth_dev structure.
3010  * @param[in, out] resource
3011  *   Pointer to port ID action resource.
3012  * @parm[in, out] dev_flow
3013  *   Pointer to the dev_flow.
3014  * @param[out] error
3015  *   pointer to error structure.
3016  *
3017  * @return
3018  *   0 on success otherwise -errno and errno is set.
3019  */
3020 static int
3021 flow_dv_port_id_action_resource_register
3022                         (struct rte_eth_dev *dev,
3023                          struct mlx5_flow_dv_port_id_action_resource *resource,
3024                          struct mlx5_flow *dev_flow,
3025                          struct rte_flow_error *error)
3026 {
3027         struct mlx5_priv *priv = dev->data->dev_private;
3028         struct mlx5_dev_ctx_shared *sh = priv->sh;
3029         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
3030         uint32_t idx = 0;
3031         int ret;
3032
3033         /* Lookup a matching resource from cache. */
3034         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list,
3035                       idx, cache_resource, next) {
3036                 if (resource->port_id == cache_resource->port_id) {
3037                         DRV_LOG(DEBUG, "port id action resource resource %p: "
3038                                 "refcnt %d++",
3039                                 (void *)cache_resource,
3040                                 rte_atomic32_read(&cache_resource->refcnt));
3041                         rte_atomic32_inc(&cache_resource->refcnt);
3042                         dev_flow->handle->rix_port_id_action = idx;
3043                         dev_flow->dv.port_id_action = cache_resource;
3044                         return 0;
3045                 }
3046         }
3047         /* Register new port id action resource. */
3048         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID],
3049                                        &dev_flow->handle->rix_port_id_action);
3050         if (!cache_resource)
3051                 return rte_flow_error_set(error, ENOMEM,
3052                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3053                                           "cannot allocate resource memory");
3054         *cache_resource = *resource;
3055         ret = mlx5_flow_os_create_flow_action_dest_port
3056                                 (priv->sh->fdb_domain, resource->port_id,
3057                                  &cache_resource->action);
3058         if (ret) {
3059                 mlx5_free(cache_resource);
3060                 return rte_flow_error_set(error, ENOMEM,
3061                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3062                                           NULL, "cannot create action");
3063         }
3064         rte_atomic32_init(&cache_resource->refcnt);
3065         rte_atomic32_inc(&cache_resource->refcnt);
3066         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
3067                      dev_flow->handle->rix_port_id_action, cache_resource,
3068                      next);
3069         dev_flow->dv.port_id_action = cache_resource;
3070         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
3071                 (void *)cache_resource,
3072                 rte_atomic32_read(&cache_resource->refcnt));
3073         return 0;
3074 }
3075
3076 /**
3077  * Find existing push vlan resource or create and register a new one.
3078  *
3079  * @param [in, out] dev
3080  *   Pointer to rte_eth_dev structure.
3081  * @param[in, out] resource
3082  *   Pointer to port ID action resource.
3083  * @parm[in, out] dev_flow
3084  *   Pointer to the dev_flow.
3085  * @param[out] error
3086  *   pointer to error structure.
3087  *
3088  * @return
3089  *   0 on success otherwise -errno and errno is set.
3090  */
3091 static int
3092 flow_dv_push_vlan_action_resource_register
3093                        (struct rte_eth_dev *dev,
3094                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
3095                         struct mlx5_flow *dev_flow,
3096                         struct rte_flow_error *error)
3097 {
3098         struct mlx5_priv *priv = dev->data->dev_private;
3099         struct mlx5_dev_ctx_shared *sh = priv->sh;
3100         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
3101         struct mlx5dv_dr_domain *domain;
3102         uint32_t idx = 0;
3103         int ret;
3104
3105         /* Lookup a matching resource from cache. */
3106         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3107                       sh->push_vlan_action_list, idx, cache_resource, next) {
3108                 if (resource->vlan_tag == cache_resource->vlan_tag &&
3109                     resource->ft_type == cache_resource->ft_type) {
3110                         DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
3111                                 "refcnt %d++",
3112                                 (void *)cache_resource,
3113                                 rte_atomic32_read(&cache_resource->refcnt));
3114                         rte_atomic32_inc(&cache_resource->refcnt);
3115                         dev_flow->handle->dvh.rix_push_vlan = idx;
3116                         dev_flow->dv.push_vlan_res = cache_resource;
3117                         return 0;
3118                 }
3119         }
3120         /* Register new push_vlan action resource. */
3121         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3122                                        &dev_flow->handle->dvh.rix_push_vlan);
3123         if (!cache_resource)
3124                 return rte_flow_error_set(error, ENOMEM,
3125                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3126                                           "cannot allocate resource memory");
3127         *cache_resource = *resource;
3128         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3129                 domain = sh->fdb_domain;
3130         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3131                 domain = sh->rx_domain;
3132         else
3133                 domain = sh->tx_domain;
3134         ret = mlx5_flow_os_create_flow_action_push_vlan
3135                                         (domain, resource->vlan_tag,
3136                                          &cache_resource->action);
3137         if (ret) {
3138                 mlx5_free(cache_resource);
3139                 return rte_flow_error_set(error, ENOMEM,
3140                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3141                                           NULL, "cannot create action");
3142         }
3143         rte_atomic32_init(&cache_resource->refcnt);
3144         rte_atomic32_inc(&cache_resource->refcnt);
3145         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3146                      &sh->push_vlan_action_list,
3147                      dev_flow->handle->dvh.rix_push_vlan,
3148                      cache_resource, next);
3149         dev_flow->dv.push_vlan_res = cache_resource;
3150         DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
3151                 (void *)cache_resource,
3152                 rte_atomic32_read(&cache_resource->refcnt));
3153         return 0;
3154 }
3155 /**
3156  * Get the size of specific rte_flow_item_type hdr size
3157  *
3158  * @param[in] item_type
3159  *   Tested rte_flow_item_type.
3160  *
3161  * @return
3162  *   sizeof struct item_type, 0 if void or irrelevant.
3163  */
3164 static size_t
3165 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
3166 {
3167         size_t retval;
3168
3169         switch (item_type) {
3170         case RTE_FLOW_ITEM_TYPE_ETH:
3171                 retval = sizeof(struct rte_ether_hdr);
3172                 break;
3173         case RTE_FLOW_ITEM_TYPE_VLAN:
3174                 retval = sizeof(struct rte_vlan_hdr);
3175                 break;
3176         case RTE_FLOW_ITEM_TYPE_IPV4:
3177                 retval = sizeof(struct rte_ipv4_hdr);
3178                 break;
3179         case RTE_FLOW_ITEM_TYPE_IPV6:
3180                 retval = sizeof(struct rte_ipv6_hdr);
3181                 break;
3182         case RTE_FLOW_ITEM_TYPE_UDP:
3183                 retval = sizeof(struct rte_udp_hdr);
3184                 break;
3185         case RTE_FLOW_ITEM_TYPE_TCP:
3186                 retval = sizeof(struct rte_tcp_hdr);
3187                 break;
3188         case RTE_FLOW_ITEM_TYPE_VXLAN:
3189         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3190                 retval = sizeof(struct rte_vxlan_hdr);
3191                 break;
3192         case RTE_FLOW_ITEM_TYPE_GRE:
3193         case RTE_FLOW_ITEM_TYPE_NVGRE:
3194                 retval = sizeof(struct rte_gre_hdr);
3195                 break;
3196         case RTE_FLOW_ITEM_TYPE_MPLS:
3197                 retval = sizeof(struct rte_mpls_hdr);
3198                 break;
3199         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
3200         default:
3201                 retval = 0;
3202                 break;
3203         }
3204         return retval;
3205 }
3206
3207 #define MLX5_ENCAP_IPV4_VERSION         0x40
3208 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
3209 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
3210 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
3211 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
3212 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
3213 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
3214
3215 /**
3216  * Convert the encap action data from list of rte_flow_item to raw buffer
3217  *
3218  * @param[in] items
3219  *   Pointer to rte_flow_item objects list.
3220  * @param[out] buf
3221  *   Pointer to the output buffer.
3222  * @param[out] size
3223  *   Pointer to the output buffer size.
3224  * @param[out] error
3225  *   Pointer to the error structure.
3226  *
3227  * @return
3228  *   0 on success, a negative errno value otherwise and rte_errno is set.
3229  */
3230 static int
3231 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
3232                            size_t *size, struct rte_flow_error *error)
3233 {
3234         struct rte_ether_hdr *eth = NULL;
3235         struct rte_vlan_hdr *vlan = NULL;
3236         struct rte_ipv4_hdr *ipv4 = NULL;
3237         struct rte_ipv6_hdr *ipv6 = NULL;
3238         struct rte_udp_hdr *udp = NULL;
3239         struct rte_vxlan_hdr *vxlan = NULL;
3240         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
3241         struct rte_gre_hdr *gre = NULL;
3242         size_t len;
3243         size_t temp_size = 0;
3244
3245         if (!items)
3246                 return rte_flow_error_set(error, EINVAL,
3247                                           RTE_FLOW_ERROR_TYPE_ACTION,
3248                                           NULL, "invalid empty data");
3249         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3250                 len = flow_dv_get_item_hdr_len(items->type);
3251                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
3252                         return rte_flow_error_set(error, EINVAL,
3253                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3254                                                   (void *)items->type,
3255                                                   "items total size is too big"
3256                                                   " for encap action");
3257                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
3258                 switch (items->type) {
3259                 case RTE_FLOW_ITEM_TYPE_ETH:
3260                         eth = (struct rte_ether_hdr *)&buf[temp_size];
3261                         break;
3262                 case RTE_FLOW_ITEM_TYPE_VLAN:
3263                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
3264                         if (!eth)
3265                                 return rte_flow_error_set(error, EINVAL,
3266                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3267                                                 (void *)items->type,
3268                                                 "eth header not found");
3269                         if (!eth->ether_type)
3270                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
3271                         break;
3272                 case RTE_FLOW_ITEM_TYPE_IPV4:
3273                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
3274                         if (!vlan && !eth)
3275                                 return rte_flow_error_set(error, EINVAL,
3276                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3277                                                 (void *)items->type,
3278                                                 "neither eth nor vlan"
3279                                                 " header found");
3280                         if (vlan && !vlan->eth_proto)
3281                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3282                         else if (eth && !eth->ether_type)
3283                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3284                         if (!ipv4->version_ihl)
3285                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
3286                                                     MLX5_ENCAP_IPV4_IHL_MIN;
3287                         if (!ipv4->time_to_live)
3288                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
3289                         break;
3290                 case RTE_FLOW_ITEM_TYPE_IPV6:
3291                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
3292                         if (!vlan && !eth)
3293                                 return rte_flow_error_set(error, EINVAL,
3294                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3295                                                 (void *)items->type,
3296                                                 "neither eth nor vlan"
3297                                                 " header found");
3298                         if (vlan && !vlan->eth_proto)
3299                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3300                         else if (eth && !eth->ether_type)
3301                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3302                         if (!ipv6->vtc_flow)
3303                                 ipv6->vtc_flow =
3304                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
3305                         if (!ipv6->hop_limits)
3306                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
3307                         break;
3308                 case RTE_FLOW_ITEM_TYPE_UDP:
3309                         udp = (struct rte_udp_hdr *)&buf[temp_size];
3310                         if (!ipv4 && !ipv6)
3311                                 return rte_flow_error_set(error, EINVAL,
3312                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3313                                                 (void *)items->type,
3314                                                 "ip header not found");
3315                         if (ipv4 && !ipv4->next_proto_id)
3316                                 ipv4->next_proto_id = IPPROTO_UDP;
3317                         else if (ipv6 && !ipv6->proto)
3318                                 ipv6->proto = IPPROTO_UDP;
3319                         break;
3320                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3321                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
3322                         if (!udp)
3323                                 return rte_flow_error_set(error, EINVAL,
3324                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3325                                                 (void *)items->type,
3326                                                 "udp header not found");
3327                         if (!udp->dst_port)
3328                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
3329                         if (!vxlan->vx_flags)
3330                                 vxlan->vx_flags =
3331                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
3332                         break;
3333                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3334                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
3335                         if (!udp)
3336                                 return rte_flow_error_set(error, EINVAL,
3337                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3338                                                 (void *)items->type,
3339                                                 "udp header not found");
3340                         if (!vxlan_gpe->proto)
3341                                 return rte_flow_error_set(error, EINVAL,
3342                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3343                                                 (void *)items->type,
3344                                                 "next protocol not found");
3345                         if (!udp->dst_port)
3346                                 udp->dst_port =
3347                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
3348                         if (!vxlan_gpe->vx_flags)
3349                                 vxlan_gpe->vx_flags =
3350                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
3351                         break;
3352                 case RTE_FLOW_ITEM_TYPE_GRE:
3353                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3354                         gre = (struct rte_gre_hdr *)&buf[temp_size];
3355                         if (!gre->proto)
3356                                 return rte_flow_error_set(error, EINVAL,
3357                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3358                                                 (void *)items->type,
3359                                                 "next protocol not found");
3360                         if (!ipv4 && !ipv6)
3361                                 return rte_flow_error_set(error, EINVAL,
3362                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3363                                                 (void *)items->type,
3364                                                 "ip header not found");
3365                         if (ipv4 && !ipv4->next_proto_id)
3366                                 ipv4->next_proto_id = IPPROTO_GRE;
3367                         else if (ipv6 && !ipv6->proto)
3368                                 ipv6->proto = IPPROTO_GRE;
3369                         break;
3370                 case RTE_FLOW_ITEM_TYPE_VOID:
3371                         break;
3372                 default:
3373                         return rte_flow_error_set(error, EINVAL,
3374                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3375                                                   (void *)items->type,
3376                                                   "unsupported item type");
3377                         break;
3378                 }
3379                 temp_size += len;
3380         }
3381         *size = temp_size;
3382         return 0;
3383 }
3384
3385 static int
3386 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
3387 {
3388         struct rte_ether_hdr *eth = NULL;
3389         struct rte_vlan_hdr *vlan = NULL;
3390         struct rte_ipv6_hdr *ipv6 = NULL;
3391         struct rte_udp_hdr *udp = NULL;
3392         char *next_hdr;
3393         uint16_t proto;
3394
3395         eth = (struct rte_ether_hdr *)data;
3396         next_hdr = (char *)(eth + 1);
3397         proto = RTE_BE16(eth->ether_type);
3398
3399         /* VLAN skipping */
3400         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
3401                 vlan = (struct rte_vlan_hdr *)next_hdr;
3402                 proto = RTE_BE16(vlan->eth_proto);
3403                 next_hdr += sizeof(struct rte_vlan_hdr);
3404         }
3405
3406         /* HW calculates IPv4 csum. no need to proceed */
3407         if (proto == RTE_ETHER_TYPE_IPV4)
3408                 return 0;
3409
3410         /* non IPv4/IPv6 header. not supported */
3411         if (proto != RTE_ETHER_TYPE_IPV6) {
3412                 return rte_flow_error_set(error, ENOTSUP,
3413                                           RTE_FLOW_ERROR_TYPE_ACTION,
3414                                           NULL, "Cannot offload non IPv4/IPv6");
3415         }
3416
3417         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
3418
3419         /* ignore non UDP */
3420         if (ipv6->proto != IPPROTO_UDP)
3421                 return 0;
3422
3423         udp = (struct rte_udp_hdr *)(ipv6 + 1);
3424         udp->dgram_cksum = 0;
3425
3426         return 0;
3427 }
3428
3429 /**
3430  * Convert L2 encap action to DV specification.
3431  *
3432  * @param[in] dev
3433  *   Pointer to rte_eth_dev structure.
3434  * @param[in] action
3435  *   Pointer to action structure.
3436  * @param[in, out] dev_flow
3437  *   Pointer to the mlx5_flow.
3438  * @param[in] transfer
3439  *   Mark if the flow is E-Switch flow.
3440  * @param[out] error
3441  *   Pointer to the error structure.
3442  *
3443  * @return
3444  *   0 on success, a negative errno value otherwise and rte_errno is set.
3445  */
3446 static int
3447 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
3448                                const struct rte_flow_action *action,
3449                                struct mlx5_flow *dev_flow,
3450                                uint8_t transfer,
3451                                struct rte_flow_error *error)
3452 {
3453         const struct rte_flow_item *encap_data;
3454         const struct rte_flow_action_raw_encap *raw_encap_data;
3455         struct mlx5_flow_dv_encap_decap_resource res = {
3456                 .reformat_type =
3457                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
3458                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3459                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
3460         };
3461
3462         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3463                 raw_encap_data =
3464                         (const struct rte_flow_action_raw_encap *)action->conf;
3465                 res.size = raw_encap_data->size;
3466                 memcpy(res.buf, raw_encap_data->data, res.size);
3467         } else {
3468                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
3469                         encap_data =
3470                                 ((const struct rte_flow_action_vxlan_encap *)
3471                                                 action->conf)->definition;
3472                 else
3473                         encap_data =
3474                                 ((const struct rte_flow_action_nvgre_encap *)
3475                                                 action->conf)->definition;
3476                 if (flow_dv_convert_encap_data(encap_data, res.buf,
3477                                                &res.size, error))
3478                         return -rte_errno;
3479         }
3480         if (flow_dv_zero_encap_udp_csum(res.buf, error))
3481                 return -rte_errno;
3482         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3483                 return rte_flow_error_set(error, EINVAL,
3484                                           RTE_FLOW_ERROR_TYPE_ACTION,
3485                                           NULL, "can't create L2 encap action");
3486         return 0;
3487 }
3488
3489 /**
3490  * Convert L2 decap action to DV specification.
3491  *
3492  * @param[in] dev
3493  *   Pointer to rte_eth_dev structure.
3494  * @param[in, out] dev_flow
3495  *   Pointer to the mlx5_flow.
3496  * @param[in] transfer
3497  *   Mark if the flow is E-Switch flow.
3498  * @param[out] error
3499  *   Pointer to the error structure.
3500  *
3501  * @return
3502  *   0 on success, a negative errno value otherwise and rte_errno is set.
3503  */
3504 static int
3505 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
3506                                struct mlx5_flow *dev_flow,
3507                                uint8_t transfer,
3508                                struct rte_flow_error *error)
3509 {
3510         struct mlx5_flow_dv_encap_decap_resource res = {
3511                 .size = 0,
3512                 .reformat_type =
3513                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
3514                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3515                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
3516         };
3517
3518         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3519                 return rte_flow_error_set(error, EINVAL,
3520                                           RTE_FLOW_ERROR_TYPE_ACTION,
3521                                           NULL, "can't create L2 decap action");
3522         return 0;
3523 }
3524
3525 /**
3526  * Convert raw decap/encap (L3 tunnel) action to DV specification.
3527  *
3528  * @param[in] dev
3529  *   Pointer to rte_eth_dev structure.
3530  * @param[in] action
3531  *   Pointer to action structure.
3532  * @param[in, out] dev_flow
3533  *   Pointer to the mlx5_flow.
3534  * @param[in] attr
3535  *   Pointer to the flow attributes.
3536  * @param[out] error
3537  *   Pointer to the error structure.
3538  *
3539  * @return
3540  *   0 on success, a negative errno value otherwise and rte_errno is set.
3541  */
3542 static int
3543 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
3544                                 const struct rte_flow_action *action,
3545                                 struct mlx5_flow *dev_flow,
3546                                 const struct rte_flow_attr *attr,
3547                                 struct rte_flow_error *error)
3548 {
3549         const struct rte_flow_action_raw_encap *encap_data;
3550         struct mlx5_flow_dv_encap_decap_resource res;
3551
3552         memset(&res, 0, sizeof(res));
3553         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
3554         res.size = encap_data->size;
3555         memcpy(res.buf, encap_data->data, res.size);
3556         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
3557                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
3558                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3559         if (attr->transfer)
3560                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3561         else
3562                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3563                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3564         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3565                 return rte_flow_error_set(error, EINVAL,
3566                                           RTE_FLOW_ERROR_TYPE_ACTION,
3567                                           NULL, "can't create encap action");
3568         return 0;
3569 }
3570
3571 /**
3572  * Create action push VLAN.
3573  *
3574  * @param[in] dev
3575  *   Pointer to rte_eth_dev structure.
3576  * @param[in] attr
3577  *   Pointer to the flow attributes.
3578  * @param[in] vlan
3579  *   Pointer to the vlan to push to the Ethernet header.
3580  * @param[in, out] dev_flow
3581  *   Pointer to the mlx5_flow.
3582  * @param[out] error
3583  *   Pointer to the error structure.
3584  *
3585  * @return
3586  *   0 on success, a negative errno value otherwise and rte_errno is set.
3587  */
3588 static int
3589 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
3590                                 const struct rte_flow_attr *attr,
3591                                 const struct rte_vlan_hdr *vlan,
3592                                 struct mlx5_flow *dev_flow,
3593                                 struct rte_flow_error *error)
3594 {
3595         struct mlx5_flow_dv_push_vlan_action_resource res;
3596
3597         memset(&res, 0, sizeof(res));
3598         res.vlan_tag =
3599                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
3600                                  vlan->vlan_tci);
3601         if (attr->transfer)
3602                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3603         else
3604                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3605                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3606         return flow_dv_push_vlan_action_resource_register
3607                                             (dev, &res, dev_flow, error);
3608 }
3609
3610 static int fdb_mirror;
3611
3612 /**
3613  * Validate the modify-header actions.
3614  *
3615  * @param[in] action_flags
3616  *   Holds the actions detected until now.
3617  * @param[in] action
3618  *   Pointer to the modify action.
3619  * @param[out] error
3620  *   Pointer to error structure.
3621  *
3622  * @return
3623  *   0 on success, a negative errno value otherwise and rte_errno is set.
3624  */
3625 static int
3626 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
3627                                    const struct rte_flow_action *action,
3628                                    struct rte_flow_error *error)
3629 {
3630         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
3631                 return rte_flow_error_set(error, EINVAL,
3632                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3633                                           NULL, "action configuration not set");
3634         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3635                 return rte_flow_error_set(error, EINVAL,
3636                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3637                                           "can't have encap action before"
3638                                           " modify action");
3639         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror)
3640                 return rte_flow_error_set(error, EINVAL,
3641                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3642                                           "can't support sample action before"
3643                                           " modify action for E-Switch"
3644                                           " mirroring");
3645         return 0;
3646 }
3647
3648 /**
3649  * Validate the modify-header MAC address actions.
3650  *
3651  * @param[in] action_flags
3652  *   Holds the actions detected until now.
3653  * @param[in] action
3654  *   Pointer to the modify action.
3655  * @param[in] item_flags
3656  *   Holds the items detected.
3657  * @param[out] error
3658  *   Pointer to error structure.
3659  *
3660  * @return
3661  *   0 on success, a negative errno value otherwise and rte_errno is set.
3662  */
3663 static int
3664 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
3665                                    const struct rte_flow_action *action,
3666                                    const uint64_t item_flags,
3667                                    struct rte_flow_error *error)
3668 {
3669         int ret = 0;
3670
3671         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3672         if (!ret) {
3673                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
3674                         return rte_flow_error_set(error, EINVAL,
3675                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3676                                                   NULL,
3677                                                   "no L2 item in pattern");
3678         }
3679         return ret;
3680 }
3681
3682 /**
3683  * Validate the modify-header IPv4 address actions.
3684  *
3685  * @param[in] action_flags
3686  *   Holds the actions detected until now.
3687  * @param[in] action
3688  *   Pointer to the modify action.
3689  * @param[in] item_flags
3690  *   Holds the items detected.
3691  * @param[out] error
3692  *   Pointer to error structure.
3693  *
3694  * @return
3695  *   0 on success, a negative errno value otherwise and rte_errno is set.
3696  */
3697 static int
3698 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3699                                     const struct rte_flow_action *action,
3700                                     const uint64_t item_flags,
3701                                     struct rte_flow_error *error)
3702 {
3703         int ret = 0;
3704         uint64_t layer;
3705
3706         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3707         if (!ret) {
3708                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3709                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3710                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3711                 if (!(item_flags & layer))
3712                         return rte_flow_error_set(error, EINVAL,
3713                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3714                                                   NULL,
3715                                                   "no ipv4 item in pattern");
3716         }
3717         return ret;
3718 }
3719
3720 /**
3721  * Validate the modify-header IPv6 address actions.
3722  *
3723  * @param[in] action_flags
3724  *   Holds the actions detected until now.
3725  * @param[in] action
3726  *   Pointer to the modify action.
3727  * @param[in] item_flags
3728  *   Holds the items detected.
3729  * @param[out] error
3730  *   Pointer to error structure.
3731  *
3732  * @return
3733  *   0 on success, a negative errno value otherwise and rte_errno is set.
3734  */
3735 static int
3736 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3737                                     const struct rte_flow_action *action,
3738                                     const uint64_t item_flags,
3739                                     struct rte_flow_error *error)
3740 {
3741         int ret = 0;
3742         uint64_t layer;
3743
3744         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3745         if (!ret) {
3746                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3747                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3748                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3749                 if (!(item_flags & layer))
3750                         return rte_flow_error_set(error, EINVAL,
3751                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3752                                                   NULL,
3753                                                   "no ipv6 item in pattern");
3754         }
3755         return ret;
3756 }
3757
3758 /**
3759  * Validate the modify-header TP actions.
3760  *
3761  * @param[in] action_flags
3762  *   Holds the actions detected until now.
3763  * @param[in] action
3764  *   Pointer to the modify action.
3765  * @param[in] item_flags
3766  *   Holds the items detected.
3767  * @param[out] error
3768  *   Pointer to error structure.
3769  *
3770  * @return
3771  *   0 on success, a negative errno value otherwise and rte_errno is set.
3772  */
3773 static int
3774 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3775                                   const struct rte_flow_action *action,
3776                                   const uint64_t item_flags,
3777                                   struct rte_flow_error *error)
3778 {
3779         int ret = 0;
3780         uint64_t layer;
3781
3782         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3783         if (!ret) {
3784                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3785                                  MLX5_FLOW_LAYER_INNER_L4 :
3786                                  MLX5_FLOW_LAYER_OUTER_L4;
3787                 if (!(item_flags & layer))
3788                         return rte_flow_error_set(error, EINVAL,
3789                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3790                                                   NULL, "no transport layer "
3791                                                   "in pattern");
3792         }
3793         return ret;
3794 }
3795
3796 /**
3797  * Validate the modify-header actions of increment/decrement
3798  * TCP Sequence-number.
3799  *
3800  * @param[in] action_flags
3801  *   Holds the actions detected until now.
3802  * @param[in] action
3803  *   Pointer to the modify action.
3804  * @param[in] item_flags
3805  *   Holds the items detected.
3806  * @param[out] error
3807  *   Pointer to error structure.
3808  *
3809  * @return
3810  *   0 on success, a negative errno value otherwise and rte_errno is set.
3811  */
3812 static int
3813 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3814                                        const struct rte_flow_action *action,
3815                                        const uint64_t item_flags,
3816                                        struct rte_flow_error *error)
3817 {
3818         int ret = 0;
3819         uint64_t layer;
3820
3821         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3822         if (!ret) {
3823                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3824                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3825                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3826                 if (!(item_flags & layer))
3827                         return rte_flow_error_set(error, EINVAL,
3828                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3829                                                   NULL, "no TCP item in"
3830                                                   " pattern");
3831                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3832                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3833                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3834                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3835                         return rte_flow_error_set(error, EINVAL,
3836                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3837                                                   NULL,
3838                                                   "cannot decrease and increase"
3839                                                   " TCP sequence number"
3840                                                   " at the same time");
3841         }
3842         return ret;
3843 }
3844
3845 /**
3846  * Validate the modify-header actions of increment/decrement
3847  * TCP Acknowledgment number.
3848  *
3849  * @param[in] action_flags
3850  *   Holds the actions detected until now.
3851  * @param[in] action
3852  *   Pointer to the modify action.
3853  * @param[in] item_flags
3854  *   Holds the items detected.
3855  * @param[out] error
3856  *   Pointer to error structure.
3857  *
3858  * @return
3859  *   0 on success, a negative errno value otherwise and rte_errno is set.
3860  */
3861 static int
3862 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3863                                        const struct rte_flow_action *action,
3864                                        const uint64_t item_flags,
3865                                        struct rte_flow_error *error)
3866 {
3867         int ret = 0;
3868         uint64_t layer;
3869
3870         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3871         if (!ret) {
3872                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3873                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3874                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3875                 if (!(item_flags & layer))
3876                         return rte_flow_error_set(error, EINVAL,
3877                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3878                                                   NULL, "no TCP item in"
3879                                                   " pattern");
3880                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3881                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3882                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3883                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3884                         return rte_flow_error_set(error, EINVAL,
3885                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3886                                                   NULL,
3887                                                   "cannot decrease and increase"
3888                                                   " TCP acknowledgment number"
3889                                                   " at the same time");
3890         }
3891         return ret;
3892 }
3893
3894 /**
3895  * Validate the modify-header TTL actions.
3896  *
3897  * @param[in] action_flags
3898  *   Holds the actions detected until now.
3899  * @param[in] action
3900  *   Pointer to the modify action.
3901  * @param[in] item_flags
3902  *   Holds the items detected.
3903  * @param[out] error
3904  *   Pointer to error structure.
3905  *
3906  * @return
3907  *   0 on success, a negative errno value otherwise and rte_errno is set.
3908  */
3909 static int
3910 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3911                                    const struct rte_flow_action *action,
3912                                    const uint64_t item_flags,
3913                                    struct rte_flow_error *error)
3914 {
3915         int ret = 0;
3916         uint64_t layer;
3917
3918         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3919         if (!ret) {
3920                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3921                                  MLX5_FLOW_LAYER_INNER_L3 :
3922                                  MLX5_FLOW_LAYER_OUTER_L3;
3923                 if (!(item_flags & layer))
3924                         return rte_flow_error_set(error, EINVAL,
3925                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3926                                                   NULL,
3927                                                   "no IP protocol in pattern");
3928         }
3929         return ret;
3930 }
3931
3932 /**
3933  * Validate jump action.
3934  *
3935  * @param[in] action
3936  *   Pointer to the jump action.
3937  * @param[in] action_flags
3938  *   Holds the actions detected until now.
3939  * @param[in] attributes
3940  *   Pointer to flow attributes
3941  * @param[in] external
3942  *   Action belongs to flow rule created by request external to PMD.
3943  * @param[out] error
3944  *   Pointer to error structure.
3945  *
3946  * @return
3947  *   0 on success, a negative errno value otherwise and rte_errno is set.
3948  */
3949 static int
3950 flow_dv_validate_action_jump(const struct rte_flow_action *action,
3951                              uint64_t action_flags,
3952                              const struct rte_flow_attr *attributes,
3953                              bool external, struct rte_flow_error *error)
3954 {
3955         uint32_t target_group, table;
3956         int ret = 0;
3957
3958         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3959                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3960                 return rte_flow_error_set(error, EINVAL,
3961                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3962                                           "can't have 2 fate actions in"
3963                                           " same flow");
3964         if (action_flags & MLX5_FLOW_ACTION_METER)
3965                 return rte_flow_error_set(error, ENOTSUP,
3966                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3967                                           "jump with meter not support");
3968         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror)
3969                 return rte_flow_error_set(error, EINVAL,
3970                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3971                                           "E-Switch mirroring can't support"
3972                                           " Sample action and jump action in"
3973                                           " same flow now");
3974         if (!action->conf)
3975                 return rte_flow_error_set(error, EINVAL,
3976                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3977                                           NULL, "action configuration not set");
3978         target_group =
3979                 ((const struct rte_flow_action_jump *)action->conf)->group;
3980         ret = mlx5_flow_group_to_table(attributes, external, target_group,
3981                                        true, &table, error);
3982         if (ret)
3983                 return ret;
3984         if (attributes->group == target_group)
3985                 return rte_flow_error_set(error, EINVAL,
3986                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3987                                           "target group must be other than"
3988                                           " the current flow group");
3989         return 0;
3990 }
3991
3992 /*
3993  * Validate the port_id action.
3994  *
3995  * @param[in] dev
3996  *   Pointer to rte_eth_dev structure.
3997  * @param[in] action_flags
3998  *   Bit-fields that holds the actions detected until now.
3999  * @param[in] action
4000  *   Port_id RTE action structure.
4001  * @param[in] attr
4002  *   Attributes of flow that includes this action.
4003  * @param[out] error
4004  *   Pointer to error structure.
4005  *
4006  * @return
4007  *   0 on success, a negative errno value otherwise and rte_errno is set.
4008  */
4009 static int
4010 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
4011                                 uint64_t action_flags,
4012                                 const struct rte_flow_action *action,
4013                                 const struct rte_flow_attr *attr,
4014                                 struct rte_flow_error *error)
4015 {
4016         const struct rte_flow_action_port_id *port_id;
4017         struct mlx5_priv *act_priv;
4018         struct mlx5_priv *dev_priv;
4019         uint16_t port;
4020
4021         if (!attr->transfer)
4022                 return rte_flow_error_set(error, ENOTSUP,
4023                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4024                                           NULL,
4025                                           "port id action is valid in transfer"
4026                                           " mode only");
4027         if (!action || !action->conf)
4028                 return rte_flow_error_set(error, ENOTSUP,
4029                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4030                                           NULL,
4031                                           "port id action parameters must be"
4032                                           " specified");
4033         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
4034                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4035                 return rte_flow_error_set(error, EINVAL,
4036                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4037                                           "can have only one fate actions in"
4038                                           " a flow");
4039         dev_priv = mlx5_dev_to_eswitch_info(dev);
4040         if (!dev_priv)
4041                 return rte_flow_error_set(error, rte_errno,
4042                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4043                                           NULL,
4044                                           "failed to obtain E-Switch info");
4045         port_id = action->conf;
4046         port = port_id->original ? dev->data->port_id : port_id->id;
4047         act_priv = mlx5_port_to_eswitch_info(port, false);
4048         if (!act_priv)
4049                 return rte_flow_error_set
4050                                 (error, rte_errno,
4051                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
4052                                  "failed to obtain E-Switch port id for port");
4053         if (act_priv->domain_id != dev_priv->domain_id)
4054                 return rte_flow_error_set
4055                                 (error, EINVAL,
4056                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4057                                  "port does not belong to"
4058                                  " E-Switch being configured");
4059         return 0;
4060 }
4061
4062 /**
4063  * Get the maximum number of modify header actions.
4064  *
4065  * @param dev
4066  *   Pointer to rte_eth_dev structure.
4067  * @param flags
4068  *   Flags bits to check if root level.
4069  *
4070  * @return
4071  *   Max number of modify header actions device can support.
4072  */
4073 static inline unsigned int
4074 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
4075                               uint64_t flags)
4076 {
4077         /*
4078          * There's no way to directly query the max capacity from FW.
4079          * The maximal value on root table should be assumed to be supported.
4080          */
4081         if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
4082                 return MLX5_MAX_MODIFY_NUM;
4083         else
4084                 return MLX5_ROOT_TBL_MODIFY_NUM;
4085 }
4086
4087 /**
4088  * Validate the meter action.
4089  *
4090  * @param[in] dev
4091  *   Pointer to rte_eth_dev structure.
4092  * @param[in] action_flags
4093  *   Bit-fields that holds the actions detected until now.
4094  * @param[in] action
4095  *   Pointer to the meter action.
4096  * @param[in] attr
4097  *   Attributes of flow that includes this action.
4098  * @param[out] error
4099  *   Pointer to error structure.
4100  *
4101  * @return
4102  *   0 on success, a negative errno value otherwise and rte_ernno is set.
4103  */
4104 static int
4105 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
4106                                 uint64_t action_flags,
4107                                 const struct rte_flow_action *action,
4108                                 const struct rte_flow_attr *attr,
4109                                 struct rte_flow_error *error)
4110 {
4111         struct mlx5_priv *priv = dev->data->dev_private;
4112         const struct rte_flow_action_meter *am = action->conf;
4113         struct mlx5_flow_meter *fm;
4114
4115         if (!am)
4116                 return rte_flow_error_set(error, EINVAL,
4117                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4118                                           "meter action conf is NULL");
4119
4120         if (action_flags & MLX5_FLOW_ACTION_METER)
4121                 return rte_flow_error_set(error, ENOTSUP,
4122                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4123                                           "meter chaining not support");
4124         if (action_flags & MLX5_FLOW_ACTION_JUMP)
4125                 return rte_flow_error_set(error, ENOTSUP,
4126                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4127                                           "meter with jump not support");
4128         if (!priv->mtr_en)
4129                 return rte_flow_error_set(error, ENOTSUP,
4130                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4131                                           NULL,
4132                                           "meter action not supported");
4133         fm = mlx5_flow_meter_find(priv, am->mtr_id);
4134         if (!fm)
4135                 return rte_flow_error_set(error, EINVAL,
4136                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4137                                           "Meter not found");
4138         if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
4139               (!fm->ingress && !attr->ingress && attr->egress) ||
4140               (!fm->egress && !attr->egress && attr->ingress))))
4141                 return rte_flow_error_set(error, EINVAL,
4142                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4143                                           "Flow attributes are either invalid "
4144                                           "or have a conflict with current "
4145                                           "meter attributes");
4146         return 0;
4147 }
4148
4149 /**
4150  * Validate the age action.
4151  *
4152  * @param[in] action_flags
4153  *   Holds the actions detected until now.
4154  * @param[in] action
4155  *   Pointer to the age action.
4156  * @param[in] dev
4157  *   Pointer to the Ethernet device structure.
4158  * @param[out] error
4159  *   Pointer to error structure.
4160  *
4161  * @return
4162  *   0 on success, a negative errno value otherwise and rte_errno is set.
4163  */
4164 static int
4165 flow_dv_validate_action_age(uint64_t action_flags,
4166                             const struct rte_flow_action *action,
4167                             struct rte_eth_dev *dev,
4168                             struct rte_flow_error *error)
4169 {
4170         struct mlx5_priv *priv = dev->data->dev_private;
4171         const struct rte_flow_action_age *age = action->conf;
4172
4173         if (!priv->config.devx || priv->counter_fallback)
4174                 return rte_flow_error_set(error, ENOTSUP,
4175                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4176                                           NULL,
4177                                           "age action not supported");
4178         if (!(action->conf))
4179                 return rte_flow_error_set(error, EINVAL,
4180                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4181                                           "configuration cannot be null");
4182         if (!(age->timeout))
4183                 return rte_flow_error_set(error, EINVAL,
4184                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4185                                           "invalid timeout value 0");
4186         if (action_flags & MLX5_FLOW_ACTION_AGE)
4187                 return rte_flow_error_set(error, EINVAL,
4188                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4189                                           "duplicate age actions set");
4190         return 0;
4191 }
4192
4193 /**
4194  * Validate the modify-header IPv4 DSCP actions.
4195  *
4196  * @param[in] action_flags
4197  *   Holds the actions detected until now.
4198  * @param[in] action
4199  *   Pointer to the modify action.
4200  * @param[in] item_flags
4201  *   Holds the items detected.
4202  * @param[out] error
4203  *   Pointer to error structure.
4204  *
4205  * @return
4206  *   0 on success, a negative errno value otherwise and rte_errno is set.
4207  */
4208 static int
4209 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
4210                                          const struct rte_flow_action *action,
4211                                          const uint64_t item_flags,
4212                                          struct rte_flow_error *error)
4213 {
4214         int ret = 0;
4215
4216         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4217         if (!ret) {
4218                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
4219                         return rte_flow_error_set(error, EINVAL,
4220                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4221                                                   NULL,
4222                                                   "no ipv4 item in pattern");
4223         }
4224         return ret;
4225 }
4226
4227 /**
4228  * Validate the modify-header IPv6 DSCP actions.
4229  *
4230  * @param[in] action_flags
4231  *   Holds the actions detected until now.
4232  * @param[in] action
4233  *   Pointer to the modify action.
4234  * @param[in] item_flags
4235  *   Holds the items detected.
4236  * @param[out] error
4237  *   Pointer to error structure.
4238  *
4239  * @return
4240  *   0 on success, a negative errno value otherwise and rte_errno is set.
4241  */
4242 static int
4243 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
4244                                          const struct rte_flow_action *action,
4245                                          const uint64_t item_flags,
4246                                          struct rte_flow_error *error)
4247 {
4248         int ret = 0;
4249
4250         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4251         if (!ret) {
4252                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
4253                         return rte_flow_error_set(error, EINVAL,
4254                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4255                                                   NULL,
4256                                                   "no ipv6 item in pattern");
4257         }
4258         return ret;
4259 }
4260
4261 /**
4262  * Match modify-header resource.
4263  *
4264  * @param entry
4265  *   Pointer to exist resource entry object.
4266  * @param ctx
4267  *   Pointer to new modify-header resource.
4268  *
4269  * @return
4270  *   0 on matching, -1 otherwise.
4271  */
4272 static int
4273 flow_dv_modify_hdr_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
4274 {
4275         struct mlx5_flow_dv_modify_hdr_resource *resource;
4276         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4277         uint32_t actions_len;
4278
4279         resource = (struct mlx5_flow_dv_modify_hdr_resource *)ctx;
4280         cache_resource = container_of(entry,
4281                                       struct mlx5_flow_dv_modify_hdr_resource,
4282                                       entry);
4283         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4284         if (resource->entry.key == cache_resource->entry.key &&
4285             resource->ft_type == cache_resource->ft_type &&
4286             resource->actions_num == cache_resource->actions_num &&
4287             resource->flags == cache_resource->flags &&
4288             !memcmp((const void *)resource->actions,
4289                     (const void *)cache_resource->actions,
4290                     actions_len))
4291                 return 0;
4292         return -1;
4293 }
4294
4295 /**
4296  * Validate the sample action.
4297  *
4298  * @param[in] action_flags
4299  *   Holds the actions detected until now.
4300  * @param[in] action
4301  *   Pointer to the sample action.
4302  * @param[in] dev
4303  *   Pointer to the Ethernet device structure.
4304  * @param[in] attr
4305  *   Attributes of flow that includes this action.
4306  * @param[out] error
4307  *   Pointer to error structure.
4308  *
4309  * @return
4310  *   0 on success, a negative errno value otherwise and rte_errno is set.
4311  */
4312 static int
4313 flow_dv_validate_action_sample(uint64_t action_flags,
4314                                const struct rte_flow_action *action,
4315                                struct rte_eth_dev *dev,
4316                                const struct rte_flow_attr *attr,
4317                                struct rte_flow_error *error)
4318 {
4319         struct mlx5_priv *priv = dev->data->dev_private;
4320         struct mlx5_dev_config *dev_conf = &priv->config;
4321         const struct rte_flow_action_sample *sample = action->conf;
4322         const struct rte_flow_action *act;
4323         uint64_t sub_action_flags = 0;
4324         uint16_t queue_index = 0xFFFF;
4325         int actions_n = 0;
4326         int ret;
4327         fdb_mirror = 0;
4328
4329         if (!sample)
4330                 return rte_flow_error_set(error, EINVAL,
4331                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4332                                           "configuration cannot be NULL");
4333         if (sample->ratio == 0)
4334                 return rte_flow_error_set(error, EINVAL,
4335                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4336                                           "ratio value starts from 1");
4337         if (!priv->config.devx || (sample->ratio > 0 && !priv->sampler_en))
4338                 return rte_flow_error_set(error, ENOTSUP,
4339                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4340                                           NULL,
4341                                           "sample action not supported");
4342         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
4343                 return rte_flow_error_set(error, EINVAL,
4344                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4345                                           "Multiple sample actions not "
4346                                           "supported");
4347         if (action_flags & MLX5_FLOW_ACTION_METER)
4348                 return rte_flow_error_set(error, EINVAL,
4349                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4350                                           "wrong action order, meter should "
4351                                           "be after sample action");
4352         if (action_flags & MLX5_FLOW_ACTION_JUMP)
4353                 return rte_flow_error_set(error, EINVAL,
4354                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4355                                           "wrong action order, jump should "
4356                                           "be after sample action");
4357         act = sample->actions;
4358         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
4359                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
4360                         return rte_flow_error_set(error, ENOTSUP,
4361                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4362                                                   act, "too many actions");
4363                 switch (act->type) {
4364                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4365                         ret = mlx5_flow_validate_action_queue(act,
4366                                                               sub_action_flags,
4367                                                               dev,
4368                                                               attr, error);
4369                         if (ret < 0)
4370                                 return ret;
4371                         queue_index = ((const struct rte_flow_action_queue *)
4372                                                         (act->conf))->index;
4373                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
4374                         ++actions_n;
4375                         break;
4376                 case RTE_FLOW_ACTION_TYPE_MARK:
4377                         ret = flow_dv_validate_action_mark(dev, act,
4378                                                            sub_action_flags,
4379                                                            attr, error);
4380                         if (ret < 0)
4381                                 return ret;
4382                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
4383                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
4384                                                 MLX5_FLOW_ACTION_MARK_EXT;
4385                         else
4386                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
4387                         ++actions_n;
4388                         break;
4389                 case RTE_FLOW_ACTION_TYPE_COUNT:
4390                         ret = flow_dv_validate_action_count(dev, error);
4391                         if (ret < 0)
4392                                 return ret;
4393                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
4394                         ++actions_n;
4395                         break;
4396                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
4397                         ret = flow_dv_validate_action_port_id(dev,
4398                                                               sub_action_flags,
4399                                                               act,
4400                                                               attr,
4401                                                               error);
4402                         if (ret)
4403                                 return ret;
4404                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
4405                         ++actions_n;
4406                         break;
4407                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4408                         ret = flow_dv_validate_action_raw_encap_decap
4409                                 (dev, NULL, act->conf, attr, &sub_action_flags,
4410                                  &actions_n, error);
4411                         if (ret < 0)
4412                                 return ret;
4413                         ++actions_n;
4414                         break;
4415                 default:
4416                         return rte_flow_error_set(error, ENOTSUP,
4417                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4418                                                   NULL,
4419                                                   "Doesn't support optional "
4420                                                   "action");
4421                 }
4422         }
4423         if (attr->ingress && !attr->transfer) {
4424                 if (!(sub_action_flags & MLX5_FLOW_ACTION_QUEUE))
4425                         return rte_flow_error_set(error, EINVAL,
4426                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4427                                                   NULL,
4428                                                   "Ingress must has a dest "
4429                                                   "QUEUE for Sample");
4430         } else if (attr->egress && !attr->transfer) {
4431                 return rte_flow_error_set(error, ENOTSUP,
4432                                           RTE_FLOW_ERROR_TYPE_ACTION,
4433                                           NULL,
4434                                           "Sample Only support Ingress "
4435                                           "or E-Switch");
4436         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
4437                 MLX5_ASSERT(attr->transfer);
4438                 if (sample->ratio > 1)
4439                         return rte_flow_error_set(error, ENOTSUP,
4440                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4441                                                   NULL,
4442                                                   "E-Switch doesn't support "
4443                                                   "any optional action "
4444                                                   "for sampling");
4445                 fdb_mirror = 1;
4446                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
4447                         return rte_flow_error_set(error, ENOTSUP,
4448                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4449                                                   NULL,
4450                                                   "unsupported action QUEUE");
4451                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
4452                         return rte_flow_error_set(error, EINVAL,
4453                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4454                                                   NULL,
4455                                                   "E-Switch must has a dest "
4456                                                   "port for mirroring");
4457         }
4458         /* Continue validation for Xcap actions.*/
4459         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
4460             (queue_index == 0xFFFF ||
4461              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
4462                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
4463                      MLX5_FLOW_XCAP_ACTIONS)
4464                         return rte_flow_error_set(error, ENOTSUP,
4465                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4466                                                   NULL, "encap and decap "
4467                                                   "combination aren't "
4468                                                   "supported");
4469                 if (!attr->transfer && attr->ingress && (sub_action_flags &
4470                                                         MLX5_FLOW_ACTION_ENCAP))
4471                         return rte_flow_error_set(error, ENOTSUP,
4472                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4473                                                   NULL, "encap is not supported"
4474                                                   " for ingress traffic");
4475         }
4476         return 0;
4477 }
4478
4479 /**
4480  * Find existing modify-header resource or create and register a new one.
4481  *
4482  * @param dev[in, out]
4483  *   Pointer to rte_eth_dev structure.
4484  * @param[in, out] resource
4485  *   Pointer to modify-header resource.
4486  * @parm[in, out] dev_flow
4487  *   Pointer to the dev_flow.
4488  * @param[out] error
4489  *   pointer to error structure.
4490  *
4491  * @return
4492  *   0 on success otherwise -errno and errno is set.
4493  */
4494 static int
4495 flow_dv_modify_hdr_resource_register
4496                         (struct rte_eth_dev *dev,
4497                          struct mlx5_flow_dv_modify_hdr_resource *resource,
4498                          struct mlx5_flow *dev_flow,
4499                          struct rte_flow_error *error)
4500 {
4501         struct mlx5_priv *priv = dev->data->dev_private;
4502         struct mlx5_dev_ctx_shared *sh = priv->sh;
4503         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4504         struct mlx5dv_dr_domain *ns;
4505         uint32_t actions_len;
4506         struct mlx5_hlist_entry *entry;
4507         union mlx5_flow_modify_hdr_key hdr_mod_key = {
4508                 {
4509                         .ft_type = resource->ft_type,
4510                         .actions_num = resource->actions_num,
4511                         .group = dev_flow->dv.group,
4512                         .cksum = 0,
4513                 }
4514         };
4515         int ret;
4516
4517         resource->flags = dev_flow->dv.group ? 0 :
4518                           MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4519         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
4520                                     resource->flags))
4521                 return rte_flow_error_set(error, EOVERFLOW,
4522                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4523                                           "too many modify header items");
4524         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
4525                 ns = sh->fdb_domain;
4526         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
4527                 ns = sh->tx_domain;
4528         else
4529                 ns = sh->rx_domain;
4530         /* Lookup a matching resource from cache. */
4531         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4532         hdr_mod_key.cksum = __rte_raw_cksum(resource->actions, actions_len, 0);
4533         resource->entry.key = hdr_mod_key.v64;
4534         entry = mlx5_hlist_lookup_ex(sh->modify_cmds, resource->entry.key,
4535                                      flow_dv_modify_hdr_resource_match,
4536                                      (void *)resource);
4537         if (entry) {
4538                 cache_resource = container_of(entry,
4539                                         struct mlx5_flow_dv_modify_hdr_resource,
4540                                         entry);
4541                 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
4542                         (void *)cache_resource,
4543                         rte_atomic32_read(&cache_resource->refcnt));
4544                 rte_atomic32_inc(&cache_resource->refcnt);
4545                 dev_flow->handle->dvh.modify_hdr = cache_resource;
4546                 return 0;
4547
4548         }
4549         /* Register new modify-header resource. */
4550         cache_resource = mlx5_malloc(MLX5_MEM_ZERO,
4551                                     sizeof(*cache_resource) + actions_len, 0,
4552                                     SOCKET_ID_ANY);
4553         if (!cache_resource)
4554                 return rte_flow_error_set(error, ENOMEM,
4555                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4556                                           "cannot allocate resource memory");
4557         *cache_resource = *resource;
4558         rte_memcpy(cache_resource->actions, resource->actions, actions_len);
4559         ret = mlx5_flow_os_create_flow_action_modify_header
4560                                         (sh->ctx, ns, cache_resource,
4561                                          actions_len, &cache_resource->action);
4562         if (ret) {
4563                 mlx5_free(cache_resource);
4564                 return rte_flow_error_set(error, ENOMEM,
4565                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4566                                           NULL, "cannot create action");
4567         }
4568         rte_atomic32_init(&cache_resource->refcnt);
4569         rte_atomic32_inc(&cache_resource->refcnt);
4570         if (mlx5_hlist_insert_ex(sh->modify_cmds, &cache_resource->entry,
4571                                  flow_dv_modify_hdr_resource_match,
4572                                  (void *)cache_resource)) {
4573                 claim_zero(mlx5_flow_os_destroy_flow_action
4574                                                 (cache_resource->action));
4575                 mlx5_free(cache_resource);
4576                 return rte_flow_error_set(error, EEXIST,
4577                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4578                                           NULL, "action exist");
4579         }
4580         dev_flow->handle->dvh.modify_hdr = cache_resource;
4581         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
4582                 (void *)cache_resource,
4583                 rte_atomic32_read(&cache_resource->refcnt));
4584         return 0;
4585 }
4586
4587 /**
4588  * Get DV flow counter by index.
4589  *
4590  * @param[in] dev
4591  *   Pointer to the Ethernet device structure.
4592  * @param[in] idx
4593  *   mlx5 flow counter index in the container.
4594  * @param[out] ppool
4595  *   mlx5 flow counter pool in the container,
4596  *
4597  * @return
4598  *   Pointer to the counter, NULL otherwise.
4599  */
4600 static struct mlx5_flow_counter *
4601 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
4602                            uint32_t idx,
4603                            struct mlx5_flow_counter_pool **ppool)
4604 {
4605         struct mlx5_priv *priv = dev->data->dev_private;
4606         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4607         struct mlx5_flow_counter_pool *pool;
4608
4609         /* Decrease to original index and clear shared bit. */
4610         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
4611         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cmng->n);
4612         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
4613         MLX5_ASSERT(pool);
4614         if (ppool)
4615                 *ppool = pool;
4616         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
4617 }
4618
4619 /**
4620  * Check the devx counter belongs to the pool.
4621  *
4622  * @param[in] pool
4623  *   Pointer to the counter pool.
4624  * @param[in] id
4625  *   The counter devx ID.
4626  *
4627  * @return
4628  *   True if counter belongs to the pool, false otherwise.
4629  */
4630 static bool
4631 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
4632 {
4633         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
4634                    MLX5_COUNTERS_PER_POOL;
4635
4636         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
4637                 return true;
4638         return false;
4639 }
4640
4641 /**
4642  * Get a pool by devx counter ID.
4643  *
4644  * @param[in] cmng
4645  *   Pointer to the counter management.
4646  * @param[in] id
4647  *   The counter devx ID.
4648  *
4649  * @return
4650  *   The counter pool pointer if exists, NULL otherwise,
4651  */
4652 static struct mlx5_flow_counter_pool *
4653 flow_dv_find_pool_by_id(struct mlx5_flow_counter_mng *cmng, int id)
4654 {
4655         uint32_t i;
4656         struct mlx5_flow_counter_pool *pool = NULL;
4657
4658         rte_spinlock_lock(&cmng->pool_update_sl);
4659         /* Check last used pool. */
4660         if (cmng->last_pool_idx != POOL_IDX_INVALID &&
4661             flow_dv_is_counter_in_pool(cmng->pools[cmng->last_pool_idx], id)) {
4662                 pool = cmng->pools[cmng->last_pool_idx];
4663                 goto out;
4664         }
4665         /* ID out of range means no suitable pool in the container. */
4666         if (id > cmng->max_id || id < cmng->min_id)
4667                 goto out;
4668         /*
4669          * Find the pool from the end of the container, since mostly counter
4670          * ID is sequence increasing, and the last pool should be the needed
4671          * one.
4672          */
4673         i = cmng->n_valid;
4674         while (i--) {
4675                 struct mlx5_flow_counter_pool *pool_tmp = cmng->pools[i];
4676
4677                 if (flow_dv_is_counter_in_pool(pool_tmp, id)) {
4678                         pool = pool_tmp;
4679                         break;
4680                 }
4681         }
4682 out:
4683         rte_spinlock_unlock(&cmng->pool_update_sl);
4684         return pool;
4685 }
4686
4687 /**
4688  * Resize a counter container.
4689  *
4690  * @param[in] dev
4691  *   Pointer to the Ethernet device structure.
4692  *
4693  * @return
4694  *   0 on success, otherwise negative errno value and rte_errno is set.
4695  */
4696 static int
4697 flow_dv_container_resize(struct rte_eth_dev *dev)
4698 {
4699         struct mlx5_priv *priv = dev->data->dev_private;
4700         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4701         void *old_pools = cmng->pools;
4702         uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
4703         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
4704         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
4705
4706         if (!pools) {
4707                 rte_errno = ENOMEM;
4708                 return -ENOMEM;
4709         }
4710         if (old_pools)
4711                 memcpy(pools, old_pools, cmng->n *
4712                                        sizeof(struct mlx5_flow_counter_pool *));
4713         cmng->n = resize;
4714         cmng->pools = pools;
4715         if (old_pools)
4716                 mlx5_free(old_pools);
4717         return 0;
4718 }
4719
4720 /**
4721  * Query a devx flow counter.
4722  *
4723  * @param[in] dev
4724  *   Pointer to the Ethernet device structure.
4725  * @param[in] cnt
4726  *   Index to the flow counter.
4727  * @param[out] pkts
4728  *   The statistics value of packets.
4729  * @param[out] bytes
4730  *   The statistics value of bytes.
4731  *
4732  * @return
4733  *   0 on success, otherwise a negative errno value and rte_errno is set.
4734  */
4735 static inline int
4736 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
4737                      uint64_t *bytes)
4738 {
4739         struct mlx5_priv *priv = dev->data->dev_private;
4740         struct mlx5_flow_counter_pool *pool = NULL;
4741         struct mlx5_flow_counter *cnt;
4742         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4743         int offset;
4744
4745         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4746         MLX5_ASSERT(pool);
4747         if (priv->counter_fallback) {
4748                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4749                 return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
4750                                         0, pkts, bytes, 0, NULL, NULL, 0);
4751         }
4752         rte_spinlock_lock(&pool->sl);
4753         if (!pool->raw) {
4754                 *pkts = 0;
4755                 *bytes = 0;
4756         } else {
4757                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
4758                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
4759                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
4760         }
4761         rte_spinlock_unlock(&pool->sl);
4762         return 0;
4763 }
4764
4765 /**
4766  * Create and initialize a new counter pool.
4767  *
4768  * @param[in] dev
4769  *   Pointer to the Ethernet device structure.
4770  * @param[out] dcs
4771  *   The devX counter handle.
4772  * @param[in] age
4773  *   Whether the pool is for counter that was allocated for aging.
4774  * @param[in/out] cont_cur
4775  *   Pointer to the container pointer, it will be update in pool resize.
4776  *
4777  * @return
4778  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
4779  */
4780 static struct mlx5_flow_counter_pool *
4781 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
4782                     uint32_t age)
4783 {
4784         struct mlx5_priv *priv = dev->data->dev_private;
4785         struct mlx5_flow_counter_pool *pool;
4786         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4787         uint32_t fallback = priv->counter_fallback;
4788         uint32_t size = sizeof(*pool);
4789
4790         size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
4791         size += (!fallback ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
4792         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE);
4793         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
4794         if (!pool) {
4795                 rte_errno = ENOMEM;
4796                 return NULL;
4797         }
4798         pool->raw = NULL;
4799         pool->type = 0;
4800         pool->type |= (!age ? 0 :  CNT_POOL_TYPE_AGE);
4801         pool->query_gen = 0;
4802         pool->min_dcs = dcs;
4803         rte_spinlock_init(&pool->sl);
4804         rte_spinlock_init(&pool->csl);
4805         TAILQ_INIT(&pool->counters[0]);
4806         TAILQ_INIT(&pool->counters[1]);
4807         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
4808         rte_spinlock_lock(&cmng->pool_update_sl);
4809         pool->index = cmng->n_valid;
4810         if (pool->index == cmng->n && flow_dv_container_resize(dev)) {
4811                 mlx5_free(pool);
4812                 rte_spinlock_unlock(&cmng->pool_update_sl);
4813                 return NULL;
4814         }
4815         cmng->pools[pool->index] = pool;
4816         cmng->n_valid++;
4817         if (unlikely(fallback)) {
4818                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
4819
4820                 if (base < cmng->min_id)
4821                         cmng->min_id = base;
4822                 if (base > cmng->max_id)
4823                         cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
4824                 cmng->last_pool_idx = pool->index;
4825                 pool->type |= CNT_POOL_TYPE_EXT;
4826         }
4827         rte_spinlock_unlock(&cmng->pool_update_sl);
4828         return pool;
4829 }
4830
4831 /**
4832  * Prepare a new counter and/or a new counter pool.
4833  *
4834  * @param[in] dev
4835  *   Pointer to the Ethernet device structure.
4836  * @param[out] cnt_free
4837  *   Where to put the pointer of a new counter.
4838  * @param[in] age
4839  *   Whether the pool is for counter that was allocated for aging.
4840  *
4841  * @return
4842  *   The counter pool pointer and @p cnt_free is set on success,
4843  *   NULL otherwise and rte_errno is set.
4844  */
4845 static struct mlx5_flow_counter_pool *
4846 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
4847                              struct mlx5_flow_counter **cnt_free,
4848                              uint32_t age)
4849 {
4850         struct mlx5_priv *priv = dev->data->dev_private;
4851         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4852         struct mlx5_flow_counter_pool *pool;
4853         struct mlx5_counters tmp_tq;
4854         struct mlx5_devx_obj *dcs = NULL;
4855         struct mlx5_flow_counter *cnt;
4856         enum mlx5_counter_type cnt_type =
4857                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
4858         uint32_t fallback = priv->counter_fallback;
4859         uint32_t i;
4860
4861         if (fallback) {
4862                 /* bulk_bitmap must be 0 for single counter allocation. */
4863                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
4864                 if (!dcs)
4865                         return NULL;
4866                 pool = flow_dv_find_pool_by_id(cmng, dcs->id);
4867                 if (!pool) {
4868                         pool = flow_dv_pool_create(dev, dcs, age);
4869                         if (!pool) {
4870                                 mlx5_devx_cmd_destroy(dcs);
4871                                 return NULL;
4872                         }
4873                 }
4874                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
4875                 cnt = MLX5_POOL_GET_CNT(pool, i);
4876                 cnt->pool = pool;
4877                 MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs;
4878                 *cnt_free = cnt;
4879                 return pool;
4880         }
4881         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
4882         if (!dcs) {
4883                 rte_errno = ENODATA;
4884                 return NULL;
4885         }
4886         pool = flow_dv_pool_create(dev, dcs, age);
4887         if (!pool) {
4888                 mlx5_devx_cmd_destroy(dcs);
4889                 return NULL;
4890         }
4891         TAILQ_INIT(&tmp_tq);
4892         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
4893                 cnt = MLX5_POOL_GET_CNT(pool, i);
4894                 cnt->pool = pool;
4895                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
4896         }
4897         rte_spinlock_lock(&cmng->csl[cnt_type]);
4898         TAILQ_CONCAT(&cmng->counters[cnt_type], &tmp_tq, next);
4899         rte_spinlock_unlock(&cmng->csl[cnt_type]);
4900         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
4901         (*cnt_free)->pool = pool;
4902         return pool;
4903 }
4904
4905 /**
4906  * Allocate a flow counter.
4907  *
4908  * @param[in] dev
4909  *   Pointer to the Ethernet device structure.
4910  * @param[in] age
4911  *   Whether the counter was allocated for aging.
4912  *
4913  * @return
4914  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
4915  */
4916 static uint32_t
4917 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t age)
4918 {
4919         struct mlx5_priv *priv = dev->data->dev_private;
4920         struct mlx5_flow_counter_pool *pool = NULL;
4921         struct mlx5_flow_counter *cnt_free = NULL;
4922         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4923         uint32_t fallback = priv->counter_fallback;
4924         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4925         enum mlx5_counter_type cnt_type =
4926                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
4927         uint32_t cnt_idx;
4928
4929         if (!priv->config.devx) {
4930                 rte_errno = ENOTSUP;
4931                 return 0;
4932         }
4933         /* Get free counters from container. */
4934         rte_spinlock_lock(&cmng->csl[cnt_type]);
4935         cnt_free = TAILQ_FIRST(&cmng->counters[cnt_type]);
4936         if (cnt_free)
4937                 TAILQ_REMOVE(&cmng->counters[cnt_type], cnt_free, next);
4938         rte_spinlock_unlock(&cmng->csl[cnt_type]);
4939         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, age))
4940                 goto err;
4941         pool = cnt_free->pool;
4942         if (fallback)
4943                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free);
4944         /* Create a DV counter action only in the first time usage. */
4945         if (!cnt_free->action) {
4946                 uint16_t offset;
4947                 struct mlx5_devx_obj *dcs;
4948                 int ret;
4949
4950                 if (!fallback) {
4951                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
4952                         dcs = pool->min_dcs;
4953                 } else {
4954                         offset = 0;
4955                         dcs = cnt_ext->dcs;
4956                 }
4957                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
4958                                                             &cnt_free->action);
4959                 if (ret) {
4960                         rte_errno = errno;
4961                         goto err;
4962                 }
4963         }
4964         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
4965                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
4966         /* Update the counter reset values. */
4967         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
4968                                  &cnt_free->bytes))
4969                 goto err;
4970         if (!fallback && !priv->sh->cmng.query_thread_on)
4971                 /* Start the asynchronous batch query by the host thread. */
4972                 mlx5_set_query_alarm(priv->sh);
4973         return cnt_idx;
4974 err:
4975         if (cnt_free) {
4976                 cnt_free->pool = pool;
4977                 rte_spinlock_lock(&cmng->csl[cnt_type]);
4978                 TAILQ_INSERT_TAIL(&cmng->counters[cnt_type], cnt_free, next);
4979                 rte_spinlock_unlock(&cmng->csl[cnt_type]);
4980         }
4981         return 0;
4982 }
4983
4984 /**
4985  * Allocate a shared flow counter.
4986  *
4987  * @param[in] ctx
4988  *   Pointer to the shared counter configuration.
4989  * @param[in] data
4990  *   Pointer to save the allocated counter index.
4991  *
4992  * @return
4993  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
4994  */
4995
4996 static int32_t
4997 flow_dv_counter_alloc_shared_cb(void *ctx, union mlx5_l3t_data *data)
4998 {
4999         struct mlx5_shared_counter_conf *conf = ctx;
5000         struct rte_eth_dev *dev = conf->dev;
5001         struct mlx5_flow_counter *cnt;
5002
5003         data->dword = flow_dv_counter_alloc(dev, 0);
5004         data->dword |= MLX5_CNT_SHARED_OFFSET;
5005         cnt = flow_dv_counter_get_by_idx(dev, data->dword, NULL);
5006         cnt->shared_info.id = conf->id;
5007         return 0;
5008 }
5009
5010 /**
5011  * Get a shared flow counter.
5012  *
5013  * @param[in] dev
5014  *   Pointer to the Ethernet device structure.
5015  * @param[in] id
5016  *   Counter identifier.
5017  *
5018  * @return
5019  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
5020  */
5021 static uint32_t
5022 flow_dv_counter_get_shared(struct rte_eth_dev *dev, uint32_t id)
5023 {
5024         struct mlx5_priv *priv = dev->data->dev_private;
5025         struct mlx5_shared_counter_conf conf = {
5026                 .dev = dev,
5027                 .id = id,
5028         };
5029         union mlx5_l3t_data data = {
5030                 .dword = 0,
5031         };
5032
5033         mlx5_l3t_prepare_entry(priv->sh->cnt_id_tbl, id, &data,
5034                                flow_dv_counter_alloc_shared_cb, &conf);
5035         return data.dword;
5036 }
5037
5038 /**
5039  * Get age param from counter index.
5040  *
5041  * @param[in] dev
5042  *   Pointer to the Ethernet device structure.
5043  * @param[in] counter
5044  *   Index to the counter handler.
5045  *
5046  * @return
5047  *   The aging parameter specified for the counter index.
5048  */
5049 static struct mlx5_age_param*
5050 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
5051                                 uint32_t counter)
5052 {
5053         struct mlx5_flow_counter *cnt;
5054         struct mlx5_flow_counter_pool *pool = NULL;
5055
5056         flow_dv_counter_get_by_idx(dev, counter, &pool);
5057         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
5058         cnt = MLX5_POOL_GET_CNT(pool, counter);
5059         return MLX5_CNT_TO_AGE(cnt);
5060 }
5061
5062 /**
5063  * Remove a flow counter from aged counter list.
5064  *
5065  * @param[in] dev
5066  *   Pointer to the Ethernet device structure.
5067  * @param[in] counter
5068  *   Index to the counter handler.
5069  * @param[in] cnt
5070  *   Pointer to the counter handler.
5071  */
5072 static void
5073 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
5074                                 uint32_t counter, struct mlx5_flow_counter *cnt)
5075 {
5076         struct mlx5_age_info *age_info;
5077         struct mlx5_age_param *age_param;
5078         struct mlx5_priv *priv = dev->data->dev_private;
5079         uint16_t expected = AGE_CANDIDATE;
5080
5081         age_info = GET_PORT_AGE_INFO(priv);
5082         age_param = flow_dv_counter_idx_get_age(dev, counter);
5083         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
5084                                          AGE_FREE, false, __ATOMIC_RELAXED,
5085                                          __ATOMIC_RELAXED)) {
5086                 /**
5087                  * We need the lock even it is age timeout,
5088                  * since counter may still in process.
5089                  */
5090                 rte_spinlock_lock(&age_info->aged_sl);
5091                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
5092                 rte_spinlock_unlock(&age_info->aged_sl);
5093                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
5094         }
5095 }
5096
5097 /**
5098  * Release a flow counter.
5099  *
5100  * @param[in] dev
5101  *   Pointer to the Ethernet device structure.
5102  * @param[in] counter
5103  *   Index to the counter handler.
5104  */
5105 static void
5106 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
5107 {
5108         struct mlx5_priv *priv = dev->data->dev_private;
5109         struct mlx5_flow_counter_pool *pool = NULL;
5110         struct mlx5_flow_counter *cnt;
5111         enum mlx5_counter_type cnt_type;
5112
5113         if (!counter)
5114                 return;
5115         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
5116         MLX5_ASSERT(pool);
5117         if (IS_SHARED_CNT(counter) &&
5118             mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, cnt->shared_info.id))
5119                 return;
5120         if (IS_AGE_POOL(pool))
5121                 flow_dv_counter_remove_from_age(dev, counter, cnt);
5122         cnt->pool = pool;
5123         /*
5124          * Put the counter back to list to be updated in none fallback mode.
5125          * Currently, we are using two list alternately, while one is in query,
5126          * add the freed counter to the other list based on the pool query_gen
5127          * value. After query finishes, add counter the list to the global
5128          * container counter list. The list changes while query starts. In
5129          * this case, lock will not be needed as query callback and release
5130          * function both operate with the different list.
5131          *
5132          */
5133         if (!priv->counter_fallback) {
5134                 rte_spinlock_lock(&pool->csl);
5135                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
5136                 rte_spinlock_unlock(&pool->csl);
5137         } else {
5138                 cnt_type = IS_AGE_POOL(pool) ? MLX5_COUNTER_TYPE_AGE :
5139                                                MLX5_COUNTER_TYPE_ORIGIN;
5140                 rte_spinlock_lock(&priv->sh->cmng.csl[cnt_type]);
5141                 TAILQ_INSERT_TAIL(&priv->sh->cmng.counters[cnt_type],
5142                                   cnt, next);
5143                 rte_spinlock_unlock(&priv->sh->cmng.csl[cnt_type]);
5144         }
5145 }
5146
5147 /**
5148  * Verify the @p attributes will be correctly understood by the NIC and store
5149  * them in the @p flow if everything is correct.
5150  *
5151  * @param[in] dev
5152  *   Pointer to dev struct.
5153  * @param[in] attributes
5154  *   Pointer to flow attributes
5155  * @param[in] external
5156  *   This flow rule is created by request external to PMD.
5157  * @param[out] error
5158  *   Pointer to error structure.
5159  *
5160  * @return
5161  *   - 0 on success and non root table.
5162  *   - 1 on success and root table.
5163  *   - a negative errno value otherwise and rte_errno is set.
5164  */
5165 static int
5166 flow_dv_validate_attributes(struct rte_eth_dev *dev,
5167                             const struct rte_flow_attr *attributes,
5168                             bool external __rte_unused,
5169                             struct rte_flow_error *error)
5170 {
5171         struct mlx5_priv *priv = dev->data->dev_private;
5172         uint32_t priority_max = priv->config.flow_prio - 1;
5173         int ret = 0;
5174
5175 #ifndef HAVE_MLX5DV_DR
5176         if (attributes->group)
5177                 return rte_flow_error_set(error, ENOTSUP,
5178                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
5179                                           NULL,
5180                                           "groups are not supported");
5181 #else
5182         uint32_t table = 0;
5183
5184         ret = mlx5_flow_group_to_table(attributes, external,
5185                                        attributes->group, !!priv->fdb_def_rule,
5186                                        &table, error);
5187         if (ret)
5188                 return ret;
5189         if (!table)
5190                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
5191 #endif
5192         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
5193             attributes->priority >= priority_max)
5194                 return rte_flow_error_set(error, ENOTSUP,
5195                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
5196                                           NULL,
5197                                           "priority out of range");
5198         if (attributes->transfer) {
5199                 if (!priv->config.dv_esw_en)
5200                         return rte_flow_error_set
5201                                 (error, ENOTSUP,
5202                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5203                                  "E-Switch dr is not supported");
5204                 if (!(priv->representor || priv->master))
5205                         return rte_flow_error_set
5206                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5207                                  NULL, "E-Switch configuration can only be"
5208                                  " done by a master or a representor device");
5209                 if (attributes->egress)
5210                         return rte_flow_error_set
5211                                 (error, ENOTSUP,
5212                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
5213                                  "egress is not supported");
5214         }
5215         if (!(attributes->egress ^ attributes->ingress))
5216                 return rte_flow_error_set(error, ENOTSUP,
5217                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
5218                                           "must specify exactly one of "
5219                                           "ingress or egress");
5220         return ret;
5221 }
5222
5223 /**
5224  * Internal validation function. For validating both actions and items.
5225  *
5226  * @param[in] dev
5227  *   Pointer to the rte_eth_dev structure.
5228  * @param[in] attr
5229  *   Pointer to the flow attributes.
5230  * @param[in] items
5231  *   Pointer to the list of items.
5232  * @param[in] actions
5233  *   Pointer to the list of actions.
5234  * @param[in] external
5235  *   This flow rule is created by request external to PMD.
5236  * @param[in] hairpin
5237  *   Number of hairpin TX actions, 0 means classic flow.
5238  * @param[out] error
5239  *   Pointer to the error structure.
5240  *
5241  * @return
5242  *   0 on success, a negative errno value otherwise and rte_errno is set.
5243  */
5244 static int
5245 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
5246                  const struct rte_flow_item items[],
5247                  const struct rte_flow_action actions[],
5248                  bool external, int hairpin, struct rte_flow_error *error)
5249 {
5250         int ret;
5251         uint64_t action_flags = 0;
5252         uint64_t item_flags = 0;
5253         uint64_t last_item = 0;
5254         uint8_t next_protocol = 0xff;
5255         uint16_t ether_type = 0;
5256         int actions_n = 0;
5257         uint8_t item_ipv6_proto = 0;
5258         const struct rte_flow_item *gre_item = NULL;
5259         const struct rte_flow_action_raw_decap *decap;
5260         const struct rte_flow_action_raw_encap *encap;
5261         const struct rte_flow_action_rss *rss;
5262         const struct rte_flow_item_tcp nic_tcp_mask = {
5263                 .hdr = {
5264                         .tcp_flags = 0xFF,
5265                         .src_port = RTE_BE16(UINT16_MAX),
5266                         .dst_port = RTE_BE16(UINT16_MAX),
5267                 }
5268         };
5269         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
5270                 .hdr = {
5271                         .src_addr =
5272                         "\xff\xff\xff\xff\xff\xff\xff\xff"
5273                         "\xff\xff\xff\xff\xff\xff\xff\xff",
5274                         .dst_addr =
5275                         "\xff\xff\xff\xff\xff\xff\xff\xff"
5276                         "\xff\xff\xff\xff\xff\xff\xff\xff",
5277                         .vtc_flow = RTE_BE32(0xffffffff),
5278                         .proto = 0xff,
5279                         .hop_limits = 0xff,
5280                 },
5281                 .has_frag_ext = 1,
5282         };
5283         const struct rte_flow_item_ecpri nic_ecpri_mask = {
5284                 .hdr = {
5285                         .common = {
5286                                 .u32 =
5287                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
5288                                         .type = 0xFF,
5289                                         }).u32),
5290                         },
5291                         .dummy[0] = 0xffffffff,
5292                 },
5293         };
5294         struct mlx5_priv *priv = dev->data->dev_private;
5295         struct mlx5_dev_config *dev_conf = &priv->config;
5296         uint16_t queue_index = 0xFFFF;
5297         const struct rte_flow_item_vlan *vlan_m = NULL;
5298         int16_t rw_act_num = 0;
5299         uint64_t is_root;
5300
5301         if (items == NULL)
5302                 return -1;
5303         ret = flow_dv_validate_attributes(dev, attr, external, error);
5304         if (ret < 0)
5305                 return ret;
5306         is_root = (uint64_t)ret;
5307         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5308                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
5309                 int type = items->type;
5310
5311                 if (!mlx5_flow_os_item_supported(type))
5312                         return rte_flow_error_set(error, ENOTSUP,
5313                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5314                                                   NULL, "item not supported");
5315                 switch (type) {
5316                 case RTE_FLOW_ITEM_TYPE_VOID:
5317                         break;
5318                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
5319                         ret = flow_dv_validate_item_port_id
5320                                         (dev, items, attr, item_flags, error);
5321                         if (ret < 0)
5322                                 return ret;
5323                         last_item = MLX5_FLOW_ITEM_PORT_ID;
5324                         break;
5325                 case RTE_FLOW_ITEM_TYPE_ETH:
5326                         ret = mlx5_flow_validate_item_eth(items, item_flags,
5327                                                           error);
5328                         if (ret < 0)
5329                                 return ret;
5330                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
5331                                              MLX5_FLOW_LAYER_OUTER_L2;
5332                         if (items->mask != NULL && items->spec != NULL) {
5333                                 ether_type =
5334                                         ((const struct rte_flow_item_eth *)
5335                                          items->spec)->type;
5336                                 ether_type &=
5337                                         ((const struct rte_flow_item_eth *)
5338                                          items->mask)->type;
5339                                 ether_type = rte_be_to_cpu_16(ether_type);
5340                         } else {
5341                                 ether_type = 0;
5342                         }
5343                         break;
5344                 case RTE_FLOW_ITEM_TYPE_VLAN:
5345                         ret = flow_dv_validate_item_vlan(items, item_flags,
5346                                                          dev, error);
5347                         if (ret < 0)
5348                                 return ret;
5349                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
5350                                              MLX5_FLOW_LAYER_OUTER_VLAN;
5351                         if (items->mask != NULL && items->spec != NULL) {
5352                                 ether_type =
5353                                         ((const struct rte_flow_item_vlan *)
5354                                          items->spec)->inner_type;
5355                                 ether_type &=
5356                                         ((const struct rte_flow_item_vlan *)
5357                                          items->mask)->inner_type;
5358                                 ether_type = rte_be_to_cpu_16(ether_type);
5359                         } else {
5360                                 ether_type = 0;
5361                         }
5362                         /* Store outer VLAN mask for of_push_vlan action. */
5363                         if (!tunnel)
5364                                 vlan_m = items->mask;
5365                         break;
5366                 case RTE_FLOW_ITEM_TYPE_IPV4:
5367                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5368                                                   &item_flags, &tunnel);
5369                         ret = flow_dv_validate_item_ipv4(items, item_flags,
5370                                                          last_item, ether_type,
5371                                                          error);
5372                         if (ret < 0)
5373                                 return ret;
5374                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5375                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5376                         if (items->mask != NULL &&
5377                             ((const struct rte_flow_item_ipv4 *)
5378                              items->mask)->hdr.next_proto_id) {
5379                                 next_protocol =
5380                                         ((const struct rte_flow_item_ipv4 *)
5381                                          (items->spec))->hdr.next_proto_id;
5382                                 next_protocol &=
5383                                         ((const struct rte_flow_item_ipv4 *)
5384                                          (items->mask))->hdr.next_proto_id;
5385                         } else {
5386                                 /* Reset for inner layer. */
5387                                 next_protocol = 0xff;
5388                         }
5389                         break;
5390                 case RTE_FLOW_ITEM_TYPE_IPV6:
5391                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5392                                                   &item_flags, &tunnel);
5393                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
5394                                                            last_item,
5395                                                            ether_type,
5396                                                            &nic_ipv6_mask,
5397                                                            error);
5398                         if (ret < 0)
5399                                 return ret;
5400                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5401                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5402                         if (items->mask != NULL &&
5403                             ((const struct rte_flow_item_ipv6 *)
5404                              items->mask)->hdr.proto) {
5405                                 item_ipv6_proto =
5406                                         ((const struct rte_flow_item_ipv6 *)
5407                                          items->spec)->hdr.proto;
5408                                 next_protocol =
5409                                         ((const struct rte_flow_item_ipv6 *)
5410                                          items->spec)->hdr.proto;
5411                                 next_protocol &=
5412                                         ((const struct rte_flow_item_ipv6 *)
5413                                          items->mask)->hdr.proto;
5414                         } else {
5415                                 /* Reset for inner layer. */
5416                                 next_protocol = 0xff;
5417                         }
5418                         break;
5419                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
5420                         ret = flow_dv_validate_item_ipv6_frag_ext(items,
5421                                                                   item_flags,
5422                                                                   error);
5423                         if (ret < 0)
5424                                 return ret;
5425                         last_item = tunnel ?
5426                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
5427                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
5428                         if (items->mask != NULL &&
5429                             ((const struct rte_flow_item_ipv6_frag_ext *)
5430                              items->mask)->hdr.next_header) {
5431                                 next_protocol =
5432                                 ((const struct rte_flow_item_ipv6_frag_ext *)
5433                                  items->spec)->hdr.next_header;
5434                                 next_protocol &=
5435                                 ((const struct rte_flow_item_ipv6_frag_ext *)
5436                                  items->mask)->hdr.next_header;
5437                         } else {
5438                                 /* Reset for inner layer. */
5439                                 next_protocol = 0xff;
5440                         }
5441                         break;
5442                 case RTE_FLOW_ITEM_TYPE_TCP:
5443                         ret = mlx5_flow_validate_item_tcp
5444                                                 (items, item_flags,
5445                                                  next_protocol,
5446                                                  &nic_tcp_mask,
5447                                                  error);
5448                         if (ret < 0)
5449                                 return ret;
5450                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5451                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5452                         break;
5453                 case RTE_FLOW_ITEM_TYPE_UDP:
5454                         ret = mlx5_flow_validate_item_udp(items, item_flags,
5455                                                           next_protocol,
5456                                                           error);
5457                         if (ret < 0)
5458                                 return ret;
5459                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5460                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5461                         break;
5462                 case RTE_FLOW_ITEM_TYPE_GRE:
5463                         ret = mlx5_flow_validate_item_gre(items, item_flags,
5464                                                           next_protocol, error);
5465                         if (ret < 0)
5466                                 return ret;
5467                         gre_item = items;
5468                         last_item = MLX5_FLOW_LAYER_GRE;
5469                         break;
5470                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5471                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
5472                                                             next_protocol,
5473                                                             error);
5474                         if (ret < 0)
5475                                 return ret;
5476                         last_item = MLX5_FLOW_LAYER_NVGRE;
5477                         break;
5478                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5479                         ret = mlx5_flow_validate_item_gre_key
5480                                 (items, item_flags, gre_item, error);
5481                         if (ret < 0)
5482                                 return ret;
5483                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5484                         break;
5485                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5486                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
5487                                                             error);
5488                         if (ret < 0)
5489                                 return ret;
5490                         last_item = MLX5_FLOW_LAYER_VXLAN;
5491                         break;
5492                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5493                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
5494                                                                 item_flags, dev,
5495                                                                 error);
5496                         if (ret < 0)
5497                                 return ret;
5498                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5499                         break;
5500                 case RTE_FLOW_ITEM_TYPE_GENEVE:
5501                         ret = mlx5_flow_validate_item_geneve(items,
5502                                                              item_flags, dev,
5503                                                              error);
5504                         if (ret < 0)
5505                                 return ret;
5506                         last_item = MLX5_FLOW_LAYER_GENEVE;
5507                         break;
5508                 case RTE_FLOW_ITEM_TYPE_MPLS:
5509                         ret = mlx5_flow_validate_item_mpls(dev, items,
5510                                                            item_flags,
5511                                                            last_item, error);
5512                         if (ret < 0)
5513                                 return ret;
5514                         last_item = MLX5_FLOW_LAYER_MPLS;
5515                         break;
5516
5517                 case RTE_FLOW_ITEM_TYPE_MARK:
5518                         ret = flow_dv_validate_item_mark(dev, items, attr,
5519                                                          error);
5520                         if (ret < 0)
5521                                 return ret;
5522                         last_item = MLX5_FLOW_ITEM_MARK;
5523                         break;
5524                 case RTE_FLOW_ITEM_TYPE_META:
5525                         ret = flow_dv_validate_item_meta(dev, items, attr,
5526                                                          error);
5527                         if (ret < 0)
5528                                 return ret;
5529                         last_item = MLX5_FLOW_ITEM_METADATA;
5530                         break;
5531                 case RTE_FLOW_ITEM_TYPE_ICMP:
5532                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
5533                                                            next_protocol,
5534                                                            error);
5535                         if (ret < 0)
5536                                 return ret;
5537                         last_item = MLX5_FLOW_LAYER_ICMP;
5538                         break;
5539                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5540                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
5541                                                             next_protocol,
5542                                                             error);
5543                         if (ret < 0)
5544                                 return ret;
5545                         item_ipv6_proto = IPPROTO_ICMPV6;
5546                         last_item = MLX5_FLOW_LAYER_ICMP6;
5547                         break;
5548                 case RTE_FLOW_ITEM_TYPE_TAG:
5549                         ret = flow_dv_validate_item_tag(dev, items,
5550                                                         attr, error);
5551                         if (ret < 0)
5552                                 return ret;
5553                         last_item = MLX5_FLOW_ITEM_TAG;
5554                         break;
5555                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
5556                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
5557                         break;
5558                 case RTE_FLOW_ITEM_TYPE_GTP:
5559                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
5560                                                         error);
5561                         if (ret < 0)
5562                                 return ret;
5563                         last_item = MLX5_FLOW_LAYER_GTP;
5564                         break;
5565                 case RTE_FLOW_ITEM_TYPE_ECPRI:
5566                         /* Capacity will be checked in the translate stage. */
5567                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
5568                                                             last_item,
5569                                                             ether_type,
5570                                                             &nic_ecpri_mask,
5571                                                             error);
5572                         if (ret < 0)
5573                                 return ret;
5574                         last_item = MLX5_FLOW_LAYER_ECPRI;
5575                         break;
5576                 default:
5577                         return rte_flow_error_set(error, ENOTSUP,
5578                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5579                                                   NULL, "item not supported");
5580                 }
5581                 item_flags |= last_item;
5582         }
5583         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5584                 int type = actions->type;
5585
5586                 if (!mlx5_flow_os_action_supported(type))
5587                         return rte_flow_error_set(error, ENOTSUP,
5588                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5589                                                   actions,
5590                                                   "action not supported");
5591                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5592                         return rte_flow_error_set(error, ENOTSUP,
5593                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5594                                                   actions, "too many actions");
5595                 switch (type) {
5596                 case RTE_FLOW_ACTION_TYPE_VOID:
5597                         break;
5598                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5599                         ret = flow_dv_validate_action_port_id(dev,
5600                                                               action_flags,
5601                                                               actions,
5602                                                               attr,
5603                                                               error);
5604                         if (ret)
5605                                 return ret;
5606                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5607                         ++actions_n;
5608                         break;
5609                 case RTE_FLOW_ACTION_TYPE_FLAG:
5610                         ret = flow_dv_validate_action_flag(dev, action_flags,
5611                                                            attr, error);
5612                         if (ret < 0)
5613                                 return ret;
5614                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5615                                 /* Count all modify-header actions as one. */
5616                                 if (!(action_flags &
5617                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5618                                         ++actions_n;
5619                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
5620                                                 MLX5_FLOW_ACTION_MARK_EXT;
5621                         } else {
5622                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
5623                                 ++actions_n;
5624                         }
5625                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5626                         break;
5627                 case RTE_FLOW_ACTION_TYPE_MARK:
5628                         ret = flow_dv_validate_action_mark(dev, actions,
5629                                                            action_flags,
5630                                                            attr, error);
5631                         if (ret < 0)
5632                                 return ret;
5633                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5634                                 /* Count all modify-header actions as one. */
5635                                 if (!(action_flags &
5636                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5637                                         ++actions_n;
5638                                 action_flags |= MLX5_FLOW_ACTION_MARK |
5639                                                 MLX5_FLOW_ACTION_MARK_EXT;
5640                         } else {
5641                                 action_flags |= MLX5_FLOW_ACTION_MARK;
5642                                 ++actions_n;
5643                         }
5644                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5645                         break;
5646                 case RTE_FLOW_ACTION_TYPE_SET_META:
5647                         ret = flow_dv_validate_action_set_meta(dev, actions,
5648                                                                action_flags,
5649                                                                attr, error);
5650                         if (ret < 0)
5651                                 return ret;
5652                         /* Count all modify-header actions as one action. */
5653                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5654                                 ++actions_n;
5655                         action_flags |= MLX5_FLOW_ACTION_SET_META;
5656                         rw_act_num += MLX5_ACT_NUM_SET_META;
5657                         break;
5658                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5659                         ret = flow_dv_validate_action_set_tag(dev, actions,
5660                                                               action_flags,
5661                                                               attr, error);
5662                         if (ret < 0)
5663                                 return ret;
5664                         /* Count all modify-header actions as one action. */
5665                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5666                                 ++actions_n;
5667                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
5668                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5669                         break;
5670                 case RTE_FLOW_ACTION_TYPE_DROP:
5671                         ret = mlx5_flow_validate_action_drop(action_flags,
5672                                                              attr, error);
5673                         if (ret < 0)
5674                                 return ret;
5675                         action_flags |= MLX5_FLOW_ACTION_DROP;
5676                         ++actions_n;
5677                         break;
5678                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5679                         ret = mlx5_flow_validate_action_queue(actions,
5680                                                               action_flags, dev,
5681                                                               attr, error);
5682                         if (ret < 0)
5683                                 return ret;
5684                         queue_index = ((const struct rte_flow_action_queue *)
5685                                                         (actions->conf))->index;
5686                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
5687                         ++actions_n;
5688                         break;
5689                 case RTE_FLOW_ACTION_TYPE_RSS:
5690                         rss = actions->conf;
5691                         ret = mlx5_flow_validate_action_rss(actions,
5692                                                             action_flags, dev,
5693                                                             attr, item_flags,
5694                                                             error);
5695                         if (ret < 0)
5696                                 return ret;
5697                         if (rss != NULL && rss->queue_num)
5698                                 queue_index = rss->queue[0];
5699                         action_flags |= MLX5_FLOW_ACTION_RSS;
5700                         ++actions_n;
5701                         break;
5702                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
5703                         ret =
5704                         mlx5_flow_validate_action_default_miss(action_flags,
5705                                         attr, error);
5706                         if (ret < 0)
5707                                 return ret;
5708                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
5709                         ++actions_n;
5710                         break;
5711                 case RTE_FLOW_ACTION_TYPE_COUNT:
5712                         ret = flow_dv_validate_action_count(dev, error);
5713                         if (ret < 0)
5714                                 return ret;
5715                         action_flags |= MLX5_FLOW_ACTION_COUNT;
5716                         ++actions_n;
5717                         break;
5718                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5719                         if (flow_dv_validate_action_pop_vlan(dev,
5720                                                              action_flags,
5721                                                              actions,
5722                                                              item_flags, attr,
5723                                                              error))
5724                                 return -rte_errno;
5725                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5726                         ++actions_n;
5727                         break;
5728                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5729                         ret = flow_dv_validate_action_push_vlan(dev,
5730                                                                 action_flags,
5731                                                                 vlan_m,
5732                                                                 actions, attr,
5733                                                                 error);
5734                         if (ret < 0)
5735                                 return ret;
5736                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5737                         ++actions_n;
5738                         break;
5739                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5740                         ret = flow_dv_validate_action_set_vlan_pcp
5741                                                 (action_flags, actions, error);
5742                         if (ret < 0)
5743                                 return ret;
5744                         /* Count PCP with push_vlan command. */
5745                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
5746                         break;
5747                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5748                         ret = flow_dv_validate_action_set_vlan_vid
5749                                                 (item_flags, action_flags,
5750                                                  actions, error);
5751                         if (ret < 0)
5752                                 return ret;
5753                         /* Count VID with push_vlan command. */
5754                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
5755                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
5756                         break;
5757                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5758                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5759                         ret = flow_dv_validate_action_l2_encap(dev,
5760                                                                action_flags,
5761                                                                actions, attr,
5762                                                                error);
5763                         if (ret < 0)
5764                                 return ret;
5765                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
5766                         ++actions_n;
5767                         break;
5768                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5769                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5770                         ret = flow_dv_validate_action_decap(dev, action_flags,
5771                                                             attr, error);
5772                         if (ret < 0)
5773                                 return ret;
5774                         action_flags |= MLX5_FLOW_ACTION_DECAP;
5775                         ++actions_n;
5776                         break;
5777                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5778                         ret = flow_dv_validate_action_raw_encap_decap
5779                                 (dev, NULL, actions->conf, attr, &action_flags,
5780                                  &actions_n, error);
5781                         if (ret < 0)
5782                                 return ret;
5783                         break;
5784                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5785                         decap = actions->conf;
5786                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5787                                 ;
5788                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5789                                 encap = NULL;
5790                                 actions--;
5791                         } else {
5792                                 encap = actions->conf;
5793                         }
5794                         ret = flow_dv_validate_action_raw_encap_decap
5795                                            (dev,
5796                                             decap ? decap : &empty_decap, encap,
5797                                             attr, &action_flags, &actions_n,
5798                                             error);
5799                         if (ret < 0)
5800                                 return ret;
5801                         break;
5802                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5803                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5804                         ret = flow_dv_validate_action_modify_mac(action_flags,
5805                                                                  actions,
5806                                                                  item_flags,
5807                                                                  error);
5808                         if (ret < 0)
5809                                 return ret;
5810                         /* Count all modify-header actions as one action. */
5811                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5812                                 ++actions_n;
5813                         action_flags |= actions->type ==
5814                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5815                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
5816                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
5817                         /*
5818                          * Even if the source and destination MAC addresses have
5819                          * overlap in the header with 4B alignment, the convert
5820                          * function will handle them separately and 4 SW actions
5821                          * will be created. And 2 actions will be added each
5822                          * time no matter how many bytes of address will be set.
5823                          */
5824                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
5825                         break;
5826                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5827                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5828                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
5829                                                                   actions,
5830                                                                   item_flags,
5831                                                                   error);
5832                         if (ret < 0)
5833                                 return ret;
5834                         /* Count all modify-header actions as one action. */
5835                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5836                                 ++actions_n;
5837                         action_flags |= actions->type ==
5838                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5839                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
5840                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
5841                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
5842                         break;
5843                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5844                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5845                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
5846                                                                   actions,
5847                                                                   item_flags,
5848                                                                   error);
5849                         if (ret < 0)
5850                                 return ret;
5851                         if (item_ipv6_proto == IPPROTO_ICMPV6)
5852                                 return rte_flow_error_set(error, ENOTSUP,
5853                                         RTE_FLOW_ERROR_TYPE_ACTION,
5854                                         actions,
5855                                         "Can't change header "
5856                                         "with ICMPv6 proto");
5857                         /* Count all modify-header actions as one action. */
5858                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5859                                 ++actions_n;
5860                         action_flags |= actions->type ==
5861                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5862                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
5863                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
5864                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
5865                         break;
5866                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5867                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5868                         ret = flow_dv_validate_action_modify_tp(action_flags,
5869                                                                 actions,
5870                                                                 item_flags,
5871                                                                 error);
5872                         if (ret < 0)
5873                                 return ret;
5874                         /* Count all modify-header actions as one action. */
5875                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5876                                 ++actions_n;
5877                         action_flags |= actions->type ==
5878                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5879                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
5880                                                 MLX5_FLOW_ACTION_SET_TP_DST;
5881                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
5882                         break;
5883                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5884                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5885                         ret = flow_dv_validate_action_modify_ttl(action_flags,
5886                                                                  actions,
5887                                                                  item_flags,
5888                                                                  error);
5889                         if (ret < 0)
5890                                 return ret;
5891                         /* Count all modify-header actions as one action. */
5892                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5893                                 ++actions_n;
5894                         action_flags |= actions->type ==
5895                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
5896                                                 MLX5_FLOW_ACTION_SET_TTL :
5897                                                 MLX5_FLOW_ACTION_DEC_TTL;
5898                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
5899                         break;
5900                 case RTE_FLOW_ACTION_TYPE_JUMP:
5901                         ret = flow_dv_validate_action_jump(actions,
5902                                                            action_flags,
5903                                                            attr, external,
5904                                                            error);
5905                         if (ret)
5906                                 return ret;
5907                         ++actions_n;
5908                         action_flags |= MLX5_FLOW_ACTION_JUMP;
5909                         break;
5910                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5911                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5912                         ret = flow_dv_validate_action_modify_tcp_seq
5913                                                                 (action_flags,
5914                                                                  actions,
5915                                                                  item_flags,
5916                                                                  error);
5917                         if (ret < 0)
5918                                 return ret;
5919                         /* Count all modify-header actions as one action. */
5920                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5921                                 ++actions_n;
5922                         action_flags |= actions->type ==
5923                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5924                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
5925                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5926                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
5927                         break;
5928                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5929                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5930                         ret = flow_dv_validate_action_modify_tcp_ack
5931                                                                 (action_flags,
5932                                                                  actions,
5933                                                                  item_flags,
5934                                                                  error);
5935                         if (ret < 0)
5936                                 return ret;
5937                         /* Count all modify-header actions as one action. */
5938                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5939                                 ++actions_n;
5940                         action_flags |= actions->type ==
5941                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
5942                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
5943                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
5944                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
5945                         break;
5946                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
5947                         break;
5948                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
5949                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
5950                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5951                         break;
5952                 case RTE_FLOW_ACTION_TYPE_METER:
5953                         ret = mlx5_flow_validate_action_meter(dev,
5954                                                               action_flags,
5955                                                               actions, attr,
5956                                                               error);
5957                         if (ret < 0)
5958                                 return ret;
5959                         action_flags |= MLX5_FLOW_ACTION_METER;
5960                         ++actions_n;
5961                         /* Meter action will add one more TAG action. */
5962                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5963                         break;
5964                 case RTE_FLOW_ACTION_TYPE_AGE:
5965                         ret = flow_dv_validate_action_age(action_flags,
5966                                                           actions, dev,
5967                                                           error);
5968                         if (ret < 0)
5969                                 return ret;
5970                         action_flags |= MLX5_FLOW_ACTION_AGE;
5971                         ++actions_n;
5972                         break;
5973                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5974                         ret = flow_dv_validate_action_modify_ipv4_dscp
5975                                                          (action_flags,
5976                                                           actions,
5977                                                           item_flags,
5978                                                           error);
5979                         if (ret < 0)
5980                                 return ret;
5981                         /* Count all modify-header actions as one action. */
5982                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5983                                 ++actions_n;
5984                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
5985                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5986                         break;
5987                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5988                         ret = flow_dv_validate_action_modify_ipv6_dscp
5989                                                                 (action_flags,
5990                                                                  actions,
5991                                                                  item_flags,
5992                                                                  error);
5993                         if (ret < 0)
5994                                 return ret;
5995                         /* Count all modify-header actions as one action. */
5996                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5997                                 ++actions_n;
5998                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
5999                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
6000                         break;
6001                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
6002                         ret = flow_dv_validate_action_sample(action_flags,
6003                                                              actions, dev,
6004                                                              attr, error);
6005                         if (ret < 0)
6006                                 return ret;
6007                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
6008                         ++actions_n;
6009                         break;
6010                 default:
6011                         return rte_flow_error_set(error, ENOTSUP,
6012                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6013                                                   actions,
6014                                                   "action not supported");
6015                 }
6016         }
6017         /*
6018          * Validate the drop action mutual exclusion with other actions.
6019          * Drop action is mutually-exclusive with any other action, except for
6020          * Count action.
6021          */
6022         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
6023             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
6024                 return rte_flow_error_set(error, EINVAL,
6025                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
6026                                           "Drop action is mutually-exclusive "
6027                                           "with any other action, except for "
6028                                           "Count action");
6029         /* Eswitch has few restrictions on using items and actions */
6030         if (attr->transfer) {
6031                 if (!mlx5_flow_ext_mreg_supported(dev) &&
6032                     action_flags & MLX5_FLOW_ACTION_FLAG)
6033                         return rte_flow_error_set(error, ENOTSUP,
6034                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6035                                                   NULL,
6036                                                   "unsupported action FLAG");
6037                 if (!mlx5_flow_ext_mreg_supported(dev) &&
6038                     action_flags & MLX5_FLOW_ACTION_MARK)
6039                         return rte_flow_error_set(error, ENOTSUP,
6040                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6041                                                   NULL,
6042                                                   "unsupported action MARK");
6043                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
6044                         return rte_flow_error_set(error, ENOTSUP,
6045                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6046                                                   NULL,
6047                                                   "unsupported action QUEUE");
6048                 if (action_flags & MLX5_FLOW_ACTION_RSS)
6049                         return rte_flow_error_set(error, ENOTSUP,
6050                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6051                                                   NULL,
6052                                                   "unsupported action RSS");
6053                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
6054                         return rte_flow_error_set(error, EINVAL,
6055                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6056                                                   actions,
6057                                                   "no fate action is found");
6058         } else {
6059                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
6060                         return rte_flow_error_set(error, EINVAL,
6061                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6062                                                   actions,
6063                                                   "no fate action is found");
6064         }
6065         /* Continue validation for Xcap and VLAN actions.*/
6066         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
6067                              MLX5_FLOW_VLAN_ACTIONS)) &&
6068             (queue_index == 0xFFFF ||
6069              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
6070                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
6071                     MLX5_FLOW_XCAP_ACTIONS)
6072                         return rte_flow_error_set(error, ENOTSUP,
6073                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6074                                                   NULL, "encap and decap "
6075                                                   "combination aren't supported");
6076                 if (!attr->transfer && attr->ingress) {
6077                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
6078                                 return rte_flow_error_set
6079                                                 (error, ENOTSUP,
6080                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6081                                                  NULL, "encap is not supported"
6082                                                  " for ingress traffic");
6083                         else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
6084                                 return rte_flow_error_set
6085                                                 (error, ENOTSUP,
6086                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6087                                                  NULL, "push VLAN action not "
6088                                                  "supported for ingress");
6089                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
6090                                         MLX5_FLOW_VLAN_ACTIONS)
6091                                 return rte_flow_error_set
6092                                                 (error, ENOTSUP,
6093                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6094                                                  NULL, "no support for "
6095                                                  "multiple VLAN actions");
6096                 }
6097         }
6098         /* Hairpin flow will add one more TAG action. */
6099         if (hairpin > 0)
6100                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6101         /* extra metadata enabled: one more TAG action will be add. */
6102         if (dev_conf->dv_flow_en &&
6103             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
6104             mlx5_flow_ext_mreg_supported(dev))
6105                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6106         if ((uint32_t)rw_act_num >
6107                         flow_dv_modify_hdr_action_max(dev, is_root)) {
6108                 return rte_flow_error_set(error, ENOTSUP,
6109                                           RTE_FLOW_ERROR_TYPE_ACTION,
6110                                           NULL, "too many header modify"
6111                                           " actions to support");
6112         }
6113         return 0;
6114 }
6115
6116 /**
6117  * Internal preparation function. Allocates the DV flow size,
6118  * this size is constant.
6119  *
6120  * @param[in] dev
6121  *   Pointer to the rte_eth_dev structure.
6122  * @param[in] attr
6123  *   Pointer to the flow attributes.
6124  * @param[in] items
6125  *   Pointer to the list of items.
6126  * @param[in] actions
6127  *   Pointer to the list of actions.
6128  * @param[out] error
6129  *   Pointer to the error structure.
6130  *
6131  * @return
6132  *   Pointer to mlx5_flow object on success,
6133  *   otherwise NULL and rte_errno is set.
6134  */
6135 static struct mlx5_flow *
6136 flow_dv_prepare(struct rte_eth_dev *dev,
6137                 const struct rte_flow_attr *attr __rte_unused,
6138                 const struct rte_flow_item items[] __rte_unused,
6139                 const struct rte_flow_action actions[] __rte_unused,
6140                 struct rte_flow_error *error)
6141 {
6142         uint32_t handle_idx = 0;
6143         struct mlx5_flow *dev_flow;
6144         struct mlx5_flow_handle *dev_handle;
6145         struct mlx5_priv *priv = dev->data->dev_private;
6146
6147         /* In case of corrupting the memory. */
6148         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
6149                 rte_flow_error_set(error, ENOSPC,
6150                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6151                                    "not free temporary device flow");
6152                 return NULL;
6153         }
6154         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
6155                                    &handle_idx);
6156         if (!dev_handle) {
6157                 rte_flow_error_set(error, ENOMEM,
6158                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6159                                    "not enough memory to create flow handle");
6160                 return NULL;
6161         }
6162         /* No multi-thread supporting. */
6163         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
6164         dev_flow->handle = dev_handle;
6165         dev_flow->handle_idx = handle_idx;
6166         /*
6167          * In some old rdma-core releases, before continuing, a check of the
6168          * length of matching parameter will be done at first. It needs to use
6169          * the length without misc4 param. If the flow has misc4 support, then
6170          * the length needs to be adjusted accordingly. Each param member is
6171          * aligned with a 64B boundary naturally.
6172          */
6173         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) -
6174                                   MLX5_ST_SZ_BYTES(fte_match_set_misc4);
6175         /*
6176          * The matching value needs to be cleared to 0 before using. In the
6177          * past, it will be automatically cleared when using rte_*alloc
6178          * API. The time consumption will be almost the same as before.
6179          */
6180         memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
6181         dev_flow->ingress = attr->ingress;
6182         dev_flow->dv.transfer = attr->transfer;
6183         return dev_flow;
6184 }
6185
6186 #ifdef RTE_LIBRTE_MLX5_DEBUG
6187 /**
6188  * Sanity check for match mask and value. Similar to check_valid_spec() in
6189  * kernel driver. If unmasked bit is present in value, it returns failure.
6190  *
6191  * @param match_mask
6192  *   pointer to match mask buffer.
6193  * @param match_value
6194  *   pointer to match value buffer.
6195  *
6196  * @return
6197  *   0 if valid, -EINVAL otherwise.
6198  */
6199 static int
6200 flow_dv_check_valid_spec(void *match_mask, void *match_value)
6201 {
6202         uint8_t *m = match_mask;
6203         uint8_t *v = match_value;
6204         unsigned int i;
6205
6206         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
6207                 if (v[i] & ~m[i]) {
6208                         DRV_LOG(ERR,
6209                                 "match_value differs from match_criteria"
6210                                 " %p[%u] != %p[%u]",
6211                                 match_value, i, match_mask, i);
6212                         return -EINVAL;
6213                 }
6214         }
6215         return 0;
6216 }
6217 #endif
6218
6219 /**
6220  * Add match of ip_version.
6221  *
6222  * @param[in] group
6223  *   Flow group.
6224  * @param[in] headers_v
6225  *   Values header pointer.
6226  * @param[in] headers_m
6227  *   Masks header pointer.
6228  * @param[in] ip_version
6229  *   The IP version to set.
6230  */
6231 static inline void
6232 flow_dv_set_match_ip_version(uint32_t group,
6233                              void *headers_v,
6234                              void *headers_m,
6235                              uint8_t ip_version)
6236 {
6237         if (group == 0)
6238                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
6239         else
6240                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
6241                          ip_version);
6242         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
6243         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
6244         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
6245 }
6246
6247 /**
6248  * Add Ethernet item to matcher and to the value.
6249  *
6250  * @param[in, out] matcher
6251  *   Flow matcher.
6252  * @param[in, out] key
6253  *   Flow matcher value.
6254  * @param[in] item
6255  *   Flow pattern to translate.
6256  * @param[in] inner
6257  *   Item is inner pattern.
6258  */
6259 static void
6260 flow_dv_translate_item_eth(void *matcher, void *key,
6261                            const struct rte_flow_item *item, int inner,
6262                            uint32_t group)
6263 {
6264         const struct rte_flow_item_eth *eth_m = item->mask;
6265         const struct rte_flow_item_eth *eth_v = item->spec;
6266         const struct rte_flow_item_eth nic_mask = {
6267                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6268                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6269                 .type = RTE_BE16(0xffff),
6270         };
6271         void *headers_m;
6272         void *headers_v;
6273         char *l24_v;
6274         unsigned int i;
6275
6276         if (!eth_v)
6277                 return;
6278         if (!eth_m)
6279                 eth_m = &nic_mask;
6280         if (inner) {
6281                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6282                                          inner_headers);
6283                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6284         } else {
6285                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6286                                          outer_headers);
6287                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6288         }
6289         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
6290                &eth_m->dst, sizeof(eth_m->dst));
6291         /* The value must be in the range of the mask. */
6292         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
6293         for (i = 0; i < sizeof(eth_m->dst); ++i)
6294                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
6295         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
6296                &eth_m->src, sizeof(eth_m->src));
6297         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
6298         /* The value must be in the range of the mask. */
6299         for (i = 0; i < sizeof(eth_m->dst); ++i)
6300                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
6301         if (eth_v->type) {
6302                 /* When ethertype is present set mask for tagged VLAN. */
6303                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6304                 /* Set value for tagged VLAN if ethertype is 802.1Q. */
6305                 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
6306                     eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
6307                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag,
6308                                  1);
6309                         /* Return here to avoid setting match on ethertype. */
6310                         return;
6311                 }
6312         }
6313         /*
6314          * HW supports match on one Ethertype, the Ethertype following the last
6315          * VLAN tag of the packet (see PRM).
6316          * Set match on ethertype only if ETH header is not followed by VLAN.
6317          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6318          * ethertype, and use ip_version field instead.
6319          * eCPRI over Ether layer will use type value 0xAEFE.
6320          */
6321         if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6322             eth_m->type == 0xFFFF) {
6323                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6324         } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6325                    eth_m->type == 0xFFFF) {
6326                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6327         } else {
6328                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6329                          rte_be_to_cpu_16(eth_m->type));
6330                 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6331                                      ethertype);
6332                 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
6333         }
6334 }
6335
6336 /**
6337  * Add VLAN item to matcher and to the value.
6338  *
6339  * @param[in, out] dev_flow
6340  *   Flow descriptor.
6341  * @param[in, out] matcher
6342  *   Flow matcher.
6343  * @param[in, out] key
6344  *   Flow matcher value.
6345  * @param[in] item
6346  *   Flow pattern to translate.
6347  * @param[in] inner
6348  *   Item is inner pattern.
6349  */
6350 static void
6351 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
6352                             void *matcher, void *key,
6353                             const struct rte_flow_item *item,
6354                             int inner, uint32_t group)
6355 {
6356         const struct rte_flow_item_vlan *vlan_m = item->mask;
6357         const struct rte_flow_item_vlan *vlan_v = item->spec;
6358         void *headers_m;
6359         void *headers_v;
6360         uint16_t tci_m;
6361         uint16_t tci_v;
6362
6363         if (inner) {
6364                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6365                                          inner_headers);
6366                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6367         } else {
6368                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6369                                          outer_headers);
6370                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6371                 /*
6372                  * This is workaround, masks are not supported,
6373                  * and pre-validated.
6374                  */
6375                 if (vlan_v)
6376                         dev_flow->handle->vf_vlan.tag =
6377                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
6378         }
6379         /*
6380          * When VLAN item exists in flow, mark packet as tagged,
6381          * even if TCI is not specified.
6382          */
6383         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6384         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
6385         if (!vlan_v)
6386                 return;
6387         if (!vlan_m)
6388                 vlan_m = &rte_flow_item_vlan_mask;
6389         tci_m = rte_be_to_cpu_16(vlan_m->tci);
6390         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
6391         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
6392         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
6393         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
6394         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
6395         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
6396         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
6397         /*
6398          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6399          * ethertype, and use ip_version field instead.
6400          */
6401         if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6402             vlan_m->inner_type == 0xFFFF) {
6403                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6404         } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6405                    vlan_m->inner_type == 0xFFFF) {
6406                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6407         } else {
6408                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6409                          rte_be_to_cpu_16(vlan_m->inner_type));
6410                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
6411                          rte_be_to_cpu_16(vlan_m->inner_type &
6412                                           vlan_v->inner_type));
6413         }
6414 }
6415
6416 /**
6417  * Add IPV4 item to matcher and to the value.
6418  *
6419  * @param[in, out] matcher
6420  *   Flow matcher.
6421  * @param[in, out] key
6422  *   Flow matcher value.
6423  * @param[in] item
6424  *   Flow pattern to translate.
6425  * @param[in] item_flags
6426  *   Bit-fields that holds the items detected until now.
6427  * @param[in] inner
6428  *   Item is inner pattern.
6429  * @param[in] group
6430  *   The group to insert the rule.
6431  */
6432 static void
6433 flow_dv_translate_item_ipv4(void *matcher, void *key,
6434                             const struct rte_flow_item *item,
6435                             const uint64_t item_flags,
6436                             int inner, uint32_t group)
6437 {
6438         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
6439         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
6440         const struct rte_flow_item_ipv4 nic_mask = {
6441                 .hdr = {
6442                         .src_addr = RTE_BE32(0xffffffff),
6443                         .dst_addr = RTE_BE32(0xffffffff),
6444                         .type_of_service = 0xff,
6445                         .next_proto_id = 0xff,
6446                         .time_to_live = 0xff,
6447                 },
6448         };
6449         void *headers_m;
6450         void *headers_v;
6451         char *l24_m;
6452         char *l24_v;
6453         uint8_t tos;
6454
6455         if (inner) {
6456                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6457                                          inner_headers);
6458                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6459         } else {
6460                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6461                                          outer_headers);
6462                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6463         }
6464         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6465         /*
6466          * On outer header (which must contains L2), or inner header with L2,
6467          * set cvlan_tag mask bit to mark this packet as untagged.
6468          * This should be done even if item->spec is empty.
6469          */
6470         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6471                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6472         if (!ipv4_v)
6473                 return;
6474         if (!ipv4_m)
6475                 ipv4_m = &nic_mask;
6476         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6477                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6478         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6479                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6480         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
6481         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
6482         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6483                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6484         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6485                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6486         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
6487         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
6488         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
6489         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
6490                  ipv4_m->hdr.type_of_service);
6491         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
6492         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
6493                  ipv4_m->hdr.type_of_service >> 2);
6494         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
6495         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6496                  ipv4_m->hdr.next_proto_id);
6497         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6498                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
6499         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6500                  ipv4_m->hdr.time_to_live);
6501         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6502                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
6503         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
6504                  !!(ipv4_m->hdr.fragment_offset));
6505         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
6506                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
6507 }
6508
6509 /**
6510  * Add IPV6 item to matcher and to the value.
6511  *
6512  * @param[in, out] matcher
6513  *   Flow matcher.
6514  * @param[in, out] key
6515  *   Flow matcher value.
6516  * @param[in] item
6517  *   Flow pattern to translate.
6518  * @param[in] item_flags
6519  *   Bit-fields that holds the items detected until now.
6520  * @param[in] inner
6521  *   Item is inner pattern.
6522  * @param[in] group
6523  *   The group to insert the rule.
6524  */
6525 static void
6526 flow_dv_translate_item_ipv6(void *matcher, void *key,
6527                             const struct rte_flow_item *item,
6528                             const uint64_t item_flags,
6529                             int inner, uint32_t group)
6530 {
6531         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
6532         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
6533         const struct rte_flow_item_ipv6 nic_mask = {
6534                 .hdr = {
6535                         .src_addr =
6536                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6537                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6538                         .dst_addr =
6539                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6540                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6541                         .vtc_flow = RTE_BE32(0xffffffff),
6542                         .proto = 0xff,
6543                         .hop_limits = 0xff,
6544                 },
6545         };
6546         void *headers_m;
6547         void *headers_v;
6548         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6549         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6550         char *l24_m;
6551         char *l24_v;
6552         uint32_t vtc_m;
6553         uint32_t vtc_v;
6554         int i;
6555         int size;
6556
6557         if (inner) {
6558                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6559                                          inner_headers);
6560                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6561         } else {
6562                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6563                                          outer_headers);
6564                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6565         }
6566         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6567         /*
6568          * On outer header (which must contains L2), or inner header with L2,
6569          * set cvlan_tag mask bit to mark this packet as untagged.
6570          * This should be done even if item->spec is empty.
6571          */
6572         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6573                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6574         if (!ipv6_v)
6575                 return;
6576         if (!ipv6_m)
6577                 ipv6_m = &nic_mask;
6578         size = sizeof(ipv6_m->hdr.dst_addr);
6579         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6580                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6581         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6582                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6583         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
6584         for (i = 0; i < size; ++i)
6585                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
6586         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6587                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6588         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6589                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6590         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
6591         for (i = 0; i < size; ++i)
6592                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
6593         /* TOS. */
6594         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
6595         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
6596         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
6597         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
6598         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
6599         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
6600         /* Label. */
6601         if (inner) {
6602                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
6603                          vtc_m);
6604                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
6605                          vtc_v);
6606         } else {
6607                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
6608                          vtc_m);
6609                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
6610                          vtc_v);
6611         }
6612         /* Protocol. */
6613         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6614                  ipv6_m->hdr.proto);
6615         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6616                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
6617         /* Hop limit. */
6618         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6619                  ipv6_m->hdr.hop_limits);
6620         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6621                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
6622         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
6623                  !!(ipv6_m->has_frag_ext));
6624         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
6625                  !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext));
6626 }
6627
6628 /**
6629  * Add IPV6 fragment extension item to matcher and to the value.
6630  *
6631  * @param[in, out] matcher
6632  *   Flow matcher.
6633  * @param[in, out] key
6634  *   Flow matcher value.
6635  * @param[in] item
6636  *   Flow pattern to translate.
6637  * @param[in] inner
6638  *   Item is inner pattern.
6639  */
6640 static void
6641 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key,
6642                                      const struct rte_flow_item *item,
6643                                      int inner)
6644 {
6645         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask;
6646         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec;
6647         const struct rte_flow_item_ipv6_frag_ext nic_mask = {
6648                 .hdr = {
6649                         .next_header = 0xff,
6650                         .frag_data = RTE_BE16(0xffff),
6651                 },
6652         };
6653         void *headers_m;
6654         void *headers_v;
6655
6656         if (inner) {
6657                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6658                                          inner_headers);
6659                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6660         } else {
6661                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6662                                          outer_headers);
6663                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6664         }
6665         /* IPv6 fragment extension item exists, so packet is IP fragment. */
6666         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
6667         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1);
6668         if (!ipv6_frag_ext_v)
6669                 return;
6670         if (!ipv6_frag_ext_m)
6671                 ipv6_frag_ext_m = &nic_mask;
6672         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6673                  ipv6_frag_ext_m->hdr.next_header);
6674         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6675                  ipv6_frag_ext_v->hdr.next_header &
6676                  ipv6_frag_ext_m->hdr.next_header);
6677 }
6678
6679 /**
6680  * Add TCP item to matcher and to the value.
6681  *
6682  * @param[in, out] matcher
6683  *   Flow matcher.
6684  * @param[in, out] key
6685  *   Flow matcher value.
6686  * @param[in] item
6687  *   Flow pattern to translate.
6688  * @param[in] inner
6689  *   Item is inner pattern.
6690  */
6691 static void
6692 flow_dv_translate_item_tcp(void *matcher, void *key,
6693                            const struct rte_flow_item *item,
6694                            int inner)
6695 {
6696         const struct rte_flow_item_tcp *tcp_m = item->mask;
6697         const struct rte_flow_item_tcp *tcp_v = item->spec;
6698         void *headers_m;
6699         void *headers_v;
6700
6701         if (inner) {
6702                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6703                                          inner_headers);
6704                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6705         } else {
6706                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6707                                          outer_headers);
6708                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6709         }
6710         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6711         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
6712         if (!tcp_v)
6713                 return;
6714         if (!tcp_m)
6715                 tcp_m = &rte_flow_item_tcp_mask;
6716         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
6717                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
6718         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
6719                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
6720         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
6721                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
6722         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
6723                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
6724         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
6725                  tcp_m->hdr.tcp_flags);
6726         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
6727                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
6728 }
6729
6730 /**
6731  * Add UDP item to matcher and to the value.
6732  *
6733  * @param[in, out] matcher
6734  *   Flow matcher.
6735  * @param[in, out] key
6736  *   Flow matcher value.
6737  * @param[in] item
6738  *   Flow pattern to translate.
6739  * @param[in] inner
6740  *   Item is inner pattern.
6741  */
6742 static void
6743 flow_dv_translate_item_udp(void *matcher, void *key,
6744                            const struct rte_flow_item *item,
6745                            int inner)
6746 {
6747         const struct rte_flow_item_udp *udp_m = item->mask;
6748         const struct rte_flow_item_udp *udp_v = item->spec;
6749         void *headers_m;
6750         void *headers_v;
6751
6752         if (inner) {
6753                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6754                                          inner_headers);
6755                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6756         } else {
6757                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6758                                          outer_headers);
6759                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6760         }
6761         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6762         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
6763         if (!udp_v)
6764                 return;
6765         if (!udp_m)
6766                 udp_m = &rte_flow_item_udp_mask;
6767         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
6768                  rte_be_to_cpu_16(udp_m->hdr.src_port));
6769         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
6770                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
6771         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
6772                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
6773         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6774                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
6775 }
6776
6777 /**
6778  * Add GRE optional Key item to matcher and to the value.
6779  *
6780  * @param[in, out] matcher
6781  *   Flow matcher.
6782  * @param[in, out] key
6783  *   Flow matcher value.
6784  * @param[in] item
6785  *   Flow pattern to translate.
6786  * @param[in] inner
6787  *   Item is inner pattern.
6788  */
6789 static void
6790 flow_dv_translate_item_gre_key(void *matcher, void *key,
6791                                    const struct rte_flow_item *item)
6792 {
6793         const rte_be32_t *key_m = item->mask;
6794         const rte_be32_t *key_v = item->spec;
6795         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6796         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6797         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
6798
6799         /* GRE K bit must be on and should already be validated */
6800         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
6801         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
6802         if (!key_v)
6803                 return;
6804         if (!key_m)
6805                 key_m = &gre_key_default_mask;
6806         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
6807                  rte_be_to_cpu_32(*key_m) >> 8);
6808         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
6809                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
6810         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
6811                  rte_be_to_cpu_32(*key_m) & 0xFF);
6812         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
6813                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
6814 }
6815
6816 /**
6817  * Add GRE item to matcher and to the value.
6818  *
6819  * @param[in, out] matcher
6820  *   Flow matcher.
6821  * @param[in, out] key
6822  *   Flow matcher value.
6823  * @param[in] item
6824  *   Flow pattern to translate.
6825  * @param[in] inner
6826  *   Item is inner pattern.
6827  */
6828 static void
6829 flow_dv_translate_item_gre(void *matcher, void *key,
6830                            const struct rte_flow_item *item,
6831                            int inner)
6832 {
6833         const struct rte_flow_item_gre *gre_m = item->mask;
6834         const struct rte_flow_item_gre *gre_v = item->spec;
6835         void *headers_m;
6836         void *headers_v;
6837         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6838         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6839         struct {
6840                 union {
6841                         __extension__
6842                         struct {
6843                                 uint16_t version:3;
6844                                 uint16_t rsvd0:9;
6845                                 uint16_t s_present:1;
6846                                 uint16_t k_present:1;
6847                                 uint16_t rsvd_bit1:1;
6848                                 uint16_t c_present:1;
6849                         };
6850                         uint16_t value;
6851                 };
6852         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
6853
6854         if (inner) {
6855                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6856                                          inner_headers);
6857                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6858         } else {
6859                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6860                                          outer_headers);
6861                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6862         }
6863         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6864         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
6865         if (!gre_v)
6866                 return;
6867         if (!gre_m)
6868                 gre_m = &rte_flow_item_gre_mask;
6869         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
6870                  rte_be_to_cpu_16(gre_m->protocol));
6871         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6872                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
6873         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
6874         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
6875         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
6876                  gre_crks_rsvd0_ver_m.c_present);
6877         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
6878                  gre_crks_rsvd0_ver_v.c_present &
6879                  gre_crks_rsvd0_ver_m.c_present);
6880         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
6881                  gre_crks_rsvd0_ver_m.k_present);
6882         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
6883                  gre_crks_rsvd0_ver_v.k_present &
6884                  gre_crks_rsvd0_ver_m.k_present);
6885         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
6886                  gre_crks_rsvd0_ver_m.s_present);
6887         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
6888                  gre_crks_rsvd0_ver_v.s_present &
6889                  gre_crks_rsvd0_ver_m.s_present);
6890 }
6891
6892 /**
6893  * Add NVGRE item to matcher and to the value.
6894  *
6895  * @param[in, out] matcher
6896  *   Flow matcher.
6897  * @param[in, out] key
6898  *   Flow matcher value.
6899  * @param[in] item
6900  *   Flow pattern to translate.
6901  * @param[in] inner
6902  *   Item is inner pattern.
6903  */
6904 static void
6905 flow_dv_translate_item_nvgre(void *matcher, void *key,
6906                              const struct rte_flow_item *item,
6907                              int inner)
6908 {
6909         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
6910         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
6911         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6912         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6913         const char *tni_flow_id_m;
6914         const char *tni_flow_id_v;
6915         char *gre_key_m;
6916         char *gre_key_v;
6917         int size;
6918         int i;
6919
6920         /* For NVGRE, GRE header fields must be set with defined values. */
6921         const struct rte_flow_item_gre gre_spec = {
6922                 .c_rsvd0_ver = RTE_BE16(0x2000),
6923                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
6924         };
6925         const struct rte_flow_item_gre gre_mask = {
6926                 .c_rsvd0_ver = RTE_BE16(0xB000),
6927                 .protocol = RTE_BE16(UINT16_MAX),
6928         };
6929         const struct rte_flow_item gre_item = {
6930                 .spec = &gre_spec,
6931                 .mask = &gre_mask,
6932                 .last = NULL,
6933         };
6934         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
6935         if (!nvgre_v)
6936                 return;
6937         if (!nvgre_m)
6938                 nvgre_m = &rte_flow_item_nvgre_mask;
6939         tni_flow_id_m = (const char *)nvgre_m->tni;
6940         tni_flow_id_v = (const char *)nvgre_v->tni;
6941         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
6942         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
6943         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
6944         memcpy(gre_key_m, tni_flow_id_m, size);
6945         for (i = 0; i < size; ++i)
6946                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
6947 }
6948
6949 /**
6950  * Add VXLAN item to matcher and to the value.
6951  *
6952  * @param[in, out] matcher
6953  *   Flow matcher.
6954  * @param[in, out] key
6955  *   Flow matcher value.
6956  * @param[in] item
6957  *   Flow pattern to translate.
6958  * @param[in] inner
6959  *   Item is inner pattern.
6960  */
6961 static void
6962 flow_dv_translate_item_vxlan(void *matcher, void *key,
6963                              const struct rte_flow_item *item,
6964                              int inner)
6965 {
6966         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
6967         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
6968         void *headers_m;
6969         void *headers_v;
6970         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6971         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6972         char *vni_m;
6973         char *vni_v;
6974         uint16_t dport;
6975         int size;
6976         int i;
6977
6978         if (inner) {
6979                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6980                                          inner_headers);
6981                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6982         } else {
6983                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6984                                          outer_headers);
6985                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6986         }
6987         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6988                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6989         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6990                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6991                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6992         }
6993         if (!vxlan_v)
6994                 return;
6995         if (!vxlan_m)
6996                 vxlan_m = &rte_flow_item_vxlan_mask;
6997         size = sizeof(vxlan_m->vni);
6998         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
6999         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
7000         memcpy(vni_m, vxlan_m->vni, size);
7001         for (i = 0; i < size; ++i)
7002                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
7003 }
7004
7005 /**
7006  * Add VXLAN-GPE item to matcher and to the value.
7007  *
7008  * @param[in, out] matcher
7009  *   Flow matcher.
7010  * @param[in, out] key
7011  *   Flow matcher value.
7012  * @param[in] item
7013  *   Flow pattern to translate.
7014  * @param[in] inner
7015  *   Item is inner pattern.
7016  */
7017
7018 static void
7019 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
7020                                  const struct rte_flow_item *item, int inner)
7021 {
7022         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
7023         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
7024         void *headers_m;
7025         void *headers_v;
7026         void *misc_m =
7027                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
7028         void *misc_v =
7029                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7030         char *vni_m;
7031         char *vni_v;
7032         uint16_t dport;
7033         int size;
7034         int i;
7035         uint8_t flags_m = 0xff;
7036         uint8_t flags_v = 0xc;
7037
7038         if (inner) {
7039                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7040                                          inner_headers);
7041                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7042         } else {
7043                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7044                                          outer_headers);
7045                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7046         }
7047         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
7048                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
7049         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7050                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7051                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7052         }
7053         if (!vxlan_v)
7054                 return;
7055         if (!vxlan_m)
7056                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
7057         size = sizeof(vxlan_m->vni);
7058         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
7059         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
7060         memcpy(vni_m, vxlan_m->vni, size);
7061         for (i = 0; i < size; ++i)
7062                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
7063         if (vxlan_m->flags) {
7064                 flags_m = vxlan_m->flags;
7065                 flags_v = vxlan_v->flags;
7066         }
7067         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
7068         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
7069         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
7070                  vxlan_m->protocol);
7071         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
7072                  vxlan_v->protocol);
7073 }
7074
7075 /**
7076  * Add Geneve item to matcher and to the value.
7077  *
7078  * @param[in, out] matcher
7079  *   Flow matcher.
7080  * @param[in, out] key
7081  *   Flow matcher value.
7082  * @param[in] item
7083  *   Flow pattern to translate.
7084  * @param[in] inner
7085  *   Item is inner pattern.
7086  */
7087
7088 static void
7089 flow_dv_translate_item_geneve(void *matcher, void *key,
7090                               const struct rte_flow_item *item, int inner)
7091 {
7092         const struct rte_flow_item_geneve *geneve_m = item->mask;
7093         const struct rte_flow_item_geneve *geneve_v = item->spec;
7094         void *headers_m;
7095         void *headers_v;
7096         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7097         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7098         uint16_t dport;
7099         uint16_t gbhdr_m;
7100         uint16_t gbhdr_v;
7101         char *vni_m;
7102         char *vni_v;
7103         size_t size, i;
7104
7105         if (inner) {
7106                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7107                                          inner_headers);
7108                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7109         } else {
7110                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7111                                          outer_headers);
7112                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7113         }
7114         dport = MLX5_UDP_PORT_GENEVE;
7115         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7116                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7117                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7118         }
7119         if (!geneve_v)
7120                 return;
7121         if (!geneve_m)
7122                 geneve_m = &rte_flow_item_geneve_mask;
7123         size = sizeof(geneve_m->vni);
7124         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
7125         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
7126         memcpy(vni_m, geneve_m->vni, size);
7127         for (i = 0; i < size; ++i)
7128                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
7129         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
7130                  rte_be_to_cpu_16(geneve_m->protocol));
7131         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
7132                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
7133         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
7134         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
7135         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
7136                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7137         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
7138                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7139         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
7140                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7141         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
7142                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
7143                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7144 }
7145
7146 /**
7147  * Add MPLS item to matcher and to the value.
7148  *
7149  * @param[in, out] matcher
7150  *   Flow matcher.
7151  * @param[in, out] key
7152  *   Flow matcher value.
7153  * @param[in] item
7154  *   Flow pattern to translate.
7155  * @param[in] prev_layer
7156  *   The protocol layer indicated in previous item.
7157  * @param[in] inner
7158  *   Item is inner pattern.
7159  */
7160 static void
7161 flow_dv_translate_item_mpls(void *matcher, void *key,
7162                             const struct rte_flow_item *item,
7163                             uint64_t prev_layer,
7164                             int inner)
7165 {
7166         const uint32_t *in_mpls_m = item->mask;
7167         const uint32_t *in_mpls_v = item->spec;
7168         uint32_t *out_mpls_m = 0;
7169         uint32_t *out_mpls_v = 0;
7170         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7171         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7172         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
7173                                      misc_parameters_2);
7174         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7175         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
7176         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7177
7178         switch (prev_layer) {
7179         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7180                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
7181                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
7182                          MLX5_UDP_PORT_MPLS);
7183                 break;
7184         case MLX5_FLOW_LAYER_GRE:
7185                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
7186                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
7187                          RTE_ETHER_TYPE_MPLS);
7188                 break;
7189         default:
7190                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
7191                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
7192                          IPPROTO_MPLS);
7193                 break;
7194         }
7195         if (!in_mpls_v)
7196                 return;
7197         if (!in_mpls_m)
7198                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
7199         switch (prev_layer) {
7200         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7201                 out_mpls_m =
7202                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7203                                                  outer_first_mpls_over_udp);
7204                 out_mpls_v =
7205                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7206                                                  outer_first_mpls_over_udp);
7207                 break;
7208         case MLX5_FLOW_LAYER_GRE:
7209                 out_mpls_m =
7210                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7211                                                  outer_first_mpls_over_gre);
7212                 out_mpls_v =
7213                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7214                                                  outer_first_mpls_over_gre);
7215                 break;
7216         default:
7217                 /* Inner MPLS not over GRE is not supported. */
7218                 if (!inner) {
7219                         out_mpls_m =
7220                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7221                                                          misc2_m,
7222                                                          outer_first_mpls);
7223                         out_mpls_v =
7224                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7225                                                          misc2_v,
7226                                                          outer_first_mpls);
7227                 }
7228                 break;
7229         }
7230         if (out_mpls_m && out_mpls_v) {
7231                 *out_mpls_m = *in_mpls_m;
7232                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
7233         }
7234 }
7235
7236 /**
7237  * Add metadata register item to matcher
7238  *
7239  * @param[in, out] matcher
7240  *   Flow matcher.
7241  * @param[in, out] key
7242  *   Flow matcher value.
7243  * @param[in] reg_type
7244  *   Type of device metadata register
7245  * @param[in] value
7246  *   Register value
7247  * @param[in] mask
7248  *   Register mask
7249  */
7250 static void
7251 flow_dv_match_meta_reg(void *matcher, void *key,
7252                        enum modify_reg reg_type,
7253                        uint32_t data, uint32_t mask)
7254 {
7255         void *misc2_m =
7256                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
7257         void *misc2_v =
7258                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7259         uint32_t temp;
7260
7261         data &= mask;
7262         switch (reg_type) {
7263         case REG_A:
7264                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
7265                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
7266                 break;
7267         case REG_B:
7268                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
7269                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
7270                 break;
7271         case REG_C_0:
7272                 /*
7273                  * The metadata register C0 field might be divided into
7274                  * source vport index and META item value, we should set
7275                  * this field according to specified mask, not as whole one.
7276                  */
7277                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
7278                 temp |= mask;
7279                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
7280                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
7281                 temp &= ~mask;
7282                 temp |= data;
7283                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
7284                 break;
7285         case REG_C_1:
7286                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
7287                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
7288                 break;
7289         case REG_C_2:
7290                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
7291                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
7292                 break;
7293         case REG_C_3:
7294                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
7295                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
7296                 break;
7297         case REG_C_4:
7298                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
7299                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
7300                 break;
7301         case REG_C_5:
7302                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
7303                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
7304                 break;
7305         case REG_C_6:
7306                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
7307                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
7308                 break;
7309         case REG_C_7:
7310                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
7311                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
7312                 break;
7313         default:
7314                 MLX5_ASSERT(false);
7315                 break;
7316         }
7317 }
7318
7319 /**
7320  * Add MARK item to matcher
7321  *
7322  * @param[in] dev
7323  *   The device to configure through.
7324  * @param[in, out] matcher
7325  *   Flow matcher.
7326  * @param[in, out] key
7327  *   Flow matcher value.
7328  * @param[in] item
7329  *   Flow pattern to translate.
7330  */
7331 static void
7332 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
7333                             void *matcher, void *key,
7334                             const struct rte_flow_item *item)
7335 {
7336         struct mlx5_priv *priv = dev->data->dev_private;
7337         const struct rte_flow_item_mark *mark;
7338         uint32_t value;
7339         uint32_t mask;
7340
7341         mark = item->mask ? (const void *)item->mask :
7342                             &rte_flow_item_mark_mask;
7343         mask = mark->id & priv->sh->dv_mark_mask;
7344         mark = (const void *)item->spec;
7345         MLX5_ASSERT(mark);
7346         value = mark->id & priv->sh->dv_mark_mask & mask;
7347         if (mask) {
7348                 enum modify_reg reg;
7349
7350                 /* Get the metadata register index for the mark. */
7351                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
7352                 MLX5_ASSERT(reg > 0);
7353                 if (reg == REG_C_0) {
7354                         struct mlx5_priv *priv = dev->data->dev_private;
7355                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7356                         uint32_t shl_c0 = rte_bsf32(msk_c0);
7357
7358                         mask &= msk_c0;
7359                         mask <<= shl_c0;
7360                         value <<= shl_c0;
7361                 }
7362                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7363         }
7364 }
7365
7366 /**
7367  * Add META item to matcher
7368  *
7369  * @param[in] dev
7370  *   The devich to configure through.
7371  * @param[in, out] matcher
7372  *   Flow matcher.
7373  * @param[in, out] key
7374  *   Flow matcher value.
7375  * @param[in] attr
7376  *   Attributes of flow that includes this item.
7377  * @param[in] item
7378  *   Flow pattern to translate.
7379  */
7380 static void
7381 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
7382                             void *matcher, void *key,
7383                             const struct rte_flow_attr *attr,
7384                             const struct rte_flow_item *item)
7385 {
7386         const struct rte_flow_item_meta *meta_m;
7387         const struct rte_flow_item_meta *meta_v;
7388
7389         meta_m = (const void *)item->mask;
7390         if (!meta_m)
7391                 meta_m = &rte_flow_item_meta_mask;
7392         meta_v = (const void *)item->spec;
7393         if (meta_v) {
7394                 int reg;
7395                 uint32_t value = meta_v->data;
7396                 uint32_t mask = meta_m->data;
7397
7398                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
7399                 if (reg < 0)
7400                         return;
7401                 /*
7402                  * In datapath code there is no endianness
7403                  * coversions for perfromance reasons, all
7404                  * pattern conversions are done in rte_flow.
7405                  */
7406                 value = rte_cpu_to_be_32(value);
7407                 mask = rte_cpu_to_be_32(mask);
7408                 if (reg == REG_C_0) {
7409                         struct mlx5_priv *priv = dev->data->dev_private;
7410                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7411                         uint32_t shl_c0 = rte_bsf32(msk_c0);
7412 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
7413                         uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
7414
7415                         value >>= shr_c0;
7416                         mask >>= shr_c0;
7417 #endif
7418                         value <<= shl_c0;
7419                         mask <<= shl_c0;
7420                         MLX5_ASSERT(msk_c0);
7421                         MLX5_ASSERT(!(~msk_c0 & mask));
7422                 }
7423                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7424         }
7425 }
7426
7427 /**
7428  * Add vport metadata Reg C0 item to matcher
7429  *
7430  * @param[in, out] matcher
7431  *   Flow matcher.
7432  * @param[in, out] key
7433  *   Flow matcher value.
7434  * @param[in] reg
7435  *   Flow pattern to translate.
7436  */
7437 static void
7438 flow_dv_translate_item_meta_vport(void *matcher, void *key,
7439                                   uint32_t value, uint32_t mask)
7440 {
7441         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
7442 }
7443
7444 /**
7445  * Add tag item to matcher
7446  *
7447  * @param[in] dev
7448  *   The devich to configure through.
7449  * @param[in, out] matcher
7450  *   Flow matcher.
7451  * @param[in, out] key
7452  *   Flow matcher value.
7453  * @param[in] item
7454  *   Flow pattern to translate.
7455  */
7456 static void
7457 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
7458                                 void *matcher, void *key,
7459                                 const struct rte_flow_item *item)
7460 {
7461         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
7462         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
7463         uint32_t mask, value;
7464
7465         MLX5_ASSERT(tag_v);
7466         value = tag_v->data;
7467         mask = tag_m ? tag_m->data : UINT32_MAX;
7468         if (tag_v->id == REG_C_0) {
7469                 struct mlx5_priv *priv = dev->data->dev_private;
7470                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7471                 uint32_t shl_c0 = rte_bsf32(msk_c0);
7472
7473                 mask &= msk_c0;
7474                 mask <<= shl_c0;
7475                 value <<= shl_c0;
7476         }
7477         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
7478 }
7479
7480 /**
7481  * Add TAG item to matcher
7482  *
7483  * @param[in] dev
7484  *   The devich to configure through.
7485  * @param[in, out] matcher
7486  *   Flow matcher.
7487  * @param[in, out] key
7488  *   Flow matcher value.
7489  * @param[in] item
7490  *   Flow pattern to translate.
7491  */
7492 static void
7493 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
7494                            void *matcher, void *key,
7495                            const struct rte_flow_item *item)
7496 {
7497         const struct rte_flow_item_tag *tag_v = item->spec;
7498         const struct rte_flow_item_tag *tag_m = item->mask;
7499         enum modify_reg reg;
7500
7501         MLX5_ASSERT(tag_v);
7502         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
7503         /* Get the metadata register index for the tag. */
7504         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
7505         MLX5_ASSERT(reg > 0);
7506         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
7507 }
7508
7509 /**
7510  * Add source vport match to the specified matcher.
7511  *
7512  * @param[in, out] matcher
7513  *   Flow matcher.
7514  * @param[in, out] key
7515  *   Flow matcher value.
7516  * @param[in] port
7517  *   Source vport value to match
7518  * @param[in] mask
7519  *   Mask
7520  */
7521 static void
7522 flow_dv_translate_item_source_vport(void *matcher, void *key,
7523                                     int16_t port, uint16_t mask)
7524 {
7525         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7526         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7527
7528         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
7529         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
7530 }
7531
7532 /**
7533  * Translate port-id item to eswitch match on  port-id.
7534  *
7535  * @param[in] dev
7536  *   The devich to configure through.
7537  * @param[in, out] matcher
7538  *   Flow matcher.
7539  * @param[in, out] key
7540  *   Flow matcher value.
7541  * @param[in] item
7542  *   Flow pattern to translate.
7543  *
7544  * @return
7545  *   0 on success, a negative errno value otherwise.
7546  */
7547 static int
7548 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
7549                                void *key, const struct rte_flow_item *item)
7550 {
7551         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
7552         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
7553         struct mlx5_priv *priv;
7554         uint16_t mask, id;
7555
7556         mask = pid_m ? pid_m->id : 0xffff;
7557         id = pid_v ? pid_v->id : dev->data->port_id;
7558         priv = mlx5_port_to_eswitch_info(id, item == NULL);
7559         if (!priv)
7560                 return -rte_errno;
7561         /* Translate to vport field or to metadata, depending on mode. */
7562         if (priv->vport_meta_mask)
7563                 flow_dv_translate_item_meta_vport(matcher, key,
7564                                                   priv->vport_meta_tag,
7565                                                   priv->vport_meta_mask);
7566         else
7567                 flow_dv_translate_item_source_vport(matcher, key,
7568                                                     priv->vport_id, mask);
7569         return 0;
7570 }
7571
7572 /**
7573  * Add ICMP6 item to matcher and to the value.
7574  *
7575  * @param[in, out] matcher
7576  *   Flow matcher.
7577  * @param[in, out] key
7578  *   Flow matcher value.
7579  * @param[in] item
7580  *   Flow pattern to translate.
7581  * @param[in] inner
7582  *   Item is inner pattern.
7583  */
7584 static void
7585 flow_dv_translate_item_icmp6(void *matcher, void *key,
7586                               const struct rte_flow_item *item,
7587                               int inner)
7588 {
7589         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
7590         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
7591         void *headers_m;
7592         void *headers_v;
7593         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7594                                      misc_parameters_3);
7595         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7596         if (inner) {
7597                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7598                                          inner_headers);
7599                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7600         } else {
7601                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7602                                          outer_headers);
7603                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7604         }
7605         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7606         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
7607         if (!icmp6_v)
7608                 return;
7609         if (!icmp6_m)
7610                 icmp6_m = &rte_flow_item_icmp6_mask;
7611         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
7612         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
7613                  icmp6_v->type & icmp6_m->type);
7614         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
7615         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
7616                  icmp6_v->code & icmp6_m->code);
7617 }
7618
7619 /**
7620  * Add ICMP item to matcher and to the value.
7621  *
7622  * @param[in, out] matcher
7623  *   Flow matcher.
7624  * @param[in, out] key
7625  *   Flow matcher value.
7626  * @param[in] item
7627  *   Flow pattern to translate.
7628  * @param[in] inner
7629  *   Item is inner pattern.
7630  */
7631 static void
7632 flow_dv_translate_item_icmp(void *matcher, void *key,
7633                             const struct rte_flow_item *item,
7634                             int inner)
7635 {
7636         const struct rte_flow_item_icmp *icmp_m = item->mask;
7637         const struct rte_flow_item_icmp *icmp_v = item->spec;
7638         uint32_t icmp_header_data_m = 0;
7639         uint32_t icmp_header_data_v = 0;
7640         void *headers_m;
7641         void *headers_v;
7642         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7643                                      misc_parameters_3);
7644         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7645         if (inner) {
7646                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7647                                          inner_headers);
7648                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7649         } else {
7650                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7651                                          outer_headers);
7652                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7653         }
7654         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7655         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
7656         if (!icmp_v)
7657                 return;
7658         if (!icmp_m)
7659                 icmp_m = &rte_flow_item_icmp_mask;
7660         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
7661                  icmp_m->hdr.icmp_type);
7662         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
7663                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
7664         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
7665                  icmp_m->hdr.icmp_code);
7666         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
7667                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
7668         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
7669         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
7670         if (icmp_header_data_m) {
7671                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
7672                 icmp_header_data_v |=
7673                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
7674                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
7675                          icmp_header_data_m);
7676                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
7677                          icmp_header_data_v & icmp_header_data_m);
7678         }
7679 }
7680
7681 /**
7682  * Add GTP item to matcher and to the value.
7683  *
7684  * @param[in, out] matcher
7685  *   Flow matcher.
7686  * @param[in, out] key
7687  *   Flow matcher value.
7688  * @param[in] item
7689  *   Flow pattern to translate.
7690  * @param[in] inner
7691  *   Item is inner pattern.
7692  */
7693 static void
7694 flow_dv_translate_item_gtp(void *matcher, void *key,
7695                            const struct rte_flow_item *item, int inner)
7696 {
7697         const struct rte_flow_item_gtp *gtp_m = item->mask;
7698         const struct rte_flow_item_gtp *gtp_v = item->spec;
7699         void *headers_m;
7700         void *headers_v;
7701         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7702                                      misc_parameters_3);
7703         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7704         uint16_t dport = RTE_GTPU_UDP_PORT;
7705
7706         if (inner) {
7707                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7708                                          inner_headers);
7709                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7710         } else {
7711                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7712                                          outer_headers);
7713                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7714         }
7715         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7716                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7717                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7718         }
7719         if (!gtp_v)
7720                 return;
7721         if (!gtp_m)
7722                 gtp_m = &rte_flow_item_gtp_mask;
7723         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
7724                  gtp_m->v_pt_rsv_flags);
7725         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
7726                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
7727         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
7728         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
7729                  gtp_v->msg_type & gtp_m->msg_type);
7730         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
7731                  rte_be_to_cpu_32(gtp_m->teid));
7732         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
7733                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
7734 }
7735
7736 /**
7737  * Add eCPRI item to matcher and to the value.
7738  *
7739  * @param[in] dev
7740  *   The devich to configure through.
7741  * @param[in, out] matcher
7742  *   Flow matcher.
7743  * @param[in, out] key
7744  *   Flow matcher value.
7745  * @param[in] item
7746  *   Flow pattern to translate.
7747  * @param[in] samples
7748  *   Sample IDs to be used in the matching.
7749  */
7750 static void
7751 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
7752                              void *key, const struct rte_flow_item *item)
7753 {
7754         struct mlx5_priv *priv = dev->data->dev_private;
7755         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
7756         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
7757         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
7758                                      misc_parameters_4);
7759         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
7760         uint32_t *samples;
7761         void *dw_m;
7762         void *dw_v;
7763
7764         if (!ecpri_v)
7765                 return;
7766         if (!ecpri_m)
7767                 ecpri_m = &rte_flow_item_ecpri_mask;
7768         /*
7769          * Maximal four DW samples are supported in a single matching now.
7770          * Two are used now for a eCPRI matching:
7771          * 1. Type: one byte, mask should be 0x00ff0000 in network order
7772          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
7773          *    if any.
7774          */
7775         if (!ecpri_m->hdr.common.u32)
7776                 return;
7777         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
7778         /* Need to take the whole DW as the mask to fill the entry. */
7779         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
7780                             prog_sample_field_value_0);
7781         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
7782                             prog_sample_field_value_0);
7783         /* Already big endian (network order) in the header. */
7784         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
7785         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32;
7786         /* Sample#0, used for matching type, offset 0. */
7787         MLX5_SET(fte_match_set_misc4, misc4_m,
7788                  prog_sample_field_id_0, samples[0]);
7789         /* It makes no sense to set the sample ID in the mask field. */
7790         MLX5_SET(fte_match_set_misc4, misc4_v,
7791                  prog_sample_field_id_0, samples[0]);
7792         /*
7793          * Checking if message body part needs to be matched.
7794          * Some wildcard rules only matching type field should be supported.
7795          */
7796         if (ecpri_m->hdr.dummy[0]) {
7797                 switch (ecpri_v->hdr.common.type) {
7798                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
7799                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
7800                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
7801                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
7802                                             prog_sample_field_value_1);
7803                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
7804                                             prog_sample_field_value_1);
7805                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
7806                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0];
7807                         /* Sample#1, to match message body, offset 4. */
7808                         MLX5_SET(fte_match_set_misc4, misc4_m,
7809                                  prog_sample_field_id_1, samples[1]);
7810                         MLX5_SET(fte_match_set_misc4, misc4_v,
7811                                  prog_sample_field_id_1, samples[1]);
7812                         break;
7813                 default:
7814                         /* Others, do not match any sample ID. */
7815                         break;
7816                 }
7817         }
7818 }
7819
7820 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
7821
7822 #define HEADER_IS_ZERO(match_criteria, headers)                              \
7823         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
7824                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
7825
7826 /**
7827  * Calculate flow matcher enable bitmap.
7828  *
7829  * @param match_criteria
7830  *   Pointer to flow matcher criteria.
7831  *
7832  * @return
7833  *   Bitmap of enabled fields.
7834  */
7835 static uint8_t
7836 flow_dv_matcher_enable(uint32_t *match_criteria)
7837 {
7838         uint8_t match_criteria_enable;
7839
7840         match_criteria_enable =
7841                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
7842                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
7843         match_criteria_enable |=
7844                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
7845                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
7846         match_criteria_enable |=
7847                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
7848                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
7849         match_criteria_enable |=
7850                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
7851                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
7852         match_criteria_enable |=
7853                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
7854                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
7855         match_criteria_enable |=
7856                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
7857                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
7858         return match_criteria_enable;
7859 }
7860
7861
7862 /**
7863  * Get a flow table.
7864  *
7865  * @param[in, out] dev
7866  *   Pointer to rte_eth_dev structure.
7867  * @param[in] table_id
7868  *   Table id to use.
7869  * @param[in] egress
7870  *   Direction of the table.
7871  * @param[in] transfer
7872  *   E-Switch or NIC flow.
7873  * @param[out] error
7874  *   pointer to error structure.
7875  *
7876  * @return
7877  *   Returns tables resource based on the index, NULL in case of failed.
7878  */
7879 static struct mlx5_flow_tbl_resource *
7880 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
7881                          uint32_t table_id, uint8_t egress,
7882                          uint8_t transfer,
7883                          struct rte_flow_error *error)
7884 {
7885         struct mlx5_priv *priv = dev->data->dev_private;
7886         struct mlx5_dev_ctx_shared *sh = priv->sh;
7887         struct mlx5_flow_tbl_resource *tbl;
7888         union mlx5_flow_tbl_key table_key = {
7889                 {
7890                         .table_id = table_id,
7891                         .reserved = 0,
7892                         .domain = !!transfer,
7893                         .direction = !!egress,
7894                 }
7895         };
7896         struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
7897                                                          table_key.v64);
7898         struct mlx5_flow_tbl_data_entry *tbl_data;
7899         uint32_t idx = 0;
7900         int ret;
7901         void *domain;
7902
7903         if (pos) {
7904                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
7905                                         entry);
7906                 tbl = &tbl_data->tbl;
7907                 rte_atomic32_inc(&tbl->refcnt);
7908                 return tbl;
7909         }
7910         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
7911         if (!tbl_data) {
7912                 rte_flow_error_set(error, ENOMEM,
7913                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7914                                    NULL,
7915                                    "cannot allocate flow table data entry");
7916                 return NULL;
7917         }
7918         tbl_data->idx = idx;
7919         tbl = &tbl_data->tbl;
7920         pos = &tbl_data->entry;
7921         if (transfer)
7922                 domain = sh->fdb_domain;
7923         else if (egress)
7924                 domain = sh->tx_domain;
7925         else
7926                 domain = sh->rx_domain;
7927         ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj);
7928         if (ret) {
7929                 rte_flow_error_set(error, ENOMEM,
7930                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7931                                    NULL, "cannot create flow table object");
7932                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7933                 return NULL;
7934         }
7935         /*
7936          * No multi-threads now, but still better to initialize the reference
7937          * count before insert it into the hash list.
7938          */
7939         rte_atomic32_init(&tbl->refcnt);
7940         /* Jump action reference count is initialized here. */
7941         rte_atomic32_init(&tbl_data->jump.refcnt);
7942         pos->key = table_key.v64;
7943         ret = mlx5_hlist_insert(sh->flow_tbls, pos);
7944         if (ret < 0) {
7945                 rte_flow_error_set(error, -ret,
7946                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7947                                    "cannot insert flow table data entry");
7948                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
7949                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7950         }
7951         rte_atomic32_inc(&tbl->refcnt);
7952         return tbl;
7953 }
7954
7955 /**
7956  * Release a flow table.
7957  *
7958  * @param[in] dev
7959  *   Pointer to rte_eth_dev structure.
7960  * @param[in] tbl
7961  *   Table resource to be released.
7962  *
7963  * @return
7964  *   Returns 0 if table was released, else return 1;
7965  */
7966 static int
7967 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
7968                              struct mlx5_flow_tbl_resource *tbl)
7969 {
7970         struct mlx5_priv *priv = dev->data->dev_private;
7971         struct mlx5_dev_ctx_shared *sh = priv->sh;
7972         struct mlx5_flow_tbl_data_entry *tbl_data =
7973                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7974
7975         if (!tbl)
7976                 return 0;
7977         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
7978                 struct mlx5_hlist_entry *pos = &tbl_data->entry;
7979
7980                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
7981                 tbl->obj = NULL;
7982                 /* remove the entry from the hash list and free memory. */
7983                 mlx5_hlist_remove(sh->flow_tbls, pos);
7984                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
7985                                 tbl_data->idx);
7986                 return 0;
7987         }
7988         return 1;
7989 }
7990
7991 /**
7992  * Register the flow matcher.
7993  *
7994  * @param[in, out] dev
7995  *   Pointer to rte_eth_dev structure.
7996  * @param[in, out] matcher
7997  *   Pointer to flow matcher.
7998  * @param[in, out] key
7999  *   Pointer to flow table key.
8000  * @parm[in, out] dev_flow
8001  *   Pointer to the dev_flow.
8002  * @param[out] error
8003  *   pointer to error structure.
8004  *
8005  * @return
8006  *   0 on success otherwise -errno and errno is set.
8007  */
8008 static int
8009 flow_dv_matcher_register(struct rte_eth_dev *dev,
8010                          struct mlx5_flow_dv_matcher *matcher,
8011                          union mlx5_flow_tbl_key *key,
8012                          struct mlx5_flow *dev_flow,
8013                          struct rte_flow_error *error)
8014 {
8015         struct mlx5_priv *priv = dev->data->dev_private;
8016         struct mlx5_dev_ctx_shared *sh = priv->sh;
8017         struct mlx5_flow_dv_matcher *cache_matcher;
8018         struct mlx5dv_flow_matcher_attr dv_attr = {
8019                 .type = IBV_FLOW_ATTR_NORMAL,
8020                 .match_mask = (void *)&matcher->mask,
8021         };
8022         struct mlx5_flow_tbl_resource *tbl;
8023         struct mlx5_flow_tbl_data_entry *tbl_data;
8024         int ret;
8025
8026         tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
8027                                        key->domain, error);
8028         if (!tbl)
8029                 return -rte_errno;      /* No need to refill the error info */
8030         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
8031         /* Lookup from cache. */
8032         LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
8033                 if (matcher->crc == cache_matcher->crc &&
8034                     matcher->priority == cache_matcher->priority &&
8035                     !memcmp((const void *)matcher->mask.buf,
8036                             (const void *)cache_matcher->mask.buf,
8037                             cache_matcher->mask.size)) {
8038                         DRV_LOG(DEBUG,
8039                                 "%s group %u priority %hd use %s "
8040                                 "matcher %p: refcnt %d++",
8041                                 key->domain ? "FDB" : "NIC", key->table_id,
8042                                 cache_matcher->priority,
8043                                 key->direction ? "tx" : "rx",
8044                                 (void *)cache_matcher,
8045                                 rte_atomic32_read(&cache_matcher->refcnt));
8046                         rte_atomic32_inc(&cache_matcher->refcnt);
8047                         dev_flow->handle->dvh.matcher = cache_matcher;
8048                         /* old matcher should not make the table ref++. */
8049                         flow_dv_tbl_resource_release(dev, tbl);
8050                         return 0;
8051                 }
8052         }
8053         /* Register new matcher. */
8054         cache_matcher = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache_matcher), 0,
8055                                     SOCKET_ID_ANY);
8056         if (!cache_matcher) {
8057                 flow_dv_tbl_resource_release(dev, tbl);
8058                 return rte_flow_error_set(error, ENOMEM,
8059                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8060                                           "cannot allocate matcher memory");
8061         }
8062         *cache_matcher = *matcher;
8063         dv_attr.match_criteria_enable =
8064                 flow_dv_matcher_enable(cache_matcher->mask.buf);
8065         dv_attr.priority = matcher->priority;
8066         if (key->direction)
8067                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
8068         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
8069                                                &cache_matcher->matcher_object);
8070         if (ret) {
8071                 mlx5_free(cache_matcher);
8072 #ifdef HAVE_MLX5DV_DR
8073                 flow_dv_tbl_resource_release(dev, tbl);
8074 #endif
8075                 return rte_flow_error_set(error, ENOMEM,
8076                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8077                                           NULL, "cannot create matcher");
8078         }
8079         /* Save the table information */
8080         cache_matcher->tbl = tbl;
8081         rte_atomic32_init(&cache_matcher->refcnt);
8082         /* only matcher ref++, table ref++ already done above in get API. */
8083         rte_atomic32_inc(&cache_matcher->refcnt);
8084         LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
8085         dev_flow->handle->dvh.matcher = cache_matcher;
8086         DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
8087                 key->domain ? "FDB" : "NIC", key->table_id,
8088                 cache_matcher->priority,
8089                 key->direction ? "tx" : "rx", (void *)cache_matcher,
8090                 rte_atomic32_read(&cache_matcher->refcnt));
8091         return 0;
8092 }
8093
8094 /**
8095  * Find existing tag resource or create and register a new one.
8096  *
8097  * @param dev[in, out]
8098  *   Pointer to rte_eth_dev structure.
8099  * @param[in, out] tag_be24
8100  *   Tag value in big endian then R-shift 8.
8101  * @parm[in, out] dev_flow
8102  *   Pointer to the dev_flow.
8103  * @param[out] error
8104  *   pointer to error structure.
8105  *
8106  * @return
8107  *   0 on success otherwise -errno and errno is set.
8108  */
8109 static int
8110 flow_dv_tag_resource_register
8111                         (struct rte_eth_dev *dev,
8112                          uint32_t tag_be24,
8113                          struct mlx5_flow *dev_flow,
8114                          struct rte_flow_error *error)
8115 {
8116         struct mlx5_priv *priv = dev->data->dev_private;
8117         struct mlx5_dev_ctx_shared *sh = priv->sh;
8118         struct mlx5_flow_dv_tag_resource *cache_resource;
8119         struct mlx5_hlist_entry *entry;
8120         int ret;
8121
8122         /* Lookup a matching resource from cache. */
8123         entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
8124         if (entry) {
8125                 cache_resource = container_of
8126                         (entry, struct mlx5_flow_dv_tag_resource, entry);
8127                 rte_atomic32_inc(&cache_resource->refcnt);
8128                 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
8129                 dev_flow->dv.tag_resource = cache_resource;
8130                 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
8131                         (void *)cache_resource,
8132                         rte_atomic32_read(&cache_resource->refcnt));
8133                 return 0;
8134         }
8135         /* Register new resource. */
8136         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG],
8137                                        &dev_flow->handle->dvh.rix_tag);
8138         if (!cache_resource)
8139                 return rte_flow_error_set(error, ENOMEM,
8140                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8141                                           "cannot allocate resource memory");
8142         cache_resource->entry.key = (uint64_t)tag_be24;
8143         ret = mlx5_flow_os_create_flow_action_tag(tag_be24,
8144                                                   &cache_resource->action);
8145         if (ret) {
8146                 mlx5_free(cache_resource);
8147                 return rte_flow_error_set(error, ENOMEM,
8148                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8149                                           NULL, "cannot create action");
8150         }
8151         rte_atomic32_init(&cache_resource->refcnt);
8152         rte_atomic32_inc(&cache_resource->refcnt);
8153         if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
8154                 mlx5_flow_os_destroy_flow_action(cache_resource->action);
8155                 mlx5_free(cache_resource);
8156                 return rte_flow_error_set(error, EEXIST,
8157                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8158                                           NULL, "cannot insert tag");
8159         }
8160         dev_flow->dv.tag_resource = cache_resource;
8161         DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
8162                 (void *)cache_resource,
8163                 rte_atomic32_read(&cache_resource->refcnt));
8164         return 0;
8165 }
8166
8167 /**
8168  * Release the tag.
8169  *
8170  * @param dev
8171  *   Pointer to Ethernet device.
8172  * @param tag_idx
8173  *   Tag index.
8174  *
8175  * @return
8176  *   1 while a reference on it exists, 0 when freed.
8177  */
8178 static int
8179 flow_dv_tag_release(struct rte_eth_dev *dev,
8180                     uint32_t tag_idx)
8181 {
8182         struct mlx5_priv *priv = dev->data->dev_private;
8183         struct mlx5_dev_ctx_shared *sh = priv->sh;
8184         struct mlx5_flow_dv_tag_resource *tag;
8185
8186         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8187         if (!tag)
8188                 return 0;
8189         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
8190                 dev->data->port_id, (void *)tag,
8191                 rte_atomic32_read(&tag->refcnt));
8192         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
8193                 claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
8194                 mlx5_hlist_remove(sh->tag_table, &tag->entry);
8195                 DRV_LOG(DEBUG, "port %u tag %p: removed",
8196                         dev->data->port_id, (void *)tag);
8197                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8198                 return 0;
8199         }
8200         return 1;
8201 }
8202
8203 /**
8204  * Translate port ID action to vport.
8205  *
8206  * @param[in] dev
8207  *   Pointer to rte_eth_dev structure.
8208  * @param[in] action
8209  *   Pointer to the port ID action.
8210  * @param[out] dst_port_id
8211  *   The target port ID.
8212  * @param[out] error
8213  *   Pointer to the error structure.
8214  *
8215  * @return
8216  *   0 on success, a negative errno value otherwise and rte_errno is set.
8217  */
8218 static int
8219 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
8220                                  const struct rte_flow_action *action,
8221                                  uint32_t *dst_port_id,
8222                                  struct rte_flow_error *error)
8223 {
8224         uint32_t port;
8225         struct mlx5_priv *priv;
8226         const struct rte_flow_action_port_id *conf =
8227                         (const struct rte_flow_action_port_id *)action->conf;
8228
8229         port = conf->original ? dev->data->port_id : conf->id;
8230         priv = mlx5_port_to_eswitch_info(port, false);
8231         if (!priv)
8232                 return rte_flow_error_set(error, -rte_errno,
8233                                           RTE_FLOW_ERROR_TYPE_ACTION,
8234                                           NULL,
8235                                           "No eswitch info was found for port");
8236 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
8237         /*
8238          * This parameter is transferred to
8239          * mlx5dv_dr_action_create_dest_ib_port().
8240          */
8241         *dst_port_id = priv->dev_port;
8242 #else
8243         /*
8244          * Legacy mode, no LAG configurations is supported.
8245          * This parameter is transferred to
8246          * mlx5dv_dr_action_create_dest_vport().
8247          */
8248         *dst_port_id = priv->vport_id;
8249 #endif
8250         return 0;
8251 }
8252
8253 /**
8254  * Create a counter with aging configuration.
8255  *
8256  * @param[in] dev
8257  *   Pointer to rte_eth_dev structure.
8258  * @param[out] count
8259  *   Pointer to the counter action configuration.
8260  * @param[in] age
8261  *   Pointer to the aging action configuration.
8262  *
8263  * @return
8264  *   Index to flow counter on success, 0 otherwise.
8265  */
8266 static uint32_t
8267 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
8268                                 struct mlx5_flow *dev_flow,
8269                                 const struct rte_flow_action_count *count,
8270                                 const struct rte_flow_action_age *age)
8271 {
8272         uint32_t counter;
8273         struct mlx5_age_param *age_param;
8274
8275         if (count && count->shared)
8276                 counter = flow_dv_counter_get_shared(dev, count->id);
8277         else
8278                 counter = flow_dv_counter_alloc(dev, !!age);
8279         if (!counter || age == NULL)
8280                 return counter;
8281         age_param  = flow_dv_counter_idx_get_age(dev, counter);
8282         age_param->context = age->context ? age->context :
8283                 (void *)(uintptr_t)(dev_flow->flow_idx);
8284         age_param->timeout = age->timeout;
8285         age_param->port_id = dev->data->port_id;
8286         __atomic_store_n(&age_param->sec_since_last_hit, 0, __ATOMIC_RELAXED);
8287         __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED);
8288         return counter;
8289 }
8290 /**
8291  * Add Tx queue matcher
8292  *
8293  * @param[in] dev
8294  *   Pointer to the dev struct.
8295  * @param[in, out] matcher
8296  *   Flow matcher.
8297  * @param[in, out] key
8298  *   Flow matcher value.
8299  * @param[in] item
8300  *   Flow pattern to translate.
8301  * @param[in] inner
8302  *   Item is inner pattern.
8303  */
8304 static void
8305 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
8306                                 void *matcher, void *key,
8307                                 const struct rte_flow_item *item)
8308 {
8309         const struct mlx5_rte_flow_item_tx_queue *queue_m;
8310         const struct mlx5_rte_flow_item_tx_queue *queue_v;
8311         void *misc_m =
8312                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8313         void *misc_v =
8314                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8315         struct mlx5_txq_ctrl *txq;
8316         uint32_t queue;
8317
8318
8319         queue_m = (const void *)item->mask;
8320         if (!queue_m)
8321                 return;
8322         queue_v = (const void *)item->spec;
8323         if (!queue_v)
8324                 return;
8325         txq = mlx5_txq_get(dev, queue_v->queue);
8326         if (!txq)
8327                 return;
8328         queue = txq->obj->sq->id;
8329         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
8330         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
8331                  queue & queue_m->queue);
8332         mlx5_txq_release(dev, queue_v->queue);
8333 }
8334
8335 /**
8336  * Set the hash fields according to the @p flow information.
8337  *
8338  * @param[in] dev_flow
8339  *   Pointer to the mlx5_flow.
8340  * @param[in] rss_desc
8341  *   Pointer to the mlx5_flow_rss_desc.
8342  */
8343 static void
8344 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
8345                        struct mlx5_flow_rss_desc *rss_desc)
8346 {
8347         uint64_t items = dev_flow->handle->layers;
8348         int rss_inner = 0;
8349         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
8350
8351         dev_flow->hash_fields = 0;
8352 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
8353         if (rss_desc->level >= 2) {
8354                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
8355                 rss_inner = 1;
8356         }
8357 #endif
8358         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
8359             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
8360                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
8361                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
8362                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
8363                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
8364                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
8365                         else
8366                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
8367                 }
8368         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
8369                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
8370                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
8371                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
8372                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
8373                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
8374                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
8375                         else
8376                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
8377                 }
8378         }
8379         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
8380             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
8381                 if (rss_types & ETH_RSS_UDP) {
8382                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
8383                                 dev_flow->hash_fields |=
8384                                                 IBV_RX_HASH_SRC_PORT_UDP;
8385                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
8386                                 dev_flow->hash_fields |=
8387                                                 IBV_RX_HASH_DST_PORT_UDP;
8388                         else
8389                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
8390                 }
8391         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
8392                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
8393                 if (rss_types & ETH_RSS_TCP) {
8394                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
8395                                 dev_flow->hash_fields |=
8396                                                 IBV_RX_HASH_SRC_PORT_TCP;
8397                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
8398                                 dev_flow->hash_fields |=
8399                                                 IBV_RX_HASH_DST_PORT_TCP;
8400                         else
8401                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
8402                 }
8403         }
8404 }
8405
8406 /**
8407  * Create an Rx Hash queue.
8408  *
8409  * @param dev
8410  *   Pointer to Ethernet device.
8411  * @param[in] dev_flow
8412  *   Pointer to the mlx5_flow.
8413  * @param[in] rss_desc
8414  *   Pointer to the mlx5_flow_rss_desc.
8415  * @param[out] hrxq_idx
8416  *   Hash Rx queue index.
8417  *
8418  * @return
8419  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
8420  */
8421 static struct mlx5_hrxq *
8422 flow_dv_handle_rx_queue(struct rte_eth_dev *dev,
8423                         struct mlx5_flow *dev_flow,
8424                         struct mlx5_flow_rss_desc *rss_desc,
8425                         uint32_t *hrxq_idx)
8426 {
8427         struct mlx5_priv *priv = dev->data->dev_private;
8428         struct mlx5_flow_handle *dh = dev_flow->handle;
8429         struct mlx5_hrxq *hrxq;
8430
8431         MLX5_ASSERT(rss_desc->queue_num);
8432         *hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
8433                                   MLX5_RSS_HASH_KEY_LEN,
8434                                   dev_flow->hash_fields,
8435                                   rss_desc->queue,
8436                                   rss_desc->queue_num);
8437         if (!*hrxq_idx) {
8438                 *hrxq_idx = mlx5_hrxq_new
8439                                 (dev, rss_desc->key,
8440                                  MLX5_RSS_HASH_KEY_LEN,
8441                                  dev_flow->hash_fields,
8442                                  rss_desc->queue,
8443                                  rss_desc->queue_num,
8444                                  !!(dh->layers &
8445                                  MLX5_FLOW_LAYER_TUNNEL));
8446                 if (!*hrxq_idx)
8447                         return NULL;
8448         }
8449         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
8450                               *hrxq_idx);
8451         return hrxq;
8452 }
8453
8454 /**
8455  * Find existing sample resource or create and register a new one.
8456  *
8457  * @param[in, out] dev
8458  *   Pointer to rte_eth_dev structure.
8459  * @param[in] attr
8460  *   Attributes of flow that includes this item.
8461  * @param[in] resource
8462  *   Pointer to sample resource.
8463  * @parm[in, out] dev_flow
8464  *   Pointer to the dev_flow.
8465  * @param[in, out] sample_dv_actions
8466  *   Pointer to sample actions list.
8467  * @param[out] error
8468  *   pointer to error structure.
8469  *
8470  * @return
8471  *   0 on success otherwise -errno and errno is set.
8472  */
8473 static int
8474 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
8475                          const struct rte_flow_attr *attr,
8476                          struct mlx5_flow_dv_sample_resource *resource,
8477                          struct mlx5_flow *dev_flow,
8478                          void **sample_dv_actions,
8479                          struct rte_flow_error *error)
8480 {
8481         struct mlx5_flow_dv_sample_resource *cache_resource;
8482         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
8483         struct mlx5_priv *priv = dev->data->dev_private;
8484         struct mlx5_dev_ctx_shared *sh = priv->sh;
8485         struct mlx5_flow_tbl_resource *tbl;
8486         uint32_t idx = 0;
8487         const uint32_t next_ft_step = 1;
8488         uint32_t next_ft_id = resource->ft_id + next_ft_step;
8489
8490         /* Lookup a matching resource from cache. */
8491         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_SAMPLE], sh->sample_action_list,
8492                       idx, cache_resource, next) {
8493                 if (resource->ratio == cache_resource->ratio &&
8494                     resource->ft_type == cache_resource->ft_type &&
8495                     resource->ft_id == cache_resource->ft_id &&
8496                     resource->set_action == cache_resource->set_action &&
8497                     !memcmp((void *)&resource->sample_act,
8498                             (void *)&cache_resource->sample_act,
8499                             sizeof(struct mlx5_flow_sub_actions_list))) {
8500                         DRV_LOG(DEBUG, "sample resource %p: refcnt %d++",
8501                                 (void *)cache_resource,
8502                                 __atomic_load_n(&cache_resource->refcnt,
8503                                                 __ATOMIC_RELAXED));
8504                         __atomic_fetch_add(&cache_resource->refcnt, 1,
8505                                            __ATOMIC_RELAXED);
8506                         dev_flow->handle->dvh.rix_sample = idx;
8507                         dev_flow->dv.sample_res = cache_resource;
8508                         return 0;
8509                 }
8510         }
8511         /* Register new sample resource. */
8512         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE],
8513                                        &dev_flow->handle->dvh.rix_sample);
8514         if (!cache_resource)
8515                 return rte_flow_error_set(error, ENOMEM,
8516                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8517                                           NULL,
8518                                           "cannot allocate resource memory");
8519         *cache_resource = *resource;
8520         /* Create normal path table level */
8521         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
8522                                         attr->egress, attr->transfer, error);
8523         if (!tbl) {
8524                 rte_flow_error_set(error, ENOMEM,
8525                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8526                                           NULL,
8527                                           "fail to create normal path table "
8528                                           "for sample");
8529                 goto error;
8530         }
8531         cache_resource->normal_path_tbl = tbl;
8532         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
8533                 cache_resource->default_miss =
8534                                 mlx5_glue->dr_create_flow_action_default_miss();
8535                 if (!cache_resource->default_miss) {
8536                         rte_flow_error_set(error, ENOMEM,
8537                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8538                                                 NULL,
8539                                                 "cannot create default miss "
8540                                                 "action");
8541                         goto error;
8542                 }
8543                 sample_dv_actions[resource->sample_act.actions_num++] =
8544                                                 cache_resource->default_miss;
8545         }
8546         /* Create a DR sample action */
8547         sampler_attr.sample_ratio = cache_resource->ratio;
8548         sampler_attr.default_next_table = tbl->obj;
8549         sampler_attr.num_sample_actions = resource->sample_act.actions_num;
8550         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
8551                                                         &sample_dv_actions[0];
8552         sampler_attr.action = cache_resource->set_action;
8553         cache_resource->verbs_action =
8554                 mlx5_glue->dr_create_flow_action_sampler(&sampler_attr);
8555         if (!cache_resource->verbs_action) {
8556                 rte_flow_error_set(error, ENOMEM,
8557                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8558                                         NULL, "cannot create sample action");
8559                 goto error;
8560         }
8561         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8562         ILIST_INSERT(sh->ipool[MLX5_IPOOL_SAMPLE], &sh->sample_action_list,
8563                      dev_flow->handle->dvh.rix_sample, cache_resource,
8564                      next);
8565         dev_flow->dv.sample_res = cache_resource;
8566         DRV_LOG(DEBUG, "new sample resource %p: refcnt %d++",
8567                 (void *)cache_resource,
8568                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8569         return 0;
8570 error:
8571         if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
8572                 if (cache_resource->default_miss)
8573                         claim_zero(mlx5_glue->destroy_flow_action
8574                                 (cache_resource->default_miss));
8575         } else {
8576                 if (cache_resource->sample_idx.rix_hrxq &&
8577                     !mlx5_hrxq_release(dev,
8578                                 cache_resource->sample_idx.rix_hrxq))
8579                         cache_resource->sample_idx.rix_hrxq = 0;
8580                 if (cache_resource->sample_idx.rix_tag &&
8581                     !flow_dv_tag_release(dev,
8582                                 cache_resource->sample_idx.rix_tag))
8583                         cache_resource->sample_idx.rix_tag = 0;
8584                 if (cache_resource->sample_idx.cnt) {
8585                         flow_dv_counter_release(dev,
8586                                 cache_resource->sample_idx.cnt);
8587                         cache_resource->sample_idx.cnt = 0;
8588                 }
8589         }
8590         if (cache_resource->normal_path_tbl)
8591                 flow_dv_tbl_resource_release(dev,
8592                                 cache_resource->normal_path_tbl);
8593         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE],
8594                                 dev_flow->handle->dvh.rix_sample);
8595         dev_flow->handle->dvh.rix_sample = 0;
8596         return -rte_errno;
8597 }
8598
8599 /**
8600  * Find existing destination array resource or create and register a new one.
8601  *
8602  * @param[in, out] dev
8603  *   Pointer to rte_eth_dev structure.
8604  * @param[in] attr
8605  *   Attributes of flow that includes this item.
8606  * @param[in] resource
8607  *   Pointer to destination array resource.
8608  * @parm[in, out] dev_flow
8609  *   Pointer to the dev_flow.
8610  * @param[out] error
8611  *   pointer to error structure.
8612  *
8613  * @return
8614  *   0 on success otherwise -errno and errno is set.
8615  */
8616 static int
8617 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
8618                          const struct rte_flow_attr *attr,
8619                          struct mlx5_flow_dv_dest_array_resource *resource,
8620                          struct mlx5_flow *dev_flow,
8621                          struct rte_flow_error *error)
8622 {
8623         struct mlx5_flow_dv_dest_array_resource *cache_resource;
8624         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
8625         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
8626         struct mlx5_priv *priv = dev->data->dev_private;
8627         struct mlx5_dev_ctx_shared *sh = priv->sh;
8628         struct mlx5_flow_sub_actions_list *sample_act;
8629         struct mlx5dv_dr_domain *domain;
8630         uint32_t idx = 0;
8631
8632         /* Lookup a matching resource from cache. */
8633         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8634                       sh->dest_array_list,
8635                       idx, cache_resource, next) {
8636                 if (resource->num_of_dest == cache_resource->num_of_dest &&
8637                     resource->ft_type == cache_resource->ft_type &&
8638                     !memcmp((void *)cache_resource->sample_act,
8639                             (void *)resource->sample_act,
8640                            (resource->num_of_dest *
8641                            sizeof(struct mlx5_flow_sub_actions_list)))) {
8642                         DRV_LOG(DEBUG, "dest array resource %p: refcnt %d++",
8643                                 (void *)cache_resource,
8644                                 __atomic_load_n(&cache_resource->refcnt,
8645                                                 __ATOMIC_RELAXED));
8646                         __atomic_fetch_add(&cache_resource->refcnt, 1,
8647                                            __ATOMIC_RELAXED);
8648                         dev_flow->handle->dvh.rix_dest_array = idx;
8649                         dev_flow->dv.dest_array_res = cache_resource;
8650                         return 0;
8651                 }
8652         }
8653         /* Register new destination array resource. */
8654         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8655                                        &dev_flow->handle->dvh.rix_dest_array);
8656         if (!cache_resource)
8657                 return rte_flow_error_set(error, ENOMEM,
8658                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8659                                           NULL,
8660                                           "cannot allocate resource memory");
8661         *cache_resource = *resource;
8662         if (attr->transfer)
8663                 domain = sh->fdb_domain;
8664         else if (attr->ingress)
8665                 domain = sh->rx_domain;
8666         else
8667                 domain = sh->tx_domain;
8668         for (idx = 0; idx < resource->num_of_dest; idx++) {
8669                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
8670                                  mlx5_malloc(MLX5_MEM_ZERO,
8671                                  sizeof(struct mlx5dv_dr_action_dest_attr),
8672                                  0, SOCKET_ID_ANY);
8673                 if (!dest_attr[idx]) {
8674                         rte_flow_error_set(error, ENOMEM,
8675                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8676                                            NULL,
8677                                            "cannot allocate resource memory");
8678                         goto error;
8679                 }
8680                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
8681                 sample_act = &resource->sample_act[idx];
8682                 if (sample_act->action_flags == MLX5_FLOW_ACTION_QUEUE) {
8683                         dest_attr[idx]->dest = sample_act->dr_queue_action;
8684                 } else if (sample_act->action_flags ==
8685                           (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP)) {
8686                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
8687                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
8688                         dest_attr[idx]->dest_reformat->reformat =
8689                                         sample_act->dr_encap_action;
8690                         dest_attr[idx]->dest_reformat->dest =
8691                                         sample_act->dr_port_id_action;
8692                 } else if (sample_act->action_flags ==
8693                            MLX5_FLOW_ACTION_PORT_ID) {
8694                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
8695                 }
8696         }
8697         /* create a dest array actioin */
8698         cache_resource->action = mlx5_glue->dr_create_flow_action_dest_array
8699                                                 (domain,
8700                                                  cache_resource->num_of_dest,
8701                                                  dest_attr);
8702         if (!cache_resource->action) {
8703                 rte_flow_error_set(error, ENOMEM,
8704                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8705                                    NULL,
8706                                    "cannot create destination array action");
8707                 goto error;
8708         }
8709         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8710         ILIST_INSERT(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8711                      &sh->dest_array_list,
8712                      dev_flow->handle->dvh.rix_dest_array, cache_resource,
8713                      next);
8714         dev_flow->dv.dest_array_res = cache_resource;
8715         DRV_LOG(DEBUG, "new destination array resource %p: refcnt %d++",
8716                 (void *)cache_resource,
8717                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8718         for (idx = 0; idx < resource->num_of_dest; idx++)
8719                 mlx5_free(dest_attr[idx]);
8720         return 0;
8721 error:
8722         for (idx = 0; idx < resource->num_of_dest; idx++) {
8723                 struct mlx5_flow_sub_actions_idx *act_res =
8724                                         &cache_resource->sample_idx[idx];
8725                 if (act_res->rix_hrxq &&
8726                     !mlx5_hrxq_release(dev,
8727                                 act_res->rix_hrxq))
8728                         act_res->rix_hrxq = 0;
8729                 if (act_res->rix_encap_decap &&
8730                         !flow_dv_encap_decap_resource_release(dev,
8731                                 act_res->rix_encap_decap))
8732                         act_res->rix_encap_decap = 0;
8733                 if (act_res->rix_port_id_action &&
8734                         !flow_dv_port_id_action_resource_release(dev,
8735                                 act_res->rix_port_id_action))
8736                         act_res->rix_port_id_action = 0;
8737                 if (dest_attr[idx])
8738                         mlx5_free(dest_attr[idx]);
8739         }
8740
8741         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8742                                 dev_flow->handle->dvh.rix_dest_array);
8743         dev_flow->handle->dvh.rix_dest_array = 0;
8744         return -rte_errno;
8745 }
8746
8747 /**
8748  * Convert Sample action to DV specification.
8749  *
8750  * @param[in] dev
8751  *   Pointer to rte_eth_dev structure.
8752  * @param[in] action
8753  *   Pointer to action structure.
8754  * @param[in, out] dev_flow
8755  *   Pointer to the mlx5_flow.
8756  * @param[in] attr
8757  *   Pointer to the flow attributes.
8758  * @param[in, out] num_of_dest
8759  *   Pointer to the num of destination.
8760  * @param[in, out] sample_actions
8761  *   Pointer to sample actions list.
8762  * @param[in, out] res
8763  *   Pointer to sample resource.
8764  * @param[out] error
8765  *   Pointer to the error structure.
8766  *
8767  * @return
8768  *   0 on success, a negative errno value otherwise and rte_errno is set.
8769  */
8770 static int
8771 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
8772                                 const struct rte_flow_action *action,
8773                                 struct mlx5_flow *dev_flow,
8774                                 const struct rte_flow_attr *attr,
8775                                 uint32_t *num_of_dest,
8776                                 void **sample_actions,
8777                                 struct mlx5_flow_dv_sample_resource *res,
8778                                 struct rte_flow_error *error)
8779 {
8780         struct mlx5_priv *priv = dev->data->dev_private;
8781         const struct rte_flow_action_sample *sample_action;
8782         const struct rte_flow_action *sub_actions;
8783         const struct rte_flow_action_queue *queue;
8784         struct mlx5_flow_sub_actions_list *sample_act;
8785         struct mlx5_flow_sub_actions_idx *sample_idx;
8786         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
8787                                               priv->rss_desc)
8788                                               [!!priv->flow_nested_idx];
8789         uint64_t action_flags = 0;
8790
8791         sample_act = &res->sample_act;
8792         sample_idx = &res->sample_idx;
8793         sample_action = (const struct rte_flow_action_sample *)action->conf;
8794         res->ratio = sample_action->ratio;
8795         sub_actions = sample_action->actions;
8796         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
8797                 int type = sub_actions->type;
8798                 uint32_t pre_rix = 0;
8799                 void *pre_r;
8800                 switch (type) {
8801                 case RTE_FLOW_ACTION_TYPE_QUEUE:
8802                 {
8803                         struct mlx5_hrxq *hrxq;
8804                         uint32_t hrxq_idx;
8805
8806                         queue = sub_actions->conf;
8807                         rss_desc->queue_num = 1;
8808                         rss_desc->queue[0] = queue->index;
8809                         hrxq = flow_dv_handle_rx_queue(dev, dev_flow,
8810                                         rss_desc, &hrxq_idx);
8811                         if (!hrxq)
8812                                 return rte_flow_error_set
8813                                         (error, rte_errno,
8814                                          RTE_FLOW_ERROR_TYPE_ACTION,
8815                                          NULL,
8816                                          "cannot create fate queue");
8817                         sample_act->dr_queue_action = hrxq->action;
8818                         sample_idx->rix_hrxq = hrxq_idx;
8819                         sample_actions[sample_act->actions_num++] =
8820                                                 hrxq->action;
8821                         (*num_of_dest)++;
8822                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
8823                         if (action_flags & MLX5_FLOW_ACTION_MARK)
8824                                 dev_flow->handle->rix_hrxq = hrxq_idx;
8825                         dev_flow->handle->fate_action =
8826                                         MLX5_FLOW_FATE_QUEUE;
8827                         break;
8828                 }
8829                 case RTE_FLOW_ACTION_TYPE_MARK:
8830                 {
8831                         uint32_t tag_be = mlx5_flow_mark_set
8832                                 (((const struct rte_flow_action_mark *)
8833                                 (sub_actions->conf))->id);
8834
8835                         dev_flow->handle->mark = 1;
8836                         pre_rix = dev_flow->handle->dvh.rix_tag;
8837                         /* Save the mark resource before sample */
8838                         pre_r = dev_flow->dv.tag_resource;
8839                         if (flow_dv_tag_resource_register(dev, tag_be,
8840                                                   dev_flow, error))
8841                                 return -rte_errno;
8842                         MLX5_ASSERT(dev_flow->dv.tag_resource);
8843                         sample_act->dr_tag_action =
8844                                 dev_flow->dv.tag_resource->action;
8845                         sample_idx->rix_tag =
8846                                 dev_flow->handle->dvh.rix_tag;
8847                         sample_actions[sample_act->actions_num++] =
8848                                                 sample_act->dr_tag_action;
8849                         /* Recover the mark resource after sample */
8850                         dev_flow->dv.tag_resource = pre_r;
8851                         dev_flow->handle->dvh.rix_tag = pre_rix;
8852                         action_flags |= MLX5_FLOW_ACTION_MARK;
8853                         break;
8854                 }
8855                 case RTE_FLOW_ACTION_TYPE_COUNT:
8856                 {
8857                         uint32_t counter;
8858
8859                         counter = flow_dv_translate_create_counter(dev,
8860                                         dev_flow, sub_actions->conf, 0);
8861                         if (!counter)
8862                                 return rte_flow_error_set
8863                                                 (error, rte_errno,
8864                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8865                                                  NULL,
8866                                                  "cannot create counter"
8867                                                  " object.");
8868                         sample_idx->cnt = counter;
8869                         sample_act->dr_cnt_action =
8870                                   (flow_dv_counter_get_by_idx(dev,
8871                                   counter, NULL))->action;
8872                         sample_actions[sample_act->actions_num++] =
8873                                                 sample_act->dr_cnt_action;
8874                         action_flags |= MLX5_FLOW_ACTION_COUNT;
8875                         break;
8876                 }
8877                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
8878                 {
8879                         struct mlx5_flow_dv_port_id_action_resource
8880                                         port_id_resource;
8881                         uint32_t port_id = 0;
8882
8883                         memset(&port_id_resource, 0, sizeof(port_id_resource));
8884                         /* Save the port id resource before sample */
8885                         pre_rix = dev_flow->handle->rix_port_id_action;
8886                         pre_r = dev_flow->dv.port_id_action;
8887                         if (flow_dv_translate_action_port_id(dev, sub_actions,
8888                                                              &port_id, error))
8889                                 return -rte_errno;
8890                         port_id_resource.port_id = port_id;
8891                         if (flow_dv_port_id_action_resource_register
8892                             (dev, &port_id_resource, dev_flow, error))
8893                                 return -rte_errno;
8894                         sample_act->dr_port_id_action =
8895                                 dev_flow->dv.port_id_action->action;
8896                         sample_idx->rix_port_id_action =
8897                                 dev_flow->handle->rix_port_id_action;
8898                         sample_actions[sample_act->actions_num++] =
8899                                                 sample_act->dr_port_id_action;
8900                         /* Recover the port id resource after sample */
8901                         dev_flow->dv.port_id_action = pre_r;
8902                         dev_flow->handle->rix_port_id_action = pre_rix;
8903                         (*num_of_dest)++;
8904                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
8905                         break;
8906                 }
8907                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
8908                         /* Save the encap resource before sample */
8909                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
8910                         pre_r = dev_flow->dv.encap_decap;
8911                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
8912                                                            dev_flow,
8913                                                            attr->transfer,
8914                                                            error))
8915                                 return -rte_errno;
8916                         sample_act->dr_encap_action =
8917                                 dev_flow->dv.encap_decap->action;
8918                         sample_idx->rix_encap_decap =
8919                                 dev_flow->handle->dvh.rix_encap_decap;
8920                         sample_actions[sample_act->actions_num++] =
8921                                                 sample_act->dr_encap_action;
8922                         /* Recover the encap resource after sample */
8923                         dev_flow->dv.encap_decap = pre_r;
8924                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
8925                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
8926                         break;
8927                 default:
8928                         return rte_flow_error_set(error, EINVAL,
8929                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8930                                 NULL,
8931                                 "Not support for sampler action");
8932                 }
8933         }
8934         sample_act->action_flags = action_flags;
8935         res->ft_id = dev_flow->dv.group;
8936         if (attr->transfer) {
8937                 union {
8938                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
8939                         uint64_t set_action;
8940                 } action_ctx = { .set_action = 0 };
8941
8942                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
8943                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
8944                          MLX5_MODIFICATION_TYPE_SET);
8945                 MLX5_SET(set_action_in, action_ctx.action_in, field,
8946                          MLX5_MODI_META_REG_C_0);
8947                 MLX5_SET(set_action_in, action_ctx.action_in, data,
8948                          priv->vport_meta_tag);
8949                 res->set_action = action_ctx.set_action;
8950         } else if (attr->ingress) {
8951                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
8952         }
8953         return 0;
8954 }
8955
8956 /**
8957  * Convert Sample action to DV specification.
8958  *
8959  * @param[in] dev
8960  *   Pointer to rte_eth_dev structure.
8961  * @param[in, out] dev_flow
8962  *   Pointer to the mlx5_flow.
8963  * @param[in] attr
8964  *   Pointer to the flow attributes.
8965  * @param[in] num_of_dest
8966  *   The num of destination.
8967  * @param[in, out] res
8968  *   Pointer to sample resource.
8969  * @param[in, out] mdest_res
8970  *   Pointer to destination array resource.
8971  * @param[in] sample_actions
8972  *   Pointer to sample path actions list.
8973  * @param[in] action_flags
8974  *   Holds the actions detected until now.
8975  * @param[out] error
8976  *   Pointer to the error structure.
8977  *
8978  * @return
8979  *   0 on success, a negative errno value otherwise and rte_errno is set.
8980  */
8981 static int
8982 flow_dv_create_action_sample(struct rte_eth_dev *dev,
8983                              struct mlx5_flow *dev_flow,
8984                              const struct rte_flow_attr *attr,
8985                              uint32_t num_of_dest,
8986                              struct mlx5_flow_dv_sample_resource *res,
8987                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
8988                              void **sample_actions,
8989                              uint64_t action_flags,
8990                              struct rte_flow_error *error)
8991 {
8992         struct mlx5_priv *priv = dev->data->dev_private;
8993         /* update normal path action resource into last index of array */
8994         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
8995         struct mlx5_flow_sub_actions_list *sample_act =
8996                                         &mdest_res->sample_act[dest_index];
8997         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
8998                                               priv->rss_desc)
8999                                               [!!priv->flow_nested_idx];
9000         uint32_t normal_idx = 0;
9001         struct mlx5_hrxq *hrxq;
9002         uint32_t hrxq_idx;
9003
9004         if (num_of_dest > 1) {
9005                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
9006                         /* Handle QP action for mirroring */
9007                         hrxq = flow_dv_handle_rx_queue(dev, dev_flow,
9008                                                        rss_desc, &hrxq_idx);
9009                         if (!hrxq)
9010                                 return rte_flow_error_set
9011                                      (error, rte_errno,
9012                                       RTE_FLOW_ERROR_TYPE_ACTION,
9013                                       NULL,
9014                                       "cannot create rx queue");
9015                         normal_idx++;
9016                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
9017                         sample_act->dr_queue_action = hrxq->action;
9018                         if (action_flags & MLX5_FLOW_ACTION_MARK)
9019                                 dev_flow->handle->rix_hrxq = hrxq_idx;
9020                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9021                 }
9022                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
9023                         normal_idx++;
9024                         mdest_res->sample_idx[dest_index].rix_encap_decap =
9025                                 dev_flow->handle->dvh.rix_encap_decap;
9026                         sample_act->dr_encap_action =
9027                                 dev_flow->dv.encap_decap->action;
9028                 }
9029                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
9030                         normal_idx++;
9031                         mdest_res->sample_idx[dest_index].rix_port_id_action =
9032                                 dev_flow->handle->rix_port_id_action;
9033                         sample_act->dr_port_id_action =
9034                                 dev_flow->dv.port_id_action->action;
9035                 }
9036                 sample_act->actions_num = normal_idx;
9037                 /* update sample action resource into first index of array */
9038                 mdest_res->ft_type = res->ft_type;
9039                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
9040                                 sizeof(struct mlx5_flow_sub_actions_idx));
9041                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
9042                                 sizeof(struct mlx5_flow_sub_actions_list));
9043                 mdest_res->num_of_dest = num_of_dest;
9044                 if (flow_dv_dest_array_resource_register(dev, attr, mdest_res,
9045                                                          dev_flow, error))
9046                         return rte_flow_error_set(error, EINVAL,
9047                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9048                                                   NULL, "can't create sample "
9049                                                   "action");
9050         } else {
9051                 if (flow_dv_sample_resource_register(dev, attr, res, dev_flow,
9052                                                      sample_actions, error))
9053                         return rte_flow_error_set(error, EINVAL,
9054                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9055                                                   NULL,
9056                                                   "can't create sample action");
9057         }
9058         return 0;
9059 }
9060
9061 /**
9062  * Fill the flow with DV spec, lock free
9063  * (mutex should be acquired by caller).
9064  *
9065  * @param[in] dev
9066  *   Pointer to rte_eth_dev structure.
9067  * @param[in, out] dev_flow
9068  *   Pointer to the sub flow.
9069  * @param[in] attr
9070  *   Pointer to the flow attributes.
9071  * @param[in] items
9072  *   Pointer to the list of items.
9073  * @param[in] actions
9074  *   Pointer to the list of actions.
9075  * @param[out] error
9076  *   Pointer to the error structure.
9077  *
9078  * @return
9079  *   0 on success, a negative errno value otherwise and rte_errno is set.
9080  */
9081 static int
9082 __flow_dv_translate(struct rte_eth_dev *dev,
9083                     struct mlx5_flow *dev_flow,
9084                     const struct rte_flow_attr *attr,
9085                     const struct rte_flow_item items[],
9086                     const struct rte_flow_action actions[],
9087                     struct rte_flow_error *error)
9088 {
9089         struct mlx5_priv *priv = dev->data->dev_private;
9090         struct mlx5_dev_config *dev_conf = &priv->config;
9091         struct rte_flow *flow = dev_flow->flow;
9092         struct mlx5_flow_handle *handle = dev_flow->handle;
9093         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
9094                                               priv->rss_desc)
9095                                               [!!priv->flow_nested_idx];
9096         uint64_t item_flags = 0;
9097         uint64_t last_item = 0;
9098         uint64_t action_flags = 0;
9099         uint64_t priority = attr->priority;
9100         struct mlx5_flow_dv_matcher matcher = {
9101                 .mask = {
9102                         .size = sizeof(matcher.mask.buf) -
9103                                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
9104                 },
9105         };
9106         int actions_n = 0;
9107         bool actions_end = false;
9108         union {
9109                 struct mlx5_flow_dv_modify_hdr_resource res;
9110                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
9111                             sizeof(struct mlx5_modification_cmd) *
9112                             (MLX5_MAX_MODIFY_NUM + 1)];
9113         } mhdr_dummy;
9114         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
9115         const struct rte_flow_action_count *count = NULL;
9116         const struct rte_flow_action_age *age = NULL;
9117         union flow_dv_attr flow_attr = { .attr = 0 };
9118         uint32_t tag_be;
9119         union mlx5_flow_tbl_key tbl_key;
9120         uint32_t modify_action_position = UINT32_MAX;
9121         void *match_mask = matcher.mask.buf;
9122         void *match_value = dev_flow->dv.value.buf;
9123         uint8_t next_protocol = 0xff;
9124         struct rte_vlan_hdr vlan = { 0 };
9125         struct mlx5_flow_dv_dest_array_resource mdest_res;
9126         struct mlx5_flow_dv_sample_resource sample_res;
9127         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
9128         struct mlx5_flow_sub_actions_list *sample_act;
9129         uint32_t sample_act_pos = UINT32_MAX;
9130         uint32_t num_of_dest = 0;
9131         int tmp_actions_n = 0;
9132         uint32_t table;
9133         int ret = 0;
9134
9135         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
9136         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
9137         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
9138                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
9139         /* update normal path action resource into last index of array */
9140         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
9141         ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
9142                                        !!priv->fdb_def_rule, &table, error);
9143         if (ret)
9144                 return ret;
9145         dev_flow->dv.group = table;
9146         if (attr->transfer)
9147                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
9148         if (priority == MLX5_FLOW_PRIO_RSVD)
9149                 priority = dev_conf->flow_prio - 1;
9150         /* number of actions must be set to 0 in case of dirty stack. */
9151         mhdr_res->actions_num = 0;
9152         for (; !actions_end ; actions++) {
9153                 const struct rte_flow_action_queue *queue;
9154                 const struct rte_flow_action_rss *rss;
9155                 const struct rte_flow_action *action = actions;
9156                 const uint8_t *rss_key;
9157                 const struct rte_flow_action_meter *mtr;
9158                 struct mlx5_flow_tbl_resource *tbl;
9159                 uint32_t port_id = 0;
9160                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
9161                 int action_type = actions->type;
9162                 const struct rte_flow_action *found_action = NULL;
9163                 struct mlx5_flow_meter *fm = NULL;
9164                 uint32_t jump_group = 0;
9165
9166                 if (!mlx5_flow_os_action_supported(action_type))
9167                         return rte_flow_error_set(error, ENOTSUP,
9168                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9169                                                   actions,
9170                                                   "action not supported");
9171                 switch (action_type) {
9172                 case RTE_FLOW_ACTION_TYPE_VOID:
9173                         break;
9174                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
9175                         if (flow_dv_translate_action_port_id(dev, action,
9176                                                              &port_id, error))
9177                                 return -rte_errno;
9178                         port_id_resource.port_id = port_id;
9179                         MLX5_ASSERT(!handle->rix_port_id_action);
9180                         if (flow_dv_port_id_action_resource_register
9181                             (dev, &port_id_resource, dev_flow, error))
9182                                 return -rte_errno;
9183                         dev_flow->dv.actions[actions_n++] =
9184                                         dev_flow->dv.port_id_action->action;
9185                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9186                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
9187                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9188                         num_of_dest++;
9189                         break;
9190                 case RTE_FLOW_ACTION_TYPE_FLAG:
9191                         action_flags |= MLX5_FLOW_ACTION_FLAG;
9192                         dev_flow->handle->mark = 1;
9193                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
9194                                 struct rte_flow_action_mark mark = {
9195                                         .id = MLX5_FLOW_MARK_DEFAULT,
9196                                 };
9197
9198                                 if (flow_dv_convert_action_mark(dev, &mark,
9199                                                                 mhdr_res,
9200                                                                 error))
9201                                         return -rte_errno;
9202                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
9203                                 break;
9204                         }
9205                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
9206                         /*
9207                          * Only one FLAG or MARK is supported per device flow
9208                          * right now. So the pointer to the tag resource must be
9209                          * zero before the register process.
9210                          */
9211                         MLX5_ASSERT(!handle->dvh.rix_tag);
9212                         if (flow_dv_tag_resource_register(dev, tag_be,
9213                                                           dev_flow, error))
9214                                 return -rte_errno;
9215                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9216                         dev_flow->dv.actions[actions_n++] =
9217                                         dev_flow->dv.tag_resource->action;
9218                         break;
9219                 case RTE_FLOW_ACTION_TYPE_MARK:
9220                         action_flags |= MLX5_FLOW_ACTION_MARK;
9221                         dev_flow->handle->mark = 1;
9222                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
9223                                 const struct rte_flow_action_mark *mark =
9224                                         (const struct rte_flow_action_mark *)
9225                                                 actions->conf;
9226
9227                                 if (flow_dv_convert_action_mark(dev, mark,
9228                                                                 mhdr_res,
9229                                                                 error))
9230                                         return -rte_errno;
9231                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
9232                                 break;
9233                         }
9234                         /* Fall-through */
9235                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
9236                         /* Legacy (non-extensive) MARK action. */
9237                         tag_be = mlx5_flow_mark_set
9238                               (((const struct rte_flow_action_mark *)
9239                                (actions->conf))->id);
9240                         MLX5_ASSERT(!handle->dvh.rix_tag);
9241                         if (flow_dv_tag_resource_register(dev, tag_be,
9242                                                           dev_flow, error))
9243                                 return -rte_errno;
9244                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9245                         dev_flow->dv.actions[actions_n++] =
9246                                         dev_flow->dv.tag_resource->action;
9247                         break;
9248                 case RTE_FLOW_ACTION_TYPE_SET_META:
9249                         if (flow_dv_convert_action_set_meta
9250                                 (dev, mhdr_res, attr,
9251                                  (const struct rte_flow_action_set_meta *)
9252                                   actions->conf, error))
9253                                 return -rte_errno;
9254                         action_flags |= MLX5_FLOW_ACTION_SET_META;
9255                         break;
9256                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
9257                         if (flow_dv_convert_action_set_tag
9258                                 (dev, mhdr_res,
9259                                  (const struct rte_flow_action_set_tag *)
9260                                   actions->conf, error))
9261                                 return -rte_errno;
9262                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9263                         break;
9264                 case RTE_FLOW_ACTION_TYPE_DROP:
9265                         action_flags |= MLX5_FLOW_ACTION_DROP;
9266                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
9267                         break;
9268                 case RTE_FLOW_ACTION_TYPE_QUEUE:
9269                         queue = actions->conf;
9270                         rss_desc->queue_num = 1;
9271                         rss_desc->queue[0] = queue->index;
9272                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
9273                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9274                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
9275                         num_of_dest++;
9276                         break;
9277                 case RTE_FLOW_ACTION_TYPE_RSS:
9278                         rss = actions->conf;
9279                         memcpy(rss_desc->queue, rss->queue,
9280                                rss->queue_num * sizeof(uint16_t));
9281                         rss_desc->queue_num = rss->queue_num;
9282                         /* NULL RSS key indicates default RSS key. */
9283                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
9284                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
9285                         /*
9286                          * rss->level and rss.types should be set in advance
9287                          * when expanding items for RSS.
9288                          */
9289                         action_flags |= MLX5_FLOW_ACTION_RSS;
9290                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9291                         break;
9292                 case RTE_FLOW_ACTION_TYPE_AGE:
9293                 case RTE_FLOW_ACTION_TYPE_COUNT:
9294                         if (!dev_conf->devx) {
9295                                 return rte_flow_error_set
9296                                               (error, ENOTSUP,
9297                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9298                                                NULL,
9299                                                "count action not supported");
9300                         }
9301                         /* Save information first, will apply later. */
9302                         if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT)
9303                                 count = action->conf;
9304                         else
9305                                 age = action->conf;
9306                         action_flags |= MLX5_FLOW_ACTION_COUNT;
9307                         break;
9308                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
9309                         dev_flow->dv.actions[actions_n++] =
9310                                                 priv->sh->pop_vlan_action;
9311                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
9312                         break;
9313                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
9314                         if (!(action_flags &
9315                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
9316                                 flow_dev_get_vlan_info_from_items(items, &vlan);
9317                         vlan.eth_proto = rte_be_to_cpu_16
9318                              ((((const struct rte_flow_action_of_push_vlan *)
9319                                                    actions->conf)->ethertype));
9320                         found_action = mlx5_flow_find_action
9321                                         (actions + 1,
9322                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
9323                         if (found_action)
9324                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
9325                         found_action = mlx5_flow_find_action
9326                                         (actions + 1,
9327                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
9328                         if (found_action)
9329                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
9330                         if (flow_dv_create_action_push_vlan
9331                                             (dev, attr, &vlan, dev_flow, error))
9332                                 return -rte_errno;
9333                         dev_flow->dv.actions[actions_n++] =
9334                                         dev_flow->dv.push_vlan_res->action;
9335                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
9336                         break;
9337                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
9338                         /* of_vlan_push action handled this action */
9339                         MLX5_ASSERT(action_flags &
9340                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
9341                         break;
9342                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
9343                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
9344                                 break;
9345                         flow_dev_get_vlan_info_from_items(items, &vlan);
9346                         mlx5_update_vlan_vid_pcp(actions, &vlan);
9347                         /* If no VLAN push - this is a modify header action */
9348                         if (flow_dv_convert_action_modify_vlan_vid
9349                                                 (mhdr_res, actions, error))
9350                                 return -rte_errno;
9351                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
9352                         break;
9353                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
9354                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
9355                         if (flow_dv_create_action_l2_encap(dev, actions,
9356                                                            dev_flow,
9357                                                            attr->transfer,
9358                                                            error))
9359                                 return -rte_errno;
9360                         dev_flow->dv.actions[actions_n++] =
9361                                         dev_flow->dv.encap_decap->action;
9362                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9363                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
9364                                 sample_act->action_flags |=
9365                                                         MLX5_FLOW_ACTION_ENCAP;
9366                         break;
9367                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
9368                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
9369                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
9370                                                            attr->transfer,
9371                                                            error))
9372                                 return -rte_errno;
9373                         dev_flow->dv.actions[actions_n++] =
9374                                         dev_flow->dv.encap_decap->action;
9375                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9376                         break;
9377                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
9378                         /* Handle encap with preceding decap. */
9379                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
9380                                 if (flow_dv_create_action_raw_encap
9381                                         (dev, actions, dev_flow, attr, error))
9382                                         return -rte_errno;
9383                                 dev_flow->dv.actions[actions_n++] =
9384                                         dev_flow->dv.encap_decap->action;
9385                         } else {
9386                                 /* Handle encap without preceding decap. */
9387                                 if (flow_dv_create_action_l2_encap
9388                                     (dev, actions, dev_flow, attr->transfer,
9389                                      error))
9390                                         return -rte_errno;
9391                                 dev_flow->dv.actions[actions_n++] =
9392                                         dev_flow->dv.encap_decap->action;
9393                         }
9394                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9395                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
9396                                 sample_act->action_flags |=
9397                                                         MLX5_FLOW_ACTION_ENCAP;
9398                         break;
9399                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
9400                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
9401                                 ;
9402                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
9403                                 if (flow_dv_create_action_l2_decap
9404                                     (dev, dev_flow, attr->transfer, error))
9405                                         return -rte_errno;
9406                                 dev_flow->dv.actions[actions_n++] =
9407                                         dev_flow->dv.encap_decap->action;
9408                         }
9409                         /* If decap is followed by encap, handle it at encap. */
9410                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9411                         break;
9412                 case RTE_FLOW_ACTION_TYPE_JUMP:
9413                         jump_group = ((const struct rte_flow_action_jump *)
9414                                                         action->conf)->group;
9415                         if (dev_flow->external && jump_group <
9416                                         MLX5_MAX_TABLES_EXTERNAL)
9417                                 jump_group *= MLX5_FLOW_TABLE_FACTOR;
9418                         ret = mlx5_flow_group_to_table(attr, dev_flow->external,
9419                                                        jump_group,
9420                                                        !!priv->fdb_def_rule,
9421                                                        &table, error);
9422                         if (ret)
9423                                 return ret;
9424                         tbl = flow_dv_tbl_resource_get(dev, table,
9425                                                        attr->egress,
9426                                                        attr->transfer, error);
9427                         if (!tbl)
9428                                 return rte_flow_error_set
9429                                                 (error, errno,
9430                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9431                                                  NULL,
9432                                                  "cannot create jump action.");
9433                         if (flow_dv_jump_tbl_resource_register
9434                             (dev, tbl, dev_flow, error)) {
9435                                 flow_dv_tbl_resource_release(dev, tbl);
9436                                 return rte_flow_error_set
9437                                                 (error, errno,
9438                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9439                                                  NULL,
9440                                                  "cannot create jump action.");
9441                         }
9442                         dev_flow->dv.actions[actions_n++] =
9443                                         dev_flow->dv.jump->action;
9444                         action_flags |= MLX5_FLOW_ACTION_JUMP;
9445                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
9446                         break;
9447                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
9448                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
9449                         if (flow_dv_convert_action_modify_mac
9450                                         (mhdr_res, actions, error))
9451                                 return -rte_errno;
9452                         action_flags |= actions->type ==
9453                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
9454                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
9455                                         MLX5_FLOW_ACTION_SET_MAC_DST;
9456                         break;
9457                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
9458                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
9459                         if (flow_dv_convert_action_modify_ipv4
9460                                         (mhdr_res, actions, error))
9461                                 return -rte_errno;
9462                         action_flags |= actions->type ==
9463                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
9464                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
9465                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
9466                         break;
9467                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
9468                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
9469                         if (flow_dv_convert_action_modify_ipv6
9470                                         (mhdr_res, actions, error))
9471                                 return -rte_errno;
9472                         action_flags |= actions->type ==
9473                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
9474                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
9475                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
9476                         break;
9477                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
9478                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
9479                         if (flow_dv_convert_action_modify_tp
9480                                         (mhdr_res, actions, items,
9481                                          &flow_attr, dev_flow, !!(action_flags &
9482                                          MLX5_FLOW_ACTION_DECAP), error))
9483                                 return -rte_errno;
9484                         action_flags |= actions->type ==
9485                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
9486                                         MLX5_FLOW_ACTION_SET_TP_SRC :
9487                                         MLX5_FLOW_ACTION_SET_TP_DST;
9488                         break;
9489                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
9490                         if (flow_dv_convert_action_modify_dec_ttl
9491                                         (mhdr_res, items, &flow_attr, dev_flow,
9492                                          !!(action_flags &
9493                                          MLX5_FLOW_ACTION_DECAP), error))
9494                                 return -rte_errno;
9495                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
9496                         break;
9497                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
9498                         if (flow_dv_convert_action_modify_ttl
9499                                         (mhdr_res, actions, items, &flow_attr,
9500                                          dev_flow, !!(action_flags &
9501                                          MLX5_FLOW_ACTION_DECAP), error))
9502                                 return -rte_errno;
9503                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
9504                         break;
9505                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
9506                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
9507                         if (flow_dv_convert_action_modify_tcp_seq
9508                                         (mhdr_res, actions, error))
9509                                 return -rte_errno;
9510                         action_flags |= actions->type ==
9511                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
9512                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
9513                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
9514                         break;
9515
9516                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
9517                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
9518                         if (flow_dv_convert_action_modify_tcp_ack
9519                                         (mhdr_res, actions, error))
9520                                 return -rte_errno;
9521                         action_flags |= actions->type ==
9522                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
9523                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
9524                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
9525                         break;
9526                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
9527                         if (flow_dv_convert_action_set_reg
9528                                         (mhdr_res, actions, error))
9529                                 return -rte_errno;
9530                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9531                         break;
9532                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
9533                         if (flow_dv_convert_action_copy_mreg
9534                                         (dev, mhdr_res, actions, error))
9535                                 return -rte_errno;
9536                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9537                         break;
9538                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
9539                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
9540                         dev_flow->handle->fate_action =
9541                                         MLX5_FLOW_FATE_DEFAULT_MISS;
9542                         break;
9543                 case RTE_FLOW_ACTION_TYPE_METER:
9544                         mtr = actions->conf;
9545                         if (!flow->meter) {
9546                                 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
9547                                                             attr, error);
9548                                 if (!fm)
9549                                         return rte_flow_error_set(error,
9550                                                 rte_errno,
9551                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9552                                                 NULL,
9553                                                 "meter not found "
9554                                                 "or invalid parameters");
9555                                 flow->meter = fm->idx;
9556                         }
9557                         /* Set the meter action. */
9558                         if (!fm) {
9559                                 fm = mlx5_ipool_get(priv->sh->ipool
9560                                                 [MLX5_IPOOL_MTR], flow->meter);
9561                                 if (!fm)
9562                                         return rte_flow_error_set(error,
9563                                                 rte_errno,
9564                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9565                                                 NULL,
9566                                                 "meter not found "
9567                                                 "or invalid parameters");
9568                         }
9569                         dev_flow->dv.actions[actions_n++] =
9570                                 fm->mfts->meter_action;
9571                         action_flags |= MLX5_FLOW_ACTION_METER;
9572                         break;
9573                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
9574                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
9575                                                               actions, error))
9576                                 return -rte_errno;
9577                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
9578                         break;
9579                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
9580                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
9581                                                               actions, error))
9582                                 return -rte_errno;
9583                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
9584                         break;
9585                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
9586                         sample_act_pos = actions_n;
9587                         ret = flow_dv_translate_action_sample(dev,
9588                                                               actions,
9589                                                               dev_flow, attr,
9590                                                               &num_of_dest,
9591                                                               sample_actions,
9592                                                               &sample_res,
9593                                                               error);
9594                         if (ret < 0)
9595                                 return ret;
9596                         actions_n++;
9597                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
9598                         /* put encap action into group if work with port id */
9599                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
9600                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
9601                                 sample_act->action_flags |=
9602                                                         MLX5_FLOW_ACTION_ENCAP;
9603                         break;
9604                 case RTE_FLOW_ACTION_TYPE_END:
9605                         actions_end = true;
9606                         if (mhdr_res->actions_num) {
9607                                 /* create modify action if needed. */
9608                                 if (flow_dv_modify_hdr_resource_register
9609                                         (dev, mhdr_res, dev_flow, error))
9610                                         return -rte_errno;
9611                                 dev_flow->dv.actions[modify_action_position] =
9612                                         handle->dvh.modify_hdr->action;
9613                         }
9614                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
9615                                 flow->counter =
9616                                         flow_dv_translate_create_counter(dev,
9617                                                 dev_flow, count, age);
9618
9619                                 if (!flow->counter)
9620                                         return rte_flow_error_set
9621                                                 (error, rte_errno,
9622                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9623                                                 NULL,
9624                                                 "cannot create counter"
9625                                                 " object.");
9626                                 dev_flow->dv.actions[actions_n] =
9627                                           (flow_dv_counter_get_by_idx(dev,
9628                                           flow->counter, NULL))->action;
9629                                 actions_n++;
9630                         }
9631                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
9632                                 ret = flow_dv_create_action_sample(dev,
9633                                                           dev_flow, attr,
9634                                                           num_of_dest,
9635                                                           &sample_res,
9636                                                           &mdest_res,
9637                                                           sample_actions,
9638                                                           action_flags,
9639                                                           error);
9640                                 if (ret < 0)
9641                                         return rte_flow_error_set
9642                                                 (error, rte_errno,
9643                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9644                                                 NULL,
9645                                                 "cannot create sample action");
9646                                 if (num_of_dest > 1) {
9647                                         dev_flow->dv.actions[sample_act_pos] =
9648                                         dev_flow->dv.dest_array_res->action;
9649                                 } else {
9650                                         dev_flow->dv.actions[sample_act_pos] =
9651                                         dev_flow->dv.sample_res->verbs_action;
9652                                 }
9653                         }
9654                         break;
9655                 default:
9656                         break;
9657                 }
9658                 if (mhdr_res->actions_num &&
9659                     modify_action_position == UINT32_MAX)
9660                         modify_action_position = actions_n++;
9661         }
9662         /*
9663          * For multiple destination (sample action with ratio=1), the encap
9664          * action and port id action will be combined into group action.
9665          * So need remove the original these actions in the flow and only
9666          * use the sample action instead of.
9667          */
9668         if (num_of_dest > 1 && sample_act->dr_port_id_action) {
9669                 int i;
9670                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
9671
9672                 for (i = 0; i < actions_n; i++) {
9673                         if ((sample_act->dr_encap_action &&
9674                                 sample_act->dr_encap_action ==
9675                                 dev_flow->dv.actions[i]) ||
9676                                 (sample_act->dr_port_id_action &&
9677                                 sample_act->dr_port_id_action ==
9678                                 dev_flow->dv.actions[i]))
9679                                 continue;
9680                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
9681                 }
9682                 memcpy((void *)dev_flow->dv.actions,
9683                                 (void *)temp_actions,
9684                                 tmp_actions_n * sizeof(void *));
9685                 actions_n = tmp_actions_n;
9686         }
9687         dev_flow->dv.actions_n = actions_n;
9688         dev_flow->act_flags = action_flags;
9689         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
9690                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
9691                 int item_type = items->type;
9692
9693                 if (!mlx5_flow_os_item_supported(item_type))
9694                         return rte_flow_error_set(error, ENOTSUP,
9695                                                   RTE_FLOW_ERROR_TYPE_ITEM,
9696                                                   NULL, "item not supported");
9697                 switch (item_type) {
9698                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
9699                         flow_dv_translate_item_port_id(dev, match_mask,
9700                                                        match_value, items);
9701                         last_item = MLX5_FLOW_ITEM_PORT_ID;
9702                         break;
9703                 case RTE_FLOW_ITEM_TYPE_ETH:
9704                         flow_dv_translate_item_eth(match_mask, match_value,
9705                                                    items, tunnel,
9706                                                    dev_flow->dv.group);
9707                         matcher.priority = action_flags &
9708                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
9709                                         !dev_flow->external ?
9710                                         MLX5_PRIORITY_MAP_L3 :
9711                                         MLX5_PRIORITY_MAP_L2;
9712                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
9713                                              MLX5_FLOW_LAYER_OUTER_L2;
9714                         break;
9715                 case RTE_FLOW_ITEM_TYPE_VLAN:
9716                         flow_dv_translate_item_vlan(dev_flow,
9717                                                     match_mask, match_value,
9718                                                     items, tunnel,
9719                                                     dev_flow->dv.group);
9720                         matcher.priority = MLX5_PRIORITY_MAP_L2;
9721                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
9722                                               MLX5_FLOW_LAYER_INNER_VLAN) :
9723                                              (MLX5_FLOW_LAYER_OUTER_L2 |
9724                                               MLX5_FLOW_LAYER_OUTER_VLAN);
9725                         break;
9726                 case RTE_FLOW_ITEM_TYPE_IPV4:
9727                         mlx5_flow_tunnel_ip_check(items, next_protocol,
9728                                                   &item_flags, &tunnel);
9729                         flow_dv_translate_item_ipv4(match_mask, match_value,
9730                                                     items, item_flags, tunnel,
9731                                                     dev_flow->dv.group);
9732                         matcher.priority = MLX5_PRIORITY_MAP_L3;
9733                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
9734                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
9735                         if (items->mask != NULL &&
9736                             ((const struct rte_flow_item_ipv4 *)
9737                              items->mask)->hdr.next_proto_id) {
9738                                 next_protocol =
9739                                         ((const struct rte_flow_item_ipv4 *)
9740                                          (items->spec))->hdr.next_proto_id;
9741                                 next_protocol &=
9742                                         ((const struct rte_flow_item_ipv4 *)
9743                                          (items->mask))->hdr.next_proto_id;
9744                         } else {
9745                                 /* Reset for inner layer. */
9746                                 next_protocol = 0xff;
9747                         }
9748                         break;
9749                 case RTE_FLOW_ITEM_TYPE_IPV6:
9750                         mlx5_flow_tunnel_ip_check(items, next_protocol,
9751                                                   &item_flags, &tunnel);
9752                         flow_dv_translate_item_ipv6(match_mask, match_value,
9753                                                     items, item_flags, tunnel,
9754                                                     dev_flow->dv.group);
9755                         matcher.priority = MLX5_PRIORITY_MAP_L3;
9756                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
9757                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
9758                         if (items->mask != NULL &&
9759                             ((const struct rte_flow_item_ipv6 *)
9760                              items->mask)->hdr.proto) {
9761                                 next_protocol =
9762                                         ((const struct rte_flow_item_ipv6 *)
9763                                          items->spec)->hdr.proto;
9764                                 next_protocol &=
9765                                         ((const struct rte_flow_item_ipv6 *)
9766                                          items->mask)->hdr.proto;
9767                         } else {
9768                                 /* Reset for inner layer. */
9769                                 next_protocol = 0xff;
9770                         }
9771                         break;
9772                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
9773                         flow_dv_translate_item_ipv6_frag_ext(match_mask,
9774                                                              match_value,
9775                                                              items, tunnel);
9776                         last_item = tunnel ?
9777                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
9778                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
9779                         if (items->mask != NULL &&
9780                             ((const struct rte_flow_item_ipv6_frag_ext *)
9781                              items->mask)->hdr.next_header) {
9782                                 next_protocol =
9783                                 ((const struct rte_flow_item_ipv6_frag_ext *)
9784                                  items->spec)->hdr.next_header;
9785                                 next_protocol &=
9786                                 ((const struct rte_flow_item_ipv6_frag_ext *)
9787                                  items->mask)->hdr.next_header;
9788                         } else {
9789                                 /* Reset for inner layer. */
9790                                 next_protocol = 0xff;
9791                         }
9792                         break;
9793                 case RTE_FLOW_ITEM_TYPE_TCP:
9794                         flow_dv_translate_item_tcp(match_mask, match_value,
9795                                                    items, tunnel);
9796                         matcher.priority = MLX5_PRIORITY_MAP_L4;
9797                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
9798                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
9799                         break;
9800                 case RTE_FLOW_ITEM_TYPE_UDP:
9801                         flow_dv_translate_item_udp(match_mask, match_value,
9802                                                    items, tunnel);
9803                         matcher.priority = MLX5_PRIORITY_MAP_L4;
9804                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
9805                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
9806                         break;
9807                 case RTE_FLOW_ITEM_TYPE_GRE:
9808                         flow_dv_translate_item_gre(match_mask, match_value,
9809                                                    items, tunnel);
9810                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9811                         last_item = MLX5_FLOW_LAYER_GRE;
9812                         break;
9813                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
9814                         flow_dv_translate_item_gre_key(match_mask,
9815                                                        match_value, items);
9816                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
9817                         break;
9818                 case RTE_FLOW_ITEM_TYPE_NVGRE:
9819                         flow_dv_translate_item_nvgre(match_mask, match_value,
9820                                                      items, tunnel);
9821                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9822                         last_item = MLX5_FLOW_LAYER_GRE;
9823                         break;
9824                 case RTE_FLOW_ITEM_TYPE_VXLAN:
9825                         flow_dv_translate_item_vxlan(match_mask, match_value,
9826                                                      items, tunnel);
9827                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9828                         last_item = MLX5_FLOW_LAYER_VXLAN;
9829                         break;
9830                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
9831                         flow_dv_translate_item_vxlan_gpe(match_mask,
9832                                                          match_value, items,
9833                                                          tunnel);
9834                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9835                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
9836                         break;
9837                 case RTE_FLOW_ITEM_TYPE_GENEVE:
9838                         flow_dv_translate_item_geneve(match_mask, match_value,
9839                                                       items, tunnel);
9840                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9841                         last_item = MLX5_FLOW_LAYER_GENEVE;
9842                         break;
9843                 case RTE_FLOW_ITEM_TYPE_MPLS:
9844                         flow_dv_translate_item_mpls(match_mask, match_value,
9845                                                     items, last_item, tunnel);
9846                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9847                         last_item = MLX5_FLOW_LAYER_MPLS;
9848                         break;
9849                 case RTE_FLOW_ITEM_TYPE_MARK:
9850                         flow_dv_translate_item_mark(dev, match_mask,
9851                                                     match_value, items);
9852                         last_item = MLX5_FLOW_ITEM_MARK;
9853                         break;
9854                 case RTE_FLOW_ITEM_TYPE_META:
9855                         flow_dv_translate_item_meta(dev, match_mask,
9856                                                     match_value, attr, items);
9857                         last_item = MLX5_FLOW_ITEM_METADATA;
9858                         break;
9859                 case RTE_FLOW_ITEM_TYPE_ICMP:
9860                         flow_dv_translate_item_icmp(match_mask, match_value,
9861                                                     items, tunnel);
9862                         last_item = MLX5_FLOW_LAYER_ICMP;
9863                         break;
9864                 case RTE_FLOW_ITEM_TYPE_ICMP6:
9865                         flow_dv_translate_item_icmp6(match_mask, match_value,
9866                                                       items, tunnel);
9867                         last_item = MLX5_FLOW_LAYER_ICMP6;
9868                         break;
9869                 case RTE_FLOW_ITEM_TYPE_TAG:
9870                         flow_dv_translate_item_tag(dev, match_mask,
9871                                                    match_value, items);
9872                         last_item = MLX5_FLOW_ITEM_TAG;
9873                         break;
9874                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
9875                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
9876                                                         match_value, items);
9877                         last_item = MLX5_FLOW_ITEM_TAG;
9878                         break;
9879                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
9880                         flow_dv_translate_item_tx_queue(dev, match_mask,
9881                                                         match_value,
9882                                                         items);
9883                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
9884                         break;
9885                 case RTE_FLOW_ITEM_TYPE_GTP:
9886                         flow_dv_translate_item_gtp(match_mask, match_value,
9887                                                    items, tunnel);
9888                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9889                         last_item = MLX5_FLOW_LAYER_GTP;
9890                         break;
9891                 case RTE_FLOW_ITEM_TYPE_ECPRI:
9892                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
9893                                 /* Create it only the first time to be used. */
9894                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
9895                                 if (ret)
9896                                         return rte_flow_error_set
9897                                                 (error, -ret,
9898                                                 RTE_FLOW_ERROR_TYPE_ITEM,
9899                                                 NULL,
9900                                                 "cannot create eCPRI parser");
9901                         }
9902                         /* Adjust the length matcher and device flow value. */
9903                         matcher.mask.size = MLX5_ST_SZ_BYTES(fte_match_param);
9904                         dev_flow->dv.value.size =
9905                                         MLX5_ST_SZ_BYTES(fte_match_param);
9906                         flow_dv_translate_item_ecpri(dev, match_mask,
9907                                                      match_value, items);
9908                         /* No other protocol should follow eCPRI layer. */
9909                         last_item = MLX5_FLOW_LAYER_ECPRI;
9910                         break;
9911                 default:
9912                         break;
9913                 }
9914                 item_flags |= last_item;
9915         }
9916         /*
9917          * When E-Switch mode is enabled, we have two cases where we need to
9918          * set the source port manually.
9919          * The first one, is in case of Nic steering rule, and the second is
9920          * E-Switch rule where no port_id item was found. In both cases
9921          * the source port is set according the current port in use.
9922          */
9923         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
9924             (priv->representor || priv->master)) {
9925                 if (flow_dv_translate_item_port_id(dev, match_mask,
9926                                                    match_value, NULL))
9927                         return -rte_errno;
9928         }
9929 #ifdef RTE_LIBRTE_MLX5_DEBUG
9930         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
9931                                               dev_flow->dv.value.buf));
9932 #endif
9933         /*
9934          * Layers may be already initialized from prefix flow if this dev_flow
9935          * is the suffix flow.
9936          */
9937         handle->layers |= item_flags;
9938         if (action_flags & MLX5_FLOW_ACTION_RSS)
9939                 flow_dv_hashfields_set(dev_flow, rss_desc);
9940         /* Register matcher. */
9941         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
9942                                     matcher.mask.size);
9943         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
9944                                                      matcher.priority);
9945         /* reserved field no needs to be set to 0 here. */
9946         tbl_key.domain = attr->transfer;
9947         tbl_key.direction = attr->egress;
9948         tbl_key.table_id = dev_flow->dv.group;
9949         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
9950                 return -rte_errno;
9951         return 0;
9952 }
9953
9954 /**
9955  * Apply the flow to the NIC, lock free,
9956  * (mutex should be acquired by caller).
9957  *
9958  * @param[in] dev
9959  *   Pointer to the Ethernet device structure.
9960  * @param[in, out] flow
9961  *   Pointer to flow structure.
9962  * @param[out] error
9963  *   Pointer to error structure.
9964  *
9965  * @return
9966  *   0 on success, a negative errno value otherwise and rte_errno is set.
9967  */
9968 static int
9969 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
9970                 struct rte_flow_error *error)
9971 {
9972         struct mlx5_flow_dv_workspace *dv;
9973         struct mlx5_flow_handle *dh;
9974         struct mlx5_flow_handle_dv *dv_h;
9975         struct mlx5_flow *dev_flow;
9976         struct mlx5_priv *priv = dev->data->dev_private;
9977         uint32_t handle_idx;
9978         int n;
9979         int err;
9980         int idx;
9981
9982         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
9983                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
9984                 dv = &dev_flow->dv;
9985                 dh = dev_flow->handle;
9986                 dv_h = &dh->dvh;
9987                 n = dv->actions_n;
9988                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
9989                         if (dv->transfer) {
9990                                 dv->actions[n++] = priv->sh->esw_drop_action;
9991                         } else {
9992                                 struct mlx5_hrxq *drop_hrxq;
9993                                 drop_hrxq = mlx5_drop_action_create(dev);
9994                                 if (!drop_hrxq) {
9995                                         rte_flow_error_set
9996                                                 (error, errno,
9997                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9998                                                  NULL,
9999                                                  "cannot get drop hash queue");
10000                                         goto error;
10001                                 }
10002                                 /*
10003                                  * Drop queues will be released by the specify
10004                                  * mlx5_drop_action_destroy() function. Assign
10005                                  * the special index to hrxq to mark the queue
10006                                  * has been allocated.
10007                                  */
10008                                 dh->rix_hrxq = UINT32_MAX;
10009                                 dv->actions[n++] = drop_hrxq->action;
10010                         }
10011                 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
10012                            !dv_h->rix_sample && !dv_h->rix_dest_array) {
10013                         struct mlx5_hrxq *hrxq;
10014                         uint32_t hrxq_idx;
10015                         struct mlx5_flow_rss_desc *rss_desc =
10016                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
10017                                 [!!priv->flow_nested_idx];
10018
10019                         MLX5_ASSERT(rss_desc->queue_num);
10020                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
10021                                                  MLX5_RSS_HASH_KEY_LEN,
10022                                                  dev_flow->hash_fields,
10023                                                  rss_desc->queue,
10024                                                  rss_desc->queue_num);
10025                         if (!hrxq_idx) {
10026                                 hrxq_idx = mlx5_hrxq_new
10027                                                 (dev, rss_desc->key,
10028                                                  MLX5_RSS_HASH_KEY_LEN,
10029                                                  dev_flow->hash_fields,
10030                                                  rss_desc->queue,
10031                                                  rss_desc->queue_num,
10032                                                  !!(dh->layers &
10033                                                  MLX5_FLOW_LAYER_TUNNEL));
10034                         }
10035                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
10036                                               hrxq_idx);
10037                         if (!hrxq) {
10038                                 rte_flow_error_set
10039                                         (error, rte_errno,
10040                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10041                                          "cannot get hash queue");
10042                                 goto error;
10043                         }
10044                         dh->rix_hrxq = hrxq_idx;
10045                         dv->actions[n++] = hrxq->action;
10046                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
10047                         if (flow_dv_default_miss_resource_register
10048                                         (dev, error)) {
10049                                 rte_flow_error_set
10050                                         (error, rte_errno,
10051                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10052                                          "cannot create default miss resource");
10053                                 goto error_default_miss;
10054                         }
10055                         dh->rix_default_fate =  MLX5_FLOW_FATE_DEFAULT_MISS;
10056                         dv->actions[n++] = priv->sh->default_miss.action;
10057                 }
10058                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
10059                                                (void *)&dv->value, n,
10060                                                dv->actions, &dh->drv_flow);
10061                 if (err) {
10062                         rte_flow_error_set(error, errno,
10063                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10064                                            NULL,
10065                                            "hardware refuses to create flow");
10066                         goto error;
10067                 }
10068                 if (priv->vmwa_context &&
10069                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
10070                         /*
10071                          * The rule contains the VLAN pattern.
10072                          * For VF we are going to create VLAN
10073                          * interface to make hypervisor set correct
10074                          * e-Switch vport context.
10075                          */
10076                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
10077                 }
10078         }
10079         return 0;
10080 error:
10081         if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
10082                 flow_dv_default_miss_resource_release(dev);
10083 error_default_miss:
10084         err = rte_errno; /* Save rte_errno before cleanup. */
10085         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
10086                        handle_idx, dh, next) {
10087                 /* hrxq is union, don't clear it if the flag is not set. */
10088                 if (dh->rix_hrxq) {
10089                         if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
10090                                 mlx5_drop_action_destroy(dev);
10091                                 dh->rix_hrxq = 0;
10092                         } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
10093                                 mlx5_hrxq_release(dev, dh->rix_hrxq);
10094                                 dh->rix_hrxq = 0;
10095                         }
10096                 }
10097                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
10098                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
10099         }
10100         rte_errno = err; /* Restore rte_errno. */
10101         return -rte_errno;
10102 }
10103
10104 /**
10105  * Release the flow matcher.
10106  *
10107  * @param dev
10108  *   Pointer to Ethernet device.
10109  * @param handle
10110  *   Pointer to mlx5_flow_handle.
10111  *
10112  * @return
10113  *   1 while a reference on it exists, 0 when freed.
10114  */
10115 static int
10116 flow_dv_matcher_release(struct rte_eth_dev *dev,
10117                         struct mlx5_flow_handle *handle)
10118 {
10119         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
10120
10121         MLX5_ASSERT(matcher->matcher_object);
10122         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
10123                 dev->data->port_id, (void *)matcher,
10124                 rte_atomic32_read(&matcher->refcnt));
10125         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
10126                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10127                            (matcher->matcher_object));
10128                 LIST_REMOVE(matcher, next);
10129                 /* table ref-- in release interface. */
10130                 flow_dv_tbl_resource_release(dev, matcher->tbl);
10131                 mlx5_free(matcher);
10132                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
10133                         dev->data->port_id, (void *)matcher);
10134                 return 0;
10135         }
10136         return 1;
10137 }
10138
10139 /**
10140  * Release an encap/decap resource.
10141  *
10142  * @param dev
10143  *   Pointer to Ethernet device.
10144  * @param encap_decap_idx
10145  *   Index of encap decap resource.
10146  *
10147  * @return
10148  *   1 while a reference on it exists, 0 when freed.
10149  */
10150 static int
10151 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
10152                                      uint32_t encap_decap_idx)
10153 {
10154         struct mlx5_priv *priv = dev->data->dev_private;
10155         uint32_t idx = encap_decap_idx;
10156         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
10157
10158         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
10159                          idx);
10160         if (!cache_resource)
10161                 return 0;
10162         MLX5_ASSERT(cache_resource->action);
10163         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
10164                 (void *)cache_resource,
10165                 rte_atomic32_read(&cache_resource->refcnt));
10166         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10167                 claim_zero(mlx5_flow_os_destroy_flow_action
10168                                                 (cache_resource->action));
10169                 mlx5_hlist_remove(priv->sh->encaps_decaps,
10170                                   &cache_resource->entry);
10171                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
10172                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
10173                         (void *)cache_resource);
10174                 return 0;
10175         }
10176         return 1;
10177 }
10178
10179 /**
10180  * Release an jump to table action resource.
10181  *
10182  * @param dev
10183  *   Pointer to Ethernet device.
10184  * @param handle
10185  *   Pointer to mlx5_flow_handle.
10186  *
10187  * @return
10188  *   1 while a reference on it exists, 0 when freed.
10189  */
10190 static int
10191 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
10192                                   struct mlx5_flow_handle *handle)
10193 {
10194         struct mlx5_priv *priv = dev->data->dev_private;
10195         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
10196         struct mlx5_flow_tbl_data_entry *tbl_data;
10197
10198         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
10199                              handle->rix_jump);
10200         if (!tbl_data)
10201                 return 0;
10202         cache_resource = &tbl_data->jump;
10203         MLX5_ASSERT(cache_resource->action);
10204         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
10205                 (void *)cache_resource,
10206                 rte_atomic32_read(&cache_resource->refcnt));
10207         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10208                 claim_zero(mlx5_flow_os_destroy_flow_action
10209                                                 (cache_resource->action));
10210                 /* jump action memory free is inside the table release. */
10211                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
10212                 DRV_LOG(DEBUG, "jump table resource %p: removed",
10213                         (void *)cache_resource);
10214                 return 0;
10215         }
10216         return 1;
10217 }
10218
10219 /**
10220  * Release a default miss resource.
10221  *
10222  * @param dev
10223  *   Pointer to Ethernet device.
10224  * @return
10225  *   1 while a reference on it exists, 0 when freed.
10226  */
10227 static int
10228 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
10229 {
10230         struct mlx5_priv *priv = dev->data->dev_private;
10231         struct mlx5_dev_ctx_shared *sh = priv->sh;
10232         struct mlx5_flow_default_miss_resource *cache_resource =
10233                         &sh->default_miss;
10234
10235         MLX5_ASSERT(cache_resource->action);
10236         DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--",
10237                         (void *)cache_resource->action,
10238                         rte_atomic32_read(&cache_resource->refcnt));
10239         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10240                 claim_zero(mlx5_glue->destroy_flow_action
10241                                 (cache_resource->action));
10242                 DRV_LOG(DEBUG, "default miss resource %p: removed",
10243                                 (void *)cache_resource->action);
10244                 return 0;
10245         }
10246         return 1;
10247 }
10248
10249 /**
10250  * Release a modify-header resource.
10251  *
10252  * @param dev
10253  *   Pointer to Ethernet device.
10254  * @param handle
10255  *   Pointer to mlx5_flow_handle.
10256  *
10257  * @return
10258  *   1 while a reference on it exists, 0 when freed.
10259  */
10260 static int
10261 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
10262                                     struct mlx5_flow_handle *handle)
10263 {
10264         struct mlx5_priv *priv = dev->data->dev_private;
10265         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
10266                                                         handle->dvh.modify_hdr;
10267
10268         MLX5_ASSERT(cache_resource->action);
10269         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
10270                 (void *)cache_resource,
10271                 rte_atomic32_read(&cache_resource->refcnt));
10272         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10273                 claim_zero(mlx5_flow_os_destroy_flow_action
10274                                                 (cache_resource->action));
10275                 mlx5_hlist_remove(priv->sh->modify_cmds,
10276                                   &cache_resource->entry);
10277                 mlx5_free(cache_resource);
10278                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
10279                         (void *)cache_resource);
10280                 return 0;
10281         }
10282         return 1;
10283 }
10284
10285 /**
10286  * Release port ID action resource.
10287  *
10288  * @param dev
10289  *   Pointer to Ethernet device.
10290  * @param handle
10291  *   Pointer to mlx5_flow_handle.
10292  *
10293  * @return
10294  *   1 while a reference on it exists, 0 when freed.
10295  */
10296 static int
10297 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
10298                                         uint32_t port_id)
10299 {
10300         struct mlx5_priv *priv = dev->data->dev_private;
10301         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
10302         uint32_t idx = port_id;
10303
10304         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
10305                                         idx);
10306         if (!cache_resource)
10307                 return 0;
10308         MLX5_ASSERT(cache_resource->action);
10309         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
10310                 (void *)cache_resource,
10311                 rte_atomic32_read(&cache_resource->refcnt));
10312         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10313                 claim_zero(mlx5_flow_os_destroy_flow_action
10314                                                 (cache_resource->action));
10315                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
10316                              &priv->sh->port_id_action_list, idx,
10317                              cache_resource, next);
10318                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx);
10319                 DRV_LOG(DEBUG, "port id action resource %p: removed",
10320                         (void *)cache_resource);
10321                 return 0;
10322         }
10323         return 1;
10324 }
10325
10326 /**
10327  * Release push vlan action resource.
10328  *
10329  * @param dev
10330  *   Pointer to Ethernet device.
10331  * @param handle
10332  *   Pointer to mlx5_flow_handle.
10333  *
10334  * @return
10335  *   1 while a reference on it exists, 0 when freed.
10336  */
10337 static int
10338 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
10339                                           struct mlx5_flow_handle *handle)
10340 {
10341         struct mlx5_priv *priv = dev->data->dev_private;
10342         uint32_t idx = handle->dvh.rix_push_vlan;
10343         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
10344
10345         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
10346                                         idx);
10347         if (!cache_resource)
10348                 return 0;
10349         MLX5_ASSERT(cache_resource->action);
10350         DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
10351                 (void *)cache_resource,
10352                 rte_atomic32_read(&cache_resource->refcnt));
10353         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10354                 claim_zero(mlx5_flow_os_destroy_flow_action
10355                                                 (cache_resource->action));
10356                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
10357                              &priv->sh->push_vlan_action_list, idx,
10358                              cache_resource, next);
10359                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
10360                 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
10361                         (void *)cache_resource);
10362                 return 0;
10363         }
10364         return 1;
10365 }
10366
10367 /**
10368  * Release the fate resource.
10369  *
10370  * @param dev
10371  *   Pointer to Ethernet device.
10372  * @param handle
10373  *   Pointer to mlx5_flow_handle.
10374  */
10375 static void
10376 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
10377                                struct mlx5_flow_handle *handle)
10378 {
10379         if (!handle->rix_fate)
10380                 return;
10381         switch (handle->fate_action) {
10382         case MLX5_FLOW_FATE_DROP:
10383                 mlx5_drop_action_destroy(dev);
10384                 break;
10385         case MLX5_FLOW_FATE_QUEUE:
10386                 mlx5_hrxq_release(dev, handle->rix_hrxq);
10387                 break;
10388         case MLX5_FLOW_FATE_JUMP:
10389                 flow_dv_jump_tbl_resource_release(dev, handle);
10390                 break;
10391         case MLX5_FLOW_FATE_PORT_ID:
10392                 flow_dv_port_id_action_resource_release(dev,
10393                                 handle->rix_port_id_action);
10394                 break;
10395         case MLX5_FLOW_FATE_DEFAULT_MISS:
10396                 flow_dv_default_miss_resource_release(dev);
10397                 break;
10398         default:
10399                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
10400                 break;
10401         }
10402         handle->rix_fate = 0;
10403 }
10404
10405 /**
10406  * Release an sample resource.
10407  *
10408  * @param dev
10409  *   Pointer to Ethernet device.
10410  * @param handle
10411  *   Pointer to mlx5_flow_handle.
10412  *
10413  * @return
10414  *   1 while a reference on it exists, 0 when freed.
10415  */
10416 static int
10417 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
10418                                      struct mlx5_flow_handle *handle)
10419 {
10420         struct mlx5_priv *priv = dev->data->dev_private;
10421         uint32_t idx = handle->dvh.rix_sample;
10422         struct mlx5_flow_dv_sample_resource *cache_resource;
10423
10424         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
10425                          idx);
10426         if (!cache_resource)
10427                 return 0;
10428         MLX5_ASSERT(cache_resource->verbs_action);
10429         DRV_LOG(DEBUG, "sample resource %p: refcnt %d--",
10430                 (void *)cache_resource,
10431                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10432         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10433                                __ATOMIC_RELAXED) == 0) {
10434                 if (cache_resource->verbs_action)
10435                         claim_zero(mlx5_glue->destroy_flow_action
10436                                         (cache_resource->verbs_action));
10437                 if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
10438                         if (cache_resource->default_miss)
10439                                 claim_zero(mlx5_glue->destroy_flow_action
10440                                   (cache_resource->default_miss));
10441                 }
10442                 if (cache_resource->normal_path_tbl)
10443                         flow_dv_tbl_resource_release(dev,
10444                                 cache_resource->normal_path_tbl);
10445         }
10446         if (cache_resource->sample_idx.rix_hrxq &&
10447                 !mlx5_hrxq_release(dev,
10448                         cache_resource->sample_idx.rix_hrxq))
10449                 cache_resource->sample_idx.rix_hrxq = 0;
10450         if (cache_resource->sample_idx.rix_tag &&
10451                 !flow_dv_tag_release(dev,
10452                         cache_resource->sample_idx.rix_tag))
10453                 cache_resource->sample_idx.rix_tag = 0;
10454         if (cache_resource->sample_idx.cnt) {
10455                 flow_dv_counter_release(dev,
10456                         cache_resource->sample_idx.cnt);
10457                 cache_resource->sample_idx.cnt = 0;
10458         }
10459         if (!__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)) {
10460                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
10461                              &priv->sh->sample_action_list, idx,
10462                              cache_resource, next);
10463                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], idx);
10464                 DRV_LOG(DEBUG, "sample resource %p: removed",
10465                         (void *)cache_resource);
10466                 return 0;
10467         }
10468         return 1;
10469 }
10470
10471 /**
10472  * Release an destination array resource.
10473  *
10474  * @param dev
10475  *   Pointer to Ethernet device.
10476  * @param handle
10477  *   Pointer to mlx5_flow_handle.
10478  *
10479  * @return
10480  *   1 while a reference on it exists, 0 when freed.
10481  */
10482 static int
10483 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
10484                                      struct mlx5_flow_handle *handle)
10485 {
10486         struct mlx5_priv *priv = dev->data->dev_private;
10487         struct mlx5_flow_dv_dest_array_resource *cache_resource;
10488         struct mlx5_flow_sub_actions_idx *mdest_act_res;
10489         uint32_t idx = handle->dvh.rix_dest_array;
10490         uint32_t i = 0;
10491
10492         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10493                          idx);
10494         if (!cache_resource)
10495                 return 0;
10496         MLX5_ASSERT(cache_resource->action);
10497         DRV_LOG(DEBUG, "destination array resource %p: refcnt %d--",
10498                 (void *)cache_resource,
10499                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10500         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10501                                __ATOMIC_RELAXED) == 0) {
10502                 if (cache_resource->action)
10503                         claim_zero(mlx5_glue->destroy_flow_action
10504                                                 (cache_resource->action));
10505                 for (; i < cache_resource->num_of_dest; i++) {
10506                         mdest_act_res = &cache_resource->sample_idx[i];
10507                         if (mdest_act_res->rix_hrxq) {
10508                                 mlx5_hrxq_release(dev,
10509                                         mdest_act_res->rix_hrxq);
10510                                 mdest_act_res->rix_hrxq = 0;
10511                         }
10512                         if (mdest_act_res->rix_encap_decap) {
10513                                 flow_dv_encap_decap_resource_release(dev,
10514                                         mdest_act_res->rix_encap_decap);
10515                                 mdest_act_res->rix_encap_decap = 0;
10516                         }
10517                         if (mdest_act_res->rix_port_id_action) {
10518                                 flow_dv_port_id_action_resource_release(dev,
10519                                         mdest_act_res->rix_port_id_action);
10520                                 mdest_act_res->rix_port_id_action = 0;
10521                         }
10522                         if (mdest_act_res->rix_tag) {
10523                                 flow_dv_tag_release(dev,
10524                                         mdest_act_res->rix_tag);
10525                                 mdest_act_res->rix_tag = 0;
10526                         }
10527                 }
10528                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10529                              &priv->sh->dest_array_list, idx,
10530                              cache_resource, next);
10531                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], idx);
10532                 DRV_LOG(DEBUG, "destination array resource %p: removed",
10533                         (void *)cache_resource);
10534                 return 0;
10535         }
10536         return 1;
10537 }
10538
10539 /**
10540  * Remove the flow from the NIC but keeps it in memory.
10541  * Lock free, (mutex should be acquired by caller).
10542  *
10543  * @param[in] dev
10544  *   Pointer to Ethernet device.
10545  * @param[in, out] flow
10546  *   Pointer to flow structure.
10547  */
10548 static void
10549 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
10550 {
10551         struct mlx5_flow_handle *dh;
10552         uint32_t handle_idx;
10553         struct mlx5_priv *priv = dev->data->dev_private;
10554
10555         if (!flow)
10556                 return;
10557         handle_idx = flow->dev_handles;
10558         while (handle_idx) {
10559                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10560                                     handle_idx);
10561                 if (!dh)
10562                         return;
10563                 if (dh->drv_flow) {
10564                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
10565                         dh->drv_flow = NULL;
10566                 }
10567                 if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
10568                     dh->fate_action == MLX5_FLOW_FATE_QUEUE ||
10569                     dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
10570                         flow_dv_fate_resource_release(dev, dh);
10571                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
10572                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
10573                 handle_idx = dh->next.next;
10574         }
10575 }
10576
10577 /**
10578  * Remove the flow from the NIC and the memory.
10579  * Lock free, (mutex should be acquired by caller).
10580  *
10581  * @param[in] dev
10582  *   Pointer to the Ethernet device structure.
10583  * @param[in, out] flow
10584  *   Pointer to flow structure.
10585  */
10586 static void
10587 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
10588 {
10589         struct mlx5_flow_handle *dev_handle;
10590         struct mlx5_priv *priv = dev->data->dev_private;
10591
10592         if (!flow)
10593                 return;
10594         __flow_dv_remove(dev, flow);
10595         if (flow->counter) {
10596                 flow_dv_counter_release(dev, flow->counter);
10597                 flow->counter = 0;
10598         }
10599         if (flow->meter) {
10600                 struct mlx5_flow_meter *fm;
10601
10602                 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
10603                                     flow->meter);
10604                 if (fm)
10605                         mlx5_flow_meter_detach(fm);
10606                 flow->meter = 0;
10607         }
10608         while (flow->dev_handles) {
10609                 uint32_t tmp_idx = flow->dev_handles;
10610
10611                 dev_handle = mlx5_ipool_get(priv->sh->ipool
10612                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
10613                 if (!dev_handle)
10614                         return;
10615                 flow->dev_handles = dev_handle->next.next;
10616                 if (dev_handle->dvh.matcher)
10617                         flow_dv_matcher_release(dev, dev_handle);
10618                 if (dev_handle->dvh.rix_sample)
10619                         flow_dv_sample_resource_release(dev, dev_handle);
10620                 if (dev_handle->dvh.rix_dest_array)
10621                         flow_dv_dest_array_resource_release(dev, dev_handle);
10622                 if (dev_handle->dvh.rix_encap_decap)
10623                         flow_dv_encap_decap_resource_release(dev,
10624                                 dev_handle->dvh.rix_encap_decap);
10625                 if (dev_handle->dvh.modify_hdr)
10626                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
10627                 if (dev_handle->dvh.rix_push_vlan)
10628                         flow_dv_push_vlan_action_resource_release(dev,
10629                                                                   dev_handle);
10630                 if (dev_handle->dvh.rix_tag)
10631                         flow_dv_tag_release(dev,
10632                                             dev_handle->dvh.rix_tag);
10633                 flow_dv_fate_resource_release(dev, dev_handle);
10634                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10635                            tmp_idx);
10636         }
10637 }
10638
10639 /**
10640  * Query a dv flow  rule for its statistics via devx.
10641  *
10642  * @param[in] dev
10643  *   Pointer to Ethernet device.
10644  * @param[in] flow
10645  *   Pointer to the sub flow.
10646  * @param[out] data
10647  *   data retrieved by the query.
10648  * @param[out] error
10649  *   Perform verbose error reporting if not NULL.
10650  *
10651  * @return
10652  *   0 on success, a negative errno value otherwise and rte_errno is set.
10653  */
10654 static int
10655 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
10656                     void *data, struct rte_flow_error *error)
10657 {
10658         struct mlx5_priv *priv = dev->data->dev_private;
10659         struct rte_flow_query_count *qc = data;
10660
10661         if (!priv->config.devx)
10662                 return rte_flow_error_set(error, ENOTSUP,
10663                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10664                                           NULL,
10665                                           "counters are not supported");
10666         if (flow->counter) {
10667                 uint64_t pkts, bytes;
10668                 struct mlx5_flow_counter *cnt;
10669
10670                 cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
10671                                                  NULL);
10672                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
10673                                                &bytes);
10674
10675                 if (err)
10676                         return rte_flow_error_set(error, -err,
10677                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10678                                         NULL, "cannot read counters");
10679                 qc->hits_set = 1;
10680                 qc->bytes_set = 1;
10681                 qc->hits = pkts - cnt->hits;
10682                 qc->bytes = bytes - cnt->bytes;
10683                 if (qc->reset) {
10684                         cnt->hits = pkts;
10685                         cnt->bytes = bytes;
10686                 }
10687                 return 0;
10688         }
10689         return rte_flow_error_set(error, EINVAL,
10690                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10691                                   NULL,
10692                                   "counters are not available");
10693 }
10694
10695 /**
10696  * Query a flow rule AGE action for aging information.
10697  *
10698  * @param[in] dev
10699  *   Pointer to Ethernet device.
10700  * @param[in] flow
10701  *   Pointer to the sub flow.
10702  * @param[out] data
10703  *   data retrieved by the query.
10704  * @param[out] error
10705  *   Perform verbose error reporting if not NULL.
10706  *
10707  * @return
10708  *   0 on success, a negative errno value otherwise and rte_errno is set.
10709  */
10710 static int
10711 flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow,
10712                   void *data, struct rte_flow_error *error)
10713 {
10714         struct rte_flow_query_age *resp = data;
10715
10716         if (flow->counter) {
10717                 struct mlx5_age_param *age_param =
10718                                 flow_dv_counter_idx_get_age(dev, flow->counter);
10719
10720                 if (!age_param || !age_param->timeout)
10721                         return rte_flow_error_set
10722                                         (error, EINVAL,
10723                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10724                                          NULL, "cannot read age data");
10725                 resp->aged = __atomic_load_n(&age_param->state,
10726                                              __ATOMIC_RELAXED) ==
10727                                                         AGE_TMOUT ? 1 : 0;
10728                 resp->sec_since_last_hit_valid = !resp->aged;
10729                 if (resp->sec_since_last_hit_valid)
10730                         resp->sec_since_last_hit =
10731                                 __atomic_load_n(&age_param->sec_since_last_hit,
10732                                                 __ATOMIC_RELAXED);
10733                 return 0;
10734         }
10735         return rte_flow_error_set(error, EINVAL,
10736                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10737                                   NULL,
10738                                   "age data not available");
10739 }
10740
10741 /**
10742  * Query a flow.
10743  *
10744  * @see rte_flow_query()
10745  * @see rte_flow_ops
10746  */
10747 static int
10748 flow_dv_query(struct rte_eth_dev *dev,
10749               struct rte_flow *flow __rte_unused,
10750               const struct rte_flow_action *actions __rte_unused,
10751               void *data __rte_unused,
10752               struct rte_flow_error *error __rte_unused)
10753 {
10754         int ret = -EINVAL;
10755
10756         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
10757                 switch (actions->type) {
10758                 case RTE_FLOW_ACTION_TYPE_VOID:
10759                         break;
10760                 case RTE_FLOW_ACTION_TYPE_COUNT:
10761                         ret = flow_dv_query_count(dev, flow, data, error);
10762                         break;
10763                 case RTE_FLOW_ACTION_TYPE_AGE:
10764                         ret = flow_dv_query_age(dev, flow, data, error);
10765                         break;
10766                 default:
10767                         return rte_flow_error_set(error, ENOTSUP,
10768                                                   RTE_FLOW_ERROR_TYPE_ACTION,
10769                                                   actions,
10770                                                   "action not supported");
10771                 }
10772         }
10773         return ret;
10774 }
10775
10776 /**
10777  * Destroy the meter table set.
10778  * Lock free, (mutex should be acquired by caller).
10779  *
10780  * @param[in] dev
10781  *   Pointer to Ethernet device.
10782  * @param[in] tbl
10783  *   Pointer to the meter table set.
10784  *
10785  * @return
10786  *   Always 0.
10787  */
10788 static int
10789 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
10790                         struct mlx5_meter_domains_infos *tbl)
10791 {
10792         struct mlx5_priv *priv = dev->data->dev_private;
10793         struct mlx5_meter_domains_infos *mtd =
10794                                 (struct mlx5_meter_domains_infos *)tbl;
10795
10796         if (!mtd || !priv->config.dv_flow_en)
10797                 return 0;
10798         if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
10799                 claim_zero(mlx5_flow_os_destroy_flow
10800                            (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
10801         if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
10802                 claim_zero(mlx5_flow_os_destroy_flow
10803                            (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
10804         if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
10805                 claim_zero(mlx5_flow_os_destroy_flow
10806                            (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
10807         if (mtd->egress.color_matcher)
10808                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10809                            (mtd->egress.color_matcher));
10810         if (mtd->egress.any_matcher)
10811                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10812                            (mtd->egress.any_matcher));
10813         if (mtd->egress.tbl)
10814                 flow_dv_tbl_resource_release(dev, mtd->egress.tbl);
10815         if (mtd->egress.sfx_tbl)
10816                 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl);
10817         if (mtd->ingress.color_matcher)
10818                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10819                            (mtd->ingress.color_matcher));
10820         if (mtd->ingress.any_matcher)
10821                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10822                            (mtd->ingress.any_matcher));
10823         if (mtd->ingress.tbl)
10824                 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl);
10825         if (mtd->ingress.sfx_tbl)
10826                 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl);
10827         if (mtd->transfer.color_matcher)
10828                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10829                            (mtd->transfer.color_matcher));
10830         if (mtd->transfer.any_matcher)
10831                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10832                            (mtd->transfer.any_matcher));
10833         if (mtd->transfer.tbl)
10834                 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl);
10835         if (mtd->transfer.sfx_tbl)
10836                 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl);
10837         if (mtd->drop_actn)
10838                 claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn));
10839         mlx5_free(mtd);
10840         return 0;
10841 }
10842
10843 /* Number of meter flow actions, count and jump or count and drop. */
10844 #define METER_ACTIONS 2
10845
10846 /**
10847  * Create specify domain meter table and suffix table.
10848  *
10849  * @param[in] dev
10850  *   Pointer to Ethernet device.
10851  * @param[in,out] mtb
10852  *   Pointer to DV meter table set.
10853  * @param[in] egress
10854  *   Table attribute.
10855  * @param[in] transfer
10856  *   Table attribute.
10857  * @param[in] color_reg_c_idx
10858  *   Reg C index for color match.
10859  *
10860  * @return
10861  *   0 on success, -1 otherwise and rte_errno is set.
10862  */
10863 static int
10864 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
10865                            struct mlx5_meter_domains_infos *mtb,
10866                            uint8_t egress, uint8_t transfer,
10867                            uint32_t color_reg_c_idx)
10868 {
10869         struct mlx5_priv *priv = dev->data->dev_private;
10870         struct mlx5_dev_ctx_shared *sh = priv->sh;
10871         struct mlx5_flow_dv_match_params mask = {
10872                 .size = sizeof(mask.buf),
10873         };
10874         struct mlx5_flow_dv_match_params value = {
10875                 .size = sizeof(value.buf),
10876         };
10877         struct mlx5dv_flow_matcher_attr dv_attr = {
10878                 .type = IBV_FLOW_ATTR_NORMAL,
10879                 .priority = 0,
10880                 .match_criteria_enable = 0,
10881                 .match_mask = (void *)&mask,
10882         };
10883         void *actions[METER_ACTIONS];
10884         struct mlx5_meter_domain_info *dtb;
10885         struct rte_flow_error error;
10886         int i = 0;
10887         int ret;
10888
10889         if (transfer)
10890                 dtb = &mtb->transfer;
10891         else if (egress)
10892                 dtb = &mtb->egress;
10893         else
10894                 dtb = &mtb->ingress;
10895         /* Create the meter table with METER level. */
10896         dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
10897                                             egress, transfer, &error);
10898         if (!dtb->tbl) {
10899                 DRV_LOG(ERR, "Failed to create meter policer table.");
10900                 return -1;
10901         }
10902         /* Create the meter suffix table with SUFFIX level. */
10903         dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
10904                                             MLX5_FLOW_TABLE_LEVEL_SUFFIX,
10905                                             egress, transfer, &error);
10906         if (!dtb->sfx_tbl) {
10907                 DRV_LOG(ERR, "Failed to create meter suffix table.");
10908                 return -1;
10909         }
10910         /* Create matchers, Any and Color. */
10911         dv_attr.priority = 3;
10912         dv_attr.match_criteria_enable = 0;
10913         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
10914                                                &dtb->any_matcher);
10915         if (ret) {
10916                 DRV_LOG(ERR, "Failed to create meter"
10917                              " policer default matcher.");
10918                 goto error_exit;
10919         }
10920         dv_attr.priority = 0;
10921         dv_attr.match_criteria_enable =
10922                                 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
10923         flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
10924                                rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
10925         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
10926                                                &dtb->color_matcher);
10927         if (ret) {
10928                 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
10929                 goto error_exit;
10930         }
10931         if (mtb->count_actns[RTE_MTR_DROPPED])
10932                 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
10933         actions[i++] = mtb->drop_actn;
10934         /* Default rule: lowest priority, match any, actions: drop. */
10935         ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i,
10936                                        actions,
10937                                        &dtb->policer_rules[RTE_MTR_DROPPED]);
10938         if (ret) {
10939                 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
10940                 goto error_exit;
10941         }
10942         return 0;
10943 error_exit:
10944         return -1;
10945 }
10946
10947 /**
10948  * Create the needed meter and suffix tables.
10949  * Lock free, (mutex should be acquired by caller).
10950  *
10951  * @param[in] dev
10952  *   Pointer to Ethernet device.
10953  * @param[in] fm
10954  *   Pointer to the flow meter.
10955  *
10956  * @return
10957  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
10958  */
10959 static struct mlx5_meter_domains_infos *
10960 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
10961                        const struct mlx5_flow_meter *fm)
10962 {
10963         struct mlx5_priv *priv = dev->data->dev_private;
10964         struct mlx5_meter_domains_infos *mtb;
10965         int ret;
10966         int i;
10967
10968         if (!priv->mtr_en) {
10969                 rte_errno = ENOTSUP;
10970                 return NULL;
10971         }
10972         mtb = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*mtb), 0, SOCKET_ID_ANY);
10973         if (!mtb) {
10974                 DRV_LOG(ERR, "Failed to allocate memory for meter.");
10975                 return NULL;
10976         }
10977         /* Create meter count actions */
10978         for (i = 0; i <= RTE_MTR_DROPPED; i++) {
10979                 struct mlx5_flow_counter *cnt;
10980                 if (!fm->policer_stats.cnt[i])
10981                         continue;
10982                 cnt = flow_dv_counter_get_by_idx(dev,
10983                       fm->policer_stats.cnt[i], NULL);
10984                 mtb->count_actns[i] = cnt->action;
10985         }
10986         /* Create drop action. */
10987         ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn);
10988         if (ret) {
10989                 DRV_LOG(ERR, "Failed to create drop action.");
10990                 goto error_exit;
10991         }
10992         /* Egress meter table. */
10993         ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
10994         if (ret) {
10995                 DRV_LOG(ERR, "Failed to prepare egress meter table.");
10996                 goto error_exit;
10997         }
10998         /* Ingress meter table. */
10999         ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
11000         if (ret) {
11001                 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
11002                 goto error_exit;
11003         }
11004         /* FDB meter table. */
11005         if (priv->config.dv_esw_en) {
11006                 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
11007                                                  priv->mtr_color_reg);
11008                 if (ret) {
11009                         DRV_LOG(ERR, "Failed to prepare fdb meter table.");
11010                         goto error_exit;
11011                 }
11012         }
11013         return mtb;
11014 error_exit:
11015         flow_dv_destroy_mtr_tbl(dev, mtb);
11016         return NULL;
11017 }
11018
11019 /**
11020  * Destroy domain policer rule.
11021  *
11022  * @param[in] dt
11023  *   Pointer to domain table.
11024  */
11025 static void
11026 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
11027 {
11028         int i;
11029
11030         for (i = 0; i < RTE_MTR_DROPPED; i++) {
11031                 if (dt->policer_rules[i]) {
11032                         claim_zero(mlx5_flow_os_destroy_flow
11033                                    (dt->policer_rules[i]));
11034                         dt->policer_rules[i] = NULL;
11035                 }
11036         }
11037         if (dt->jump_actn) {
11038                 claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn));
11039                 dt->jump_actn = NULL;
11040         }
11041 }
11042
11043 /**
11044  * Destroy policer rules.
11045  *
11046  * @param[in] dev
11047  *   Pointer to Ethernet device.
11048  * @param[in] fm
11049  *   Pointer to flow meter structure.
11050  * @param[in] attr
11051  *   Pointer to flow attributes.
11052  *
11053  * @return
11054  *   Always 0.
11055  */
11056 static int
11057 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
11058                               const struct mlx5_flow_meter *fm,
11059                               const struct rte_flow_attr *attr)
11060 {
11061         struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
11062
11063         if (!mtb)
11064                 return 0;
11065         if (attr->egress)
11066                 flow_dv_destroy_domain_policer_rule(&mtb->egress);
11067         if (attr->ingress)
11068                 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
11069         if (attr->transfer)
11070                 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
11071         return 0;
11072 }
11073
11074 /**
11075  * Create specify domain meter policer rule.
11076  *
11077  * @param[in] fm
11078  *   Pointer to flow meter structure.
11079  * @param[in] mtb
11080  *   Pointer to DV meter table set.
11081  * @param[in] mtr_reg_c
11082  *   Color match REG_C.
11083  *
11084  * @return
11085  *   0 on success, -1 otherwise.
11086  */
11087 static int
11088 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
11089                                     struct mlx5_meter_domain_info *dtb,
11090                                     uint8_t mtr_reg_c)
11091 {
11092         struct mlx5_flow_dv_match_params matcher = {
11093                 .size = sizeof(matcher.buf),
11094         };
11095         struct mlx5_flow_dv_match_params value = {
11096                 .size = sizeof(value.buf),
11097         };
11098         struct mlx5_meter_domains_infos *mtb = fm->mfts;
11099         void *actions[METER_ACTIONS];
11100         int i;
11101         int ret = 0;
11102
11103         /* Create jump action. */
11104         if (!dtb->jump_actn)
11105                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
11106                                 (dtb->sfx_tbl->obj, &dtb->jump_actn);
11107         if (ret) {
11108                 DRV_LOG(ERR, "Failed to create policer jump action.");
11109                 goto error;
11110         }
11111         for (i = 0; i < RTE_MTR_DROPPED; i++) {
11112                 int j = 0;
11113
11114                 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
11115                                        rte_col_2_mlx5_col(i), UINT8_MAX);
11116                 if (mtb->count_actns[i])
11117                         actions[j++] = mtb->count_actns[i];
11118                 if (fm->action[i] == MTR_POLICER_ACTION_DROP)
11119                         actions[j++] = mtb->drop_actn;
11120                 else
11121                         actions[j++] = dtb->jump_actn;
11122                 ret = mlx5_flow_os_create_flow(dtb->color_matcher,
11123                                                (void *)&value, j, actions,
11124                                                &dtb->policer_rules[i]);
11125                 if (ret) {
11126                         DRV_LOG(ERR, "Failed to create policer rule.");
11127                         goto error;
11128                 }
11129         }
11130         return 0;
11131 error:
11132         rte_errno = errno;
11133         return -1;
11134 }
11135
11136 /**
11137  * Create policer rules.
11138  *
11139  * @param[in] dev
11140  *   Pointer to Ethernet device.
11141  * @param[in] fm
11142  *   Pointer to flow meter structure.
11143  * @param[in] attr
11144  *   Pointer to flow attributes.
11145  *
11146  * @return
11147  *   0 on success, -1 otherwise.
11148  */
11149 static int
11150 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
11151                              struct mlx5_flow_meter *fm,
11152                              const struct rte_flow_attr *attr)
11153 {
11154         struct mlx5_priv *priv = dev->data->dev_private;
11155         struct mlx5_meter_domains_infos *mtb = fm->mfts;
11156         int ret;
11157
11158         if (attr->egress) {
11159                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
11160                                                 priv->mtr_color_reg);
11161                 if (ret) {
11162                         DRV_LOG(ERR, "Failed to create egress policer.");
11163                         goto error;
11164                 }
11165         }
11166         if (attr->ingress) {
11167                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
11168                                                 priv->mtr_color_reg);
11169                 if (ret) {
11170                         DRV_LOG(ERR, "Failed to create ingress policer.");
11171                         goto error;
11172                 }
11173         }
11174         if (attr->transfer) {
11175                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
11176                                                 priv->mtr_color_reg);
11177                 if (ret) {
11178                         DRV_LOG(ERR, "Failed to create transfer policer.");
11179                         goto error;
11180                 }
11181         }
11182         return 0;
11183 error:
11184         flow_dv_destroy_policer_rules(dev, fm, attr);
11185         return -1;
11186 }
11187
11188 /**
11189  * Validate the batch counter support in root table.
11190  *
11191  * Create a simple flow with invalid counter and drop action on root table to
11192  * validate if batch counter with offset on root table is supported or not.
11193  *
11194  * @param[in] dev
11195  *   Pointer to rte_eth_dev structure.
11196  *
11197  * @return
11198  *   0 on success, a negative errno value otherwise and rte_errno is set.
11199  */
11200 int
11201 mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev)
11202 {
11203         struct mlx5_priv *priv = dev->data->dev_private;
11204         struct mlx5_dev_ctx_shared *sh = priv->sh;
11205         struct mlx5_flow_dv_match_params mask = {
11206                 .size = sizeof(mask.buf),
11207         };
11208         struct mlx5_flow_dv_match_params value = {
11209                 .size = sizeof(value.buf),
11210         };
11211         struct mlx5dv_flow_matcher_attr dv_attr = {
11212                 .type = IBV_FLOW_ATTR_NORMAL,
11213                 .priority = 0,
11214                 .match_criteria_enable = 0,
11215                 .match_mask = (void *)&mask,
11216         };
11217         void *actions[2] = { 0 };
11218         struct mlx5_flow_tbl_resource *tbl = NULL, *dest_tbl = NULL;
11219         struct mlx5_devx_obj *dcs = NULL;
11220         void *matcher = NULL;
11221         void *flow = NULL;
11222         int i, ret = -1;
11223
11224         tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, NULL);
11225         if (!tbl)
11226                 goto err;
11227         dest_tbl = flow_dv_tbl_resource_get(dev, 1, 0, 0, NULL);
11228         if (!dest_tbl)
11229                 goto err;
11230         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
11231         if (!dcs)
11232                 goto err;
11233         ret = mlx5_flow_os_create_flow_action_count(dcs->obj, UINT16_MAX,
11234                                                     &actions[0]);
11235         if (ret)
11236                 goto err;
11237         ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
11238                                 (dest_tbl->obj, &actions[1]);
11239         if (ret)
11240                 goto err;
11241         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
11242         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
11243                                                &matcher);
11244         if (ret)
11245                 goto err;
11246         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 2,
11247                                        actions, &flow);
11248 err:
11249         /*
11250          * If batch counter with offset is not supported, the driver will not
11251          * validate the invalid offset value, flow create should success.
11252          * In this case, it means batch counter is not supported in root table.
11253          *
11254          * Otherwise, if flow create is failed, counter offset is supported.
11255          */
11256         if (flow) {
11257                 DRV_LOG(INFO, "Batch counter is not supported in root "
11258                               "table. Switch to fallback mode.");
11259                 rte_errno = ENOTSUP;
11260                 ret = -rte_errno;
11261                 claim_zero(mlx5_flow_os_destroy_flow(flow));
11262         } else {
11263                 /* Check matcher to make sure validate fail at flow create. */
11264                 if (!matcher || (matcher && errno != EINVAL))
11265                         DRV_LOG(ERR, "Unexpected error in counter offset "
11266                                      "support detection");
11267                 ret = 0;
11268         }
11269         for (i = 0; i < 2; i++) {
11270                 if (actions[i])
11271                         claim_zero(mlx5_flow_os_destroy_flow_action
11272                                    (actions[i]));
11273         }
11274         if (matcher)
11275                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
11276         if (tbl)
11277                 flow_dv_tbl_resource_release(dev, tbl);
11278         if (dest_tbl)
11279                 flow_dv_tbl_resource_release(dev, dest_tbl);
11280         if (dcs)
11281                 claim_zero(mlx5_devx_cmd_destroy(dcs));
11282         return ret;
11283 }
11284
11285 /**
11286  * Query a devx counter.
11287  *
11288  * @param[in] dev
11289  *   Pointer to the Ethernet device structure.
11290  * @param[in] cnt
11291  *   Index to the flow counter.
11292  * @param[in] clear
11293  *   Set to clear the counter statistics.
11294  * @param[out] pkts
11295  *   The statistics value of packets.
11296  * @param[out] bytes
11297  *   The statistics value of bytes.
11298  *
11299  * @return
11300  *   0 on success, otherwise return -1.
11301  */
11302 static int
11303 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
11304                       uint64_t *pkts, uint64_t *bytes)
11305 {
11306         struct mlx5_priv *priv = dev->data->dev_private;
11307         struct mlx5_flow_counter *cnt;
11308         uint64_t inn_pkts, inn_bytes;
11309         int ret;
11310
11311         if (!priv->config.devx)
11312                 return -1;
11313
11314         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
11315         if (ret)
11316                 return -1;
11317         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
11318         *pkts = inn_pkts - cnt->hits;
11319         *bytes = inn_bytes - cnt->bytes;
11320         if (clear) {
11321                 cnt->hits = inn_pkts;
11322                 cnt->bytes = inn_bytes;
11323         }
11324         return 0;
11325 }
11326
11327 /**
11328  * Get aged-out flows.
11329  *
11330  * @param[in] dev
11331  *   Pointer to the Ethernet device structure.
11332  * @param[in] context
11333  *   The address of an array of pointers to the aged-out flows contexts.
11334  * @param[in] nb_contexts
11335  *   The length of context array pointers.
11336  * @param[out] error
11337  *   Perform verbose error reporting if not NULL. Initialized in case of
11338  *   error only.
11339  *
11340  * @return
11341  *   how many contexts get in success, otherwise negative errno value.
11342  *   if nb_contexts is 0, return the amount of all aged contexts.
11343  *   if nb_contexts is not 0 , return the amount of aged flows reported
11344  *   in the context array.
11345  * @note: only stub for now
11346  */
11347 static int
11348 flow_get_aged_flows(struct rte_eth_dev *dev,
11349                     void **context,
11350                     uint32_t nb_contexts,
11351                     struct rte_flow_error *error)
11352 {
11353         struct mlx5_priv *priv = dev->data->dev_private;
11354         struct mlx5_age_info *age_info;
11355         struct mlx5_age_param *age_param;
11356         struct mlx5_flow_counter *counter;
11357         int nb_flows = 0;
11358
11359         if (nb_contexts && !context)
11360                 return rte_flow_error_set(error, EINVAL,
11361                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11362                                           NULL,
11363                                           "Should assign at least one flow or"
11364                                           " context to get if nb_contexts != 0");
11365         age_info = GET_PORT_AGE_INFO(priv);
11366         rte_spinlock_lock(&age_info->aged_sl);
11367         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
11368                 nb_flows++;
11369                 if (nb_contexts) {
11370                         age_param = MLX5_CNT_TO_AGE(counter);
11371                         context[nb_flows - 1] = age_param->context;
11372                         if (!(--nb_contexts))
11373                                 break;
11374                 }
11375         }
11376         rte_spinlock_unlock(&age_info->aged_sl);
11377         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
11378         return nb_flows;
11379 }
11380
11381 /*
11382  * Mutex-protected thunk to lock-free  __flow_dv_translate().
11383  */
11384 static int
11385 flow_dv_translate(struct rte_eth_dev *dev,
11386                   struct mlx5_flow *dev_flow,
11387                   const struct rte_flow_attr *attr,
11388                   const struct rte_flow_item items[],
11389                   const struct rte_flow_action actions[],
11390                   struct rte_flow_error *error)
11391 {
11392         int ret;
11393
11394         flow_dv_shared_lock(dev);
11395         ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
11396         flow_dv_shared_unlock(dev);
11397         return ret;
11398 }
11399
11400 /*
11401  * Mutex-protected thunk to lock-free  __flow_dv_apply().
11402  */
11403 static int
11404 flow_dv_apply(struct rte_eth_dev *dev,
11405               struct rte_flow *flow,
11406               struct rte_flow_error *error)
11407 {
11408         int ret;
11409
11410         flow_dv_shared_lock(dev);
11411         ret = __flow_dv_apply(dev, flow, error);
11412         flow_dv_shared_unlock(dev);
11413         return ret;
11414 }
11415
11416 /*
11417  * Mutex-protected thunk to lock-free __flow_dv_remove().
11418  */
11419 static void
11420 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
11421 {
11422         flow_dv_shared_lock(dev);
11423         __flow_dv_remove(dev, flow);
11424         flow_dv_shared_unlock(dev);
11425 }
11426
11427 /*
11428  * Mutex-protected thunk to lock-free __flow_dv_destroy().
11429  */
11430 static void
11431 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
11432 {
11433         flow_dv_shared_lock(dev);
11434         __flow_dv_destroy(dev, flow);
11435         flow_dv_shared_unlock(dev);
11436 }
11437
11438 /*
11439  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
11440  */
11441 static uint32_t
11442 flow_dv_counter_allocate(struct rte_eth_dev *dev)
11443 {
11444         uint32_t cnt;
11445
11446         flow_dv_shared_lock(dev);
11447         cnt = flow_dv_counter_alloc(dev, 0);
11448         flow_dv_shared_unlock(dev);
11449         return cnt;
11450 }
11451
11452 /*
11453  * Mutex-protected thunk to lock-free flow_dv_counter_release().
11454  */
11455 static void
11456 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
11457 {
11458         flow_dv_shared_lock(dev);
11459         flow_dv_counter_release(dev, cnt);
11460         flow_dv_shared_unlock(dev);
11461 }
11462
11463 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
11464         .validate = flow_dv_validate,
11465         .prepare = flow_dv_prepare,
11466         .translate = flow_dv_translate,
11467         .apply = flow_dv_apply,
11468         .remove = flow_dv_remove,
11469         .destroy = flow_dv_destroy,
11470         .query = flow_dv_query,
11471         .create_mtr_tbls = flow_dv_create_mtr_tbl,
11472         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
11473         .create_policer_rules = flow_dv_create_policer_rules,
11474         .destroy_policer_rules = flow_dv_destroy_policer_rules,
11475         .counter_alloc = flow_dv_counter_allocate,
11476         .counter_free = flow_dv_counter_free,
11477         .counter_query = flow_dv_counter_query,
11478         .get_aged_flows = flow_get_aged_flows,
11479 };
11480
11481 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
11482