drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <rte_ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_ip.h>
  19 #include <rte_gre.h>
  20 #include <rte_vxlan.h>
  21 #include <rte_gtp.h>
  22 #include <rte_eal_paging.h>
  23 #include <rte_mpls.h>
  24
  25 #include <mlx5_glue.h>
  26 #include <mlx5_devx_cmds.h>
  27 #include <mlx5_prm.h>
  28 #include <mlx5_malloc.h>
  29
  30 #include "mlx5_defs.h"
  31 #include "mlx5.h"
  32 #include "mlx5_common_os.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_flow_os.h"
  35 #include "mlx5_rxtx.h"
  36
  37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  38
  39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  41 #endif
  42
  43 #ifndef HAVE_MLX5DV_DR_ESWITCH
  44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  46 #endif
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR
  50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  51 #endif
  52
  53 /* VLAN header definitions */
  54 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  55 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  56 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  57 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  58 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  59
  60 union flow_dv_attr {
  61         struct {
  62                 uint32_t valid:1;
  63                 uint32_t ipv4:1;
  64                 uint32_t ipv6:1;
  65                 uint32_t tcp:1;
  66                 uint32_t udp:1;
  67                 uint32_t reserved:27;
  68         };
  69         uint32_t attr;
  70 };
  71
  72 static int
  73 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
  74                              struct mlx5_flow_tbl_resource *tbl);
  75
  76 static int
  77 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev);
  78
  79 static int
  80 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  81                                       uint32_t encap_decap_idx);
  82
  83 static int
  84 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  85                                         uint32_t port_id);
  86
  87 /**
  88  * Initialize flow attributes structure according to flow items' types.
  89  *
  90  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  91  * mode. For tunnel mode, the items to be modified are the outermost ones.
  92  *
  93  * @param[in] item
  94  *   Pointer to item specification.
  95  * @param[out] attr
  96  *   Pointer to flow attributes structure.
  97  * @param[in] dev_flow
  98  *   Pointer to the sub flow.
  99  * @param[in] tunnel_decap
 100  *   Whether action is after tunnel decapsulation.
 101  */
 102 static void
 103 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 104                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 105 {
 106         uint64_t layers = dev_flow->handle->layers;
 107
 108         /*
 109          * If layers is already initialized, it means this dev_flow is the
 110          * suffix flow, the layers flags is set by the prefix flow. Need to
 111          * use the layer flags from prefix flow as the suffix flow may not
 112          * have the user defined items as the flow is split.
 113          */
 114         if (layers) {
 115                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 116                         attr->ipv4 = 1;
 117                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 118                         attr->ipv6 = 1;
 119                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 120                         attr->tcp = 1;
 121                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 122                         attr->udp = 1;
 123                 attr->valid = 1;
 124                 return;
 125         }
 126         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 127                 uint8_t next_protocol = 0xff;
 128                 switch (item->type) {
 129                 case RTE_FLOW_ITEM_TYPE_GRE:
 130                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 131                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 132                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 133                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 134                 case RTE_FLOW_ITEM_TYPE_MPLS:
 135                         if (tunnel_decap)
 136                                 attr->attr = 0;
 137                         break;
 138                 case RTE_FLOW_ITEM_TYPE_IPV4:
 139                         if (!attr->ipv6)
 140                                 attr->ipv4 = 1;
 141                         if (item->mask != NULL &&
 142                             ((const struct rte_flow_item_ipv4 *)
 143                             item->mask)->hdr.next_proto_id)
 144                                 next_protocol =
 145                                     ((const struct rte_flow_item_ipv4 *)
 146                                       (item->spec))->hdr.next_proto_id &
 147                                     ((const struct rte_flow_item_ipv4 *)
 148                                       (item->mask))->hdr.next_proto_id;
 149                         if ((next_protocol == IPPROTO_IPIP ||
 150                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 151                                 attr->attr = 0;
 152                         break;
 153                 case RTE_FLOW_ITEM_TYPE_IPV6:
 154                         if (!attr->ipv4)
 155                                 attr->ipv6 = 1;
 156                         if (item->mask != NULL &&
 157                             ((const struct rte_flow_item_ipv6 *)
 158                             item->mask)->hdr.proto)
 159                                 next_protocol =
 160                                     ((const struct rte_flow_item_ipv6 *)
 161                                       (item->spec))->hdr.proto &
 162                                     ((const struct rte_flow_item_ipv6 *)
 163                                       (item->mask))->hdr.proto;
 164                         if ((next_protocol == IPPROTO_IPIP ||
 165                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 166                                 attr->attr = 0;
 167                         break;
 168                 case RTE_FLOW_ITEM_TYPE_UDP:
 169                         if (!attr->tcp)
 170                                 attr->udp = 1;
 171                         break;
 172                 case RTE_FLOW_ITEM_TYPE_TCP:
 173                         if (!attr->udp)
 174                                 attr->tcp = 1;
 175                         break;
 176                 default:
 177                         break;
 178                 }
 179         }
 180         attr->valid = 1;
 181 }
 182
 183 /**
 184  * Convert rte_mtr_color to mlx5 color.
 185  *
 186  * @param[in] rcol
 187  *   rte_mtr_color.
 188  *
 189  * @return
 190  *   mlx5 color.
 191  */
 192 static int
 193 rte_col_2_mlx5_col(enum rte_color rcol)
 194 {
 195         switch (rcol) {
 196         case RTE_COLOR_GREEN:
 197                 return MLX5_FLOW_COLOR_GREEN;
 198         case RTE_COLOR_YELLOW:
 199                 return MLX5_FLOW_COLOR_YELLOW;
 200         case RTE_COLOR_RED:
 201                 return MLX5_FLOW_COLOR_RED;
 202         default:
 203                 break;
 204         }
 205         return MLX5_FLOW_COLOR_UNDEFINED;
 206 }
 207
 208 struct field_modify_info {
 209         uint32_t size; /* Size of field in protocol header, in bytes. */
 210         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 211         enum mlx5_modification_field id;
 212 };
 213
 214 struct field_modify_info modify_eth[] = {
 215         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 216         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 217         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 218         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 219         {0, 0, 0},
 220 };
 221
 222 struct field_modify_info modify_vlan_out_first_vid[] = {
 223         /* Size in bits !!! */
 224         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 225         {0, 0, 0},
 226 };
 227
 228 struct field_modify_info modify_ipv4[] = {
 229         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 230         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 231         {4, 12, MLX5_MODI_OUT_SIPV4},
 232         {4, 16, MLX5_MODI_OUT_DIPV4},
 233         {0, 0, 0},
 234 };
 235
 236 struct field_modify_info modify_ipv6[] = {
 237         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 238         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 239         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 240         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 241         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 242         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 243         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 244         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 245         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 246         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 247         {0, 0, 0},
 248 };
 249
 250 struct field_modify_info modify_udp[] = {
 251         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 252         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 253         {0, 0, 0},
 254 };
 255
 256 struct field_modify_info modify_tcp[] = {
 257         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 258         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 259         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 260         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 261         {0, 0, 0},
 262 };
 263
 264 static void
 265 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 266                           uint8_t next_protocol, uint64_t *item_flags,
 267                           int *tunnel)
 268 {
 269         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 270                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 271         if (next_protocol == IPPROTO_IPIP) {
 272                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 273                 *tunnel = 1;
 274         }
 275         if (next_protocol == IPPROTO_IPV6) {
 276                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 277                 *tunnel = 1;
 278         }
 279 }
 280
 281 /**
 282  * Acquire the synchronizing object to protect multithreaded access
 283  * to shared dv context. Lock occurs only if context is actually
 284  * shared, i.e. we have multiport IB device and representors are
 285  * created.
 286  *
 287  * @param[in] dev
 288  *   Pointer to the rte_eth_dev structure.
 289  */
 290 static void
 291 flow_dv_shared_lock(struct rte_eth_dev *dev)
 292 {
 293         struct mlx5_priv *priv = dev->data->dev_private;
 294         struct mlx5_dev_ctx_shared *sh = priv->sh;
 295
 296         if (sh->dv_refcnt > 1) {
 297                 int ret;
 298
 299                 ret = pthread_mutex_lock(&sh->dv_mutex);
 300                 MLX5_ASSERT(!ret);
 301                 (void)ret;
 302         }
 303 }
 304
 305 static void
 306 flow_dv_shared_unlock(struct rte_eth_dev *dev)
 307 {
 308         struct mlx5_priv *priv = dev->data->dev_private;
 309         struct mlx5_dev_ctx_shared *sh = priv->sh;
 310
 311         if (sh->dv_refcnt > 1) {
 312                 int ret;
 313
 314                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 315                 MLX5_ASSERT(!ret);
 316                 (void)ret;
 317         }
 318 }
 319
 320 /* Update VLAN's VID/PCP based on input rte_flow_action.
 321  *
 322  * @param[in] action
 323  *   Pointer to struct rte_flow_action.
 324  * @param[out] vlan
 325  *   Pointer to struct rte_vlan_hdr.
 326  */
 327 static void
 328 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 329                          struct rte_vlan_hdr *vlan)
 330 {
 331         uint16_t vlan_tci;
 332         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 333                 vlan_tci =
 334                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 335                                                action->conf)->vlan_pcp;
 336                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 337                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 338                 vlan->vlan_tci |= vlan_tci;
 339         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 340                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 341                 vlan->vlan_tci |= rte_be_to_cpu_16
 342                     (((const struct rte_flow_action_of_set_vlan_vid *)
 343                                              action->conf)->vlan_vid);
 344         }
 345 }
 346
 347 /**
 348  * Fetch 1, 2, 3 or 4 byte field from the byte array
 349  * and return as unsigned integer in host-endian format.
 350  *
 351  * @param[in] data
 352  *   Pointer to data array.
 353  * @param[in] size
 354  *   Size of field to extract.
 355  *
 356  * @return
 357  *   converted field in host endian format.
 358  */
 359 static inline uint32_t
 360 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 361 {
 362         uint32_t ret;
 363
 364         switch (size) {
 365         case 1:
 366                 ret = *data;
 367                 break;
 368         case 2:
 369                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 370                 break;
 371         case 3:
 372                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 373                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 374                 break;
 375         case 4:
 376                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 377                 break;
 378         default:
 379                 MLX5_ASSERT(false);
 380                 ret = 0;
 381                 break;
 382         }
 383         return ret;
 384 }
 385
 386 /**
 387  * Convert modify-header action to DV specification.
 388  *
 389  * Data length of each action is determined by provided field description
 390  * and the item mask. Data bit offset and width of each action is determined
 391  * by provided item mask.
 392  *
 393  * @param[in] item
 394  *   Pointer to item specification.
 395  * @param[in] field
 396  *   Pointer to field modification information.
 397  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 398  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 399  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 400  * @param[in] dcopy
 401  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 402  *   Negative offset value sets the same offset as source offset.
 403  *   size field is ignored, value is taken from source field.
 404  * @param[in,out] resource
 405  *   Pointer to the modify-header resource.
 406  * @param[in] type
 407  *   Type of modification.
 408  * @param[out] error
 409  *   Pointer to the error structure.
 410  *
 411  * @return
 412  *   0 on success, a negative errno value otherwise and rte_errno is set.
 413  */
 414 static int
 415 flow_dv_convert_modify_action(struct rte_flow_item *item,
 416                               struct field_modify_info *field,
 417                               struct field_modify_info *dcopy,
 418                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 419                               uint32_t type, struct rte_flow_error *error)
 420 {
 421         uint32_t i = resource->actions_num;
 422         struct mlx5_modification_cmd *actions = resource->actions;
 423
 424         /*
 425          * The item and mask are provided in big-endian format.
 426          * The fields should be presented as in big-endian format either.
 427          * Mask must be always present, it defines the actual field width.
 428          */
 429         MLX5_ASSERT(item->mask);
 430         MLX5_ASSERT(field->size);
 431         do {
 432                 unsigned int size_b;
 433                 unsigned int off_b;
 434                 uint32_t mask;
 435                 uint32_t data;
 436
 437                 if (i >= MLX5_MAX_MODIFY_NUM)
 438                         return rte_flow_error_set(error, EINVAL,
 439                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 440                                  "too many items to modify");
 441                 /* Fetch variable byte size mask from the array. */
 442                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 443                                            field->offset, field->size);
 444                 if (!mask) {
 445                         ++field;
 446                         continue;
 447                 }
 448                 /* Deduce actual data width in bits from mask value. */
 449                 off_b = rte_bsf32(mask);
 450                 size_b = sizeof(uint32_t) * CHAR_BIT -
 451                          off_b - __builtin_clz(mask);
 452                 MLX5_ASSERT(size_b);
 453                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 454                 actions[i] = (struct mlx5_modification_cmd) {
 455                         .action_type = type,
 456                         .field = field->id,
 457                         .offset = off_b,
 458                         .length = size_b,
 459                 };
 460                 /* Convert entire record to expected big-endian format. */
 461                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 462                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 463                         MLX5_ASSERT(dcopy);
 464                         actions[i].dst_field = dcopy->id;
 465                         actions[i].dst_offset =
 466                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 467                         /* Convert entire record to big-endian format. */
 468                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 469                 } else {
 470                         MLX5_ASSERT(item->spec);
 471                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 472                                                    field->offset, field->size);
 473                         /* Shift out the trailing masked bits from data. */
 474                         data = (data & mask) >> off_b;
 475                         actions[i].data1 = rte_cpu_to_be_32(data);
 476                 }
 477                 ++i;
 478                 ++field;
 479         } while (field->size);
 480         if (resource->actions_num == i)
 481                 return rte_flow_error_set(error, EINVAL,
 482                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 483                                           "invalid modification flow item");
 484         resource->actions_num = i;
 485         return 0;
 486 }
 487
 488 /**
 489  * Convert modify-header set IPv4 address action to DV specification.
 490  *
 491  * @param[in,out] resource
 492  *   Pointer to the modify-header resource.
 493  * @param[in] action
 494  *   Pointer to action specification.
 495  * @param[out] error
 496  *   Pointer to the error structure.
 497  *
 498  * @return
 499  *   0 on success, a negative errno value otherwise and rte_errno is set.
 500  */
 501 static int
 502 flow_dv_convert_action_modify_ipv4
 503                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 504                          const struct rte_flow_action *action,
 505                          struct rte_flow_error *error)
 506 {
 507         const struct rte_flow_action_set_ipv4 *conf =
 508                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 509         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 510         struct rte_flow_item_ipv4 ipv4;
 511         struct rte_flow_item_ipv4 ipv4_mask;
 512
 513         memset(&ipv4, 0, sizeof(ipv4));
 514         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 515         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 516                 ipv4.hdr.src_addr = conf->ipv4_addr;
 517                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 518         } else {
 519                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 520                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 521         }
 522         item.spec = &ipv4;
 523         item.mask = &ipv4_mask;
 524         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 525                                              MLX5_MODIFICATION_TYPE_SET, error);
 526 }
 527
 528 /**
 529  * Convert modify-header set IPv6 address action to DV specification.
 530  *
 531  * @param[in,out] resource
 532  *   Pointer to the modify-header resource.
 533  * @param[in] action
 534  *   Pointer to action specification.
 535  * @param[out] error
 536  *   Pointer to the error structure.
 537  *
 538  * @return
 539  *   0 on success, a negative errno value otherwise and rte_errno is set.
 540  */
 541 static int
 542 flow_dv_convert_action_modify_ipv6
 543                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 544                          const struct rte_flow_action *action,
 545                          struct rte_flow_error *error)
 546 {
 547         const struct rte_flow_action_set_ipv6 *conf =
 548                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 549         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 550         struct rte_flow_item_ipv6 ipv6;
 551         struct rte_flow_item_ipv6 ipv6_mask;
 552
 553         memset(&ipv6, 0, sizeof(ipv6));
 554         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 555         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 556                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 557                        sizeof(ipv6.hdr.src_addr));
 558                 memcpy(&ipv6_mask.hdr.src_addr,
 559                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 560                        sizeof(ipv6.hdr.src_addr));
 561         } else {
 562                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 563                        sizeof(ipv6.hdr.dst_addr));
 564                 memcpy(&ipv6_mask.hdr.dst_addr,
 565                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 566                        sizeof(ipv6.hdr.dst_addr));
 567         }
 568         item.spec = &ipv6;
 569         item.mask = &ipv6_mask;
 570         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 571                                              MLX5_MODIFICATION_TYPE_SET, error);
 572 }
 573
 574 /**
 575  * Convert modify-header set MAC address action to DV specification.
 576  *
 577  * @param[in,out] resource
 578  *   Pointer to the modify-header resource.
 579  * @param[in] action
 580  *   Pointer to action specification.
 581  * @param[out] error
 582  *   Pointer to the error structure.
 583  *
 584  * @return
 585  *   0 on success, a negative errno value otherwise and rte_errno is set.
 586  */
 587 static int
 588 flow_dv_convert_action_modify_mac
 589                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 590                          const struct rte_flow_action *action,
 591                          struct rte_flow_error *error)
 592 {
 593         const struct rte_flow_action_set_mac *conf =
 594                 (const struct rte_flow_action_set_mac *)(action->conf);
 595         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 596         struct rte_flow_item_eth eth;
 597         struct rte_flow_item_eth eth_mask;
 598
 599         memset(&eth, 0, sizeof(eth));
 600         memset(&eth_mask, 0, sizeof(eth_mask));
 601         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 602                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 603                        sizeof(eth.src.addr_bytes));
 604                 memcpy(&eth_mask.src.addr_bytes,
 605                        &rte_flow_item_eth_mask.src.addr_bytes,
 606                        sizeof(eth_mask.src.addr_bytes));
 607         } else {
 608                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 609                        sizeof(eth.dst.addr_bytes));
 610                 memcpy(&eth_mask.dst.addr_bytes,
 611                        &rte_flow_item_eth_mask.dst.addr_bytes,
 612                        sizeof(eth_mask.dst.addr_bytes));
 613         }
 614         item.spec = &eth;
 615         item.mask = &eth_mask;
 616         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 617                                              MLX5_MODIFICATION_TYPE_SET, error);
 618 }
 619
 620 /**
 621  * Convert modify-header set VLAN VID action to DV specification.
 622  *
 623  * @param[in,out] resource
 624  *   Pointer to the modify-header resource.
 625  * @param[in] action
 626  *   Pointer to action specification.
 627  * @param[out] error
 628  *   Pointer to the error structure.
 629  *
 630  * @return
 631  *   0 on success, a negative errno value otherwise and rte_errno is set.
 632  */
 633 static int
 634 flow_dv_convert_action_modify_vlan_vid
 635                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 636                          const struct rte_flow_action *action,
 637                          struct rte_flow_error *error)
 638 {
 639         const struct rte_flow_action_of_set_vlan_vid *conf =
 640                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 641         int i = resource->actions_num;
 642         struct mlx5_modification_cmd *actions = resource->actions;
 643         struct field_modify_info *field = modify_vlan_out_first_vid;
 644
 645         if (i >= MLX5_MAX_MODIFY_NUM)
 646                 return rte_flow_error_set(error, EINVAL,
 647                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 648                          "too many items to modify");
 649         actions[i] = (struct mlx5_modification_cmd) {
 650                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 651                 .field = field->id,
 652                 .length = field->size,
 653                 .offset = field->offset,
 654         };
 655         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 656         actions[i].data1 = conf->vlan_vid;
 657         actions[i].data1 = actions[i].data1 << 16;
 658         resource->actions_num = ++i;
 659         return 0;
 660 }
 661
 662 /**
 663  * Convert modify-header set TP action to DV specification.
 664  *
 665  * @param[in,out] resource
 666  *   Pointer to the modify-header resource.
 667  * @param[in] action
 668  *   Pointer to action specification.
 669  * @param[in] items
 670  *   Pointer to rte_flow_item objects list.
 671  * @param[in] attr
 672  *   Pointer to flow attributes structure.
 673  * @param[in] dev_flow
 674  *   Pointer to the sub flow.
 675  * @param[in] tunnel_decap
 676  *   Whether action is after tunnel decapsulation.
 677  * @param[out] error
 678  *   Pointer to the error structure.
 679  *
 680  * @return
 681  *   0 on success, a negative errno value otherwise and rte_errno is set.
 682  */
 683 static int
 684 flow_dv_convert_action_modify_tp
 685                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 686                          const struct rte_flow_action *action,
 687                          const struct rte_flow_item *items,
 688                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 689                          bool tunnel_decap, struct rte_flow_error *error)
 690 {
 691         const struct rte_flow_action_set_tp *conf =
 692                 (const struct rte_flow_action_set_tp *)(action->conf);
 693         struct rte_flow_item item;
 694         struct rte_flow_item_udp udp;
 695         struct rte_flow_item_udp udp_mask;
 696         struct rte_flow_item_tcp tcp;
 697         struct rte_flow_item_tcp tcp_mask;
 698         struct field_modify_info *field;
 699
 700         if (!attr->valid)
 701                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 702         if (attr->udp) {
 703                 memset(&udp, 0, sizeof(udp));
 704                 memset(&udp_mask, 0, sizeof(udp_mask));
 705                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 706                         udp.hdr.src_port = conf->port;
 707                         udp_mask.hdr.src_port =
 708                                         rte_flow_item_udp_mask.hdr.src_port;
 709                 } else {
 710                         udp.hdr.dst_port = conf->port;
 711                         udp_mask.hdr.dst_port =
 712                                         rte_flow_item_udp_mask.hdr.dst_port;
 713                 }
 714                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 715                 item.spec = &udp;
 716                 item.mask = &udp_mask;
 717                 field = modify_udp;
 718         } else {
 719                 MLX5_ASSERT(attr->tcp);
 720                 memset(&tcp, 0, sizeof(tcp));
 721                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 722                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 723                         tcp.hdr.src_port = conf->port;
 724                         tcp_mask.hdr.src_port =
 725                                         rte_flow_item_tcp_mask.hdr.src_port;
 726                 } else {
 727                         tcp.hdr.dst_port = conf->port;
 728                         tcp_mask.hdr.dst_port =
 729                                         rte_flow_item_tcp_mask.hdr.dst_port;
 730                 }
 731                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 732                 item.spec = &tcp;
 733                 item.mask = &tcp_mask;
 734                 field = modify_tcp;
 735         }
 736         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 737                                              MLX5_MODIFICATION_TYPE_SET, error);
 738 }
 739
 740 /**
 741  * Convert modify-header set TTL action to DV specification.
 742  *
 743  * @param[in,out] resource
 744  *   Pointer to the modify-header resource.
 745  * @param[in] action
 746  *   Pointer to action specification.
 747  * @param[in] items
 748  *   Pointer to rte_flow_item objects list.
 749  * @param[in] attr
 750  *   Pointer to flow attributes structure.
 751  * @param[in] dev_flow
 752  *   Pointer to the sub flow.
 753  * @param[in] tunnel_decap
 754  *   Whether action is after tunnel decapsulation.
 755  * @param[out] error
 756  *   Pointer to the error structure.
 757  *
 758  * @return
 759  *   0 on success, a negative errno value otherwise and rte_errno is set.
 760  */
 761 static int
 762 flow_dv_convert_action_modify_ttl
 763                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 764                          const struct rte_flow_action *action,
 765                          const struct rte_flow_item *items,
 766                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 767                          bool tunnel_decap, struct rte_flow_error *error)
 768 {
 769         const struct rte_flow_action_set_ttl *conf =
 770                 (const struct rte_flow_action_set_ttl *)(action->conf);
 771         struct rte_flow_item item;
 772         struct rte_flow_item_ipv4 ipv4;
 773         struct rte_flow_item_ipv4 ipv4_mask;
 774         struct rte_flow_item_ipv6 ipv6;
 775         struct rte_flow_item_ipv6 ipv6_mask;
 776         struct field_modify_info *field;
 777
 778         if (!attr->valid)
 779                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 780         if (attr->ipv4) {
 781                 memset(&ipv4, 0, sizeof(ipv4));
 782                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 783                 ipv4.hdr.time_to_live = conf->ttl_value;
 784                 ipv4_mask.hdr.time_to_live = 0xFF;
 785                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 786                 item.spec = &ipv4;
 787                 item.mask = &ipv4_mask;
 788                 field = modify_ipv4;
 789         } else {
 790                 MLX5_ASSERT(attr->ipv6);
 791                 memset(&ipv6, 0, sizeof(ipv6));
 792                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 793                 ipv6.hdr.hop_limits = conf->ttl_value;
 794                 ipv6_mask.hdr.hop_limits = 0xFF;
 795                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 796                 item.spec = &ipv6;
 797                 item.mask = &ipv6_mask;
 798                 field = modify_ipv6;
 799         }
 800         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 801                                              MLX5_MODIFICATION_TYPE_SET, error);
 802 }
 803
 804 /**
 805  * Convert modify-header decrement TTL action to DV specification.
 806  *
 807  * @param[in,out] resource
 808  *   Pointer to the modify-header resource.
 809  * @param[in] action
 810  *   Pointer to action specification.
 811  * @param[in] items
 812  *   Pointer to rte_flow_item objects list.
 813  * @param[in] attr
 814  *   Pointer to flow attributes structure.
 815  * @param[in] dev_flow
 816  *   Pointer to the sub flow.
 817  * @param[in] tunnel_decap
 818  *   Whether action is after tunnel decapsulation.
 819  * @param[out] error
 820  *   Pointer to the error structure.
 821  *
 822  * @return
 823  *   0 on success, a negative errno value otherwise and rte_errno is set.
 824  */
 825 static int
 826 flow_dv_convert_action_modify_dec_ttl
 827                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 828                          const struct rte_flow_item *items,
 829                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 830                          bool tunnel_decap, struct rte_flow_error *error)
 831 {
 832         struct rte_flow_item item;
 833         struct rte_flow_item_ipv4 ipv4;
 834         struct rte_flow_item_ipv4 ipv4_mask;
 835         struct rte_flow_item_ipv6 ipv6;
 836         struct rte_flow_item_ipv6 ipv6_mask;
 837         struct field_modify_info *field;
 838
 839         if (!attr->valid)
 840                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 841         if (attr->ipv4) {
 842                 memset(&ipv4, 0, sizeof(ipv4));
 843                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 844                 ipv4.hdr.time_to_live = 0xFF;
 845                 ipv4_mask.hdr.time_to_live = 0xFF;
 846                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 847                 item.spec = &ipv4;
 848                 item.mask = &ipv4_mask;
 849                 field = modify_ipv4;
 850         } else {
 851                 MLX5_ASSERT(attr->ipv6);
 852                 memset(&ipv6, 0, sizeof(ipv6));
 853                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 854                 ipv6.hdr.hop_limits = 0xFF;
 855                 ipv6_mask.hdr.hop_limits = 0xFF;
 856                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 857                 item.spec = &ipv6;
 858                 item.mask = &ipv6_mask;
 859                 field = modify_ipv6;
 860         }
 861         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 862                                              MLX5_MODIFICATION_TYPE_ADD, error);
 863 }
 864
 865 /**
 866  * Convert modify-header increment/decrement TCP Sequence number
 867  * to DV specification.
 868  *
 869  * @param[in,out] resource
 870  *   Pointer to the modify-header resource.
 871  * @param[in] action
 872  *   Pointer to action specification.
 873  * @param[out] error
 874  *   Pointer to the error structure.
 875  *
 876  * @return
 877  *   0 on success, a negative errno value otherwise and rte_errno is set.
 878  */
 879 static int
 880 flow_dv_convert_action_modify_tcp_seq
 881                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 882                          const struct rte_flow_action *action,
 883                          struct rte_flow_error *error)
 884 {
 885         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 886         uint64_t value = rte_be_to_cpu_32(*conf);
 887         struct rte_flow_item item;
 888         struct rte_flow_item_tcp tcp;
 889         struct rte_flow_item_tcp tcp_mask;
 890
 891         memset(&tcp, 0, sizeof(tcp));
 892         memset(&tcp_mask, 0, sizeof(tcp_mask));
 893         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 894                 /*
 895                  * The HW has no decrement operation, only increment operation.
 896                  * To simulate decrement X from Y using increment operation
 897                  * we need to add UINT32_MAX X times to Y.
 898                  * Each adding of UINT32_MAX decrements Y by 1.
 899                  */
 900                 value *= UINT32_MAX;
 901         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 902         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 903         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 904         item.spec = &tcp;
 905         item.mask = &tcp_mask;
 906         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 907                                              MLX5_MODIFICATION_TYPE_ADD, error);
 908 }
 909
 910 /**
 911  * Convert modify-header increment/decrement TCP Acknowledgment number
 912  * to DV specification.
 913  *
 914  * @param[in,out] resource
 915  *   Pointer to the modify-header resource.
 916  * @param[in] action
 917  *   Pointer to action specification.
 918  * @param[out] error
 919  *   Pointer to the error structure.
 920  *
 921  * @return
 922  *   0 on success, a negative errno value otherwise and rte_errno is set.
 923  */
 924 static int
 925 flow_dv_convert_action_modify_tcp_ack
 926                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 927                          const struct rte_flow_action *action,
 928                          struct rte_flow_error *error)
 929 {
 930         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 931         uint64_t value = rte_be_to_cpu_32(*conf);
 932         struct rte_flow_item item;
 933         struct rte_flow_item_tcp tcp;
 934         struct rte_flow_item_tcp tcp_mask;
 935
 936         memset(&tcp, 0, sizeof(tcp));
 937         memset(&tcp_mask, 0, sizeof(tcp_mask));
 938         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 939                 /*
 940                  * The HW has no decrement operation, only increment operation.
 941                  * To simulate decrement X from Y using increment operation
 942                  * we need to add UINT32_MAX X times to Y.
 943                  * Each adding of UINT32_MAX decrements Y by 1.
 944                  */
 945                 value *= UINT32_MAX;
 946         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 947         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 948         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 949         item.spec = &tcp;
 950         item.mask = &tcp_mask;
 951         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 952                                              MLX5_MODIFICATION_TYPE_ADD, error);
 953 }
 954
 955 static enum mlx5_modification_field reg_to_field[] = {
 956         [REG_NON] = MLX5_MODI_OUT_NONE,
 957         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 958         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 959         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 960         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 961         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 962         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 963         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 964         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 965         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 966         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 967 };
 968
 969 /**
 970  * Convert register set to DV specification.
 971  *
 972  * @param[in,out] resource
 973  *   Pointer to the modify-header resource.
 974  * @param[in] action
 975  *   Pointer to action specification.
 976  * @param[out] error
 977  *   Pointer to the error structure.
 978  *
 979  * @return
 980  *   0 on success, a negative errno value otherwise and rte_errno is set.
 981  */
 982 static int
 983 flow_dv_convert_action_set_reg
 984                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 985                          const struct rte_flow_action *action,
 986                          struct rte_flow_error *error)
 987 {
 988         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 989         struct mlx5_modification_cmd *actions = resource->actions;
 990         uint32_t i = resource->actions_num;
 991
 992         if (i >= MLX5_MAX_MODIFY_NUM)
 993                 return rte_flow_error_set(error, EINVAL,
 994                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 995                                           "too many items to modify");
 996         MLX5_ASSERT(conf->id != REG_NON);
 997         MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field));
 998         actions[i] = (struct mlx5_modification_cmd) {
 999                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1000                 .field = reg_to_field[conf->id],
1001         };
1002         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1003         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1004         ++i;
1005         resource->actions_num = i;
1006         return 0;
1007 }
1008
1009 /**
1010  * Convert SET_TAG action to DV specification.
1011  *
1012  * @param[in] dev
1013  *   Pointer to the rte_eth_dev structure.
1014  * @param[in,out] resource
1015  *   Pointer to the modify-header resource.
1016  * @param[in] conf
1017  *   Pointer to action specification.
1018  * @param[out] error
1019  *   Pointer to the error structure.
1020  *
1021  * @return
1022  *   0 on success, a negative errno value otherwise and rte_errno is set.
1023  */
1024 static int
1025 flow_dv_convert_action_set_tag
1026                         (struct rte_eth_dev *dev,
1027                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1028                          const struct rte_flow_action_set_tag *conf,
1029                          struct rte_flow_error *error)
1030 {
1031         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1032         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1033         struct rte_flow_item item = {
1034                 .spec = &data,
1035                 .mask = &mask,
1036         };
1037         struct field_modify_info reg_c_x[] = {
1038                 [1] = {0, 0, 0},
1039         };
1040         enum mlx5_modification_field reg_type;
1041         int ret;
1042
1043         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1044         if (ret < 0)
1045                 return ret;
1046         MLX5_ASSERT(ret != REG_NON);
1047         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1048         reg_type = reg_to_field[ret];
1049         MLX5_ASSERT(reg_type > 0);
1050         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1051         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1052                                              MLX5_MODIFICATION_TYPE_SET, error);
1053 }
1054
1055 /**
1056  * Convert internal COPY_REG action to DV specification.
1057  *
1058  * @param[in] dev
1059  *   Pointer to the rte_eth_dev structure.
1060  * @param[in,out] res
1061  *   Pointer to the modify-header resource.
1062  * @param[in] action
1063  *   Pointer to action specification.
1064  * @param[out] error
1065  *   Pointer to the error structure.
1066  *
1067  * @return
1068  *   0 on success, a negative errno value otherwise and rte_errno is set.
1069  */
1070 static int
1071 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1072                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1073                                  const struct rte_flow_action *action,
1074                                  struct rte_flow_error *error)
1075 {
1076         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1077         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1078         struct rte_flow_item item = {
1079                 .spec = NULL,
1080                 .mask = &mask,
1081         };
1082         struct field_modify_info reg_src[] = {
1083                 {4, 0, reg_to_field[conf->src]},
1084                 {0, 0, 0},
1085         };
1086         struct field_modify_info reg_dst = {
1087                 .offset = 0,
1088                 .id = reg_to_field[conf->dst],
1089         };
1090         /* Adjust reg_c[0] usage according to reported mask. */
1091         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1092                 struct mlx5_priv *priv = dev->data->dev_private;
1093                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1094
1095                 MLX5_ASSERT(reg_c0);
1096                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1097                 if (conf->dst == REG_C_0) {
1098                         /* Copy to reg_c[0], within mask only. */
1099                         reg_dst.offset = rte_bsf32(reg_c0);
1100                         /*
1101                          * Mask is ignoring the enianness, because
1102                          * there is no conversion in datapath.
1103                          */
1104 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1105                         /* Copy from destination lower bits to reg_c[0]. */
1106                         mask = reg_c0 >> reg_dst.offset;
1107 #else
1108                         /* Copy from destination upper bits to reg_c[0]. */
1109                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1110                                           rte_fls_u32(reg_c0));
1111 #endif
1112                 } else {
1113                         mask = rte_cpu_to_be_32(reg_c0);
1114 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1115                         /* Copy from reg_c[0] to destination lower bits. */
1116                         reg_dst.offset = 0;
1117 #else
1118                         /* Copy from reg_c[0] to destination upper bits. */
1119                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1120                                          (rte_fls_u32(reg_c0) -
1121                                           rte_bsf32(reg_c0));
1122 #endif
1123                 }
1124         }
1125         return flow_dv_convert_modify_action(&item,
1126                                              reg_src, &reg_dst, res,
1127                                              MLX5_MODIFICATION_TYPE_COPY,
1128                                              error);
1129 }
1130
1131 /**
1132  * Convert MARK action to DV specification. This routine is used
1133  * in extensive metadata only and requires metadata register to be
1134  * handled. In legacy mode hardware tag resource is engaged.
1135  *
1136  * @param[in] dev
1137  *   Pointer to the rte_eth_dev structure.
1138  * @param[in] conf
1139  *   Pointer to MARK action specification.
1140  * @param[in,out] resource
1141  *   Pointer to the modify-header resource.
1142  * @param[out] error
1143  *   Pointer to the error structure.
1144  *
1145  * @return
1146  *   0 on success, a negative errno value otherwise and rte_errno is set.
1147  */
1148 static int
1149 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1150                             const struct rte_flow_action_mark *conf,
1151                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1152                             struct rte_flow_error *error)
1153 {
1154         struct mlx5_priv *priv = dev->data->dev_private;
1155         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1156                                            priv->sh->dv_mark_mask);
1157         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1158         struct rte_flow_item item = {
1159                 .spec = &data,
1160                 .mask = &mask,
1161         };
1162         struct field_modify_info reg_c_x[] = {
1163                 [1] = {0, 0, 0},
1164         };
1165         int reg;
1166
1167         if (!mask)
1168                 return rte_flow_error_set(error, EINVAL,
1169                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1170                                           NULL, "zero mark action mask");
1171         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1172         if (reg < 0)
1173                 return reg;
1174         MLX5_ASSERT(reg > 0);
1175         if (reg == REG_C_0) {
1176                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1177                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1178
1179                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1180                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1181                 mask = rte_cpu_to_be_32(mask << shl_c0);
1182         }
1183         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1184         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1185                                              MLX5_MODIFICATION_TYPE_SET, error);
1186 }
1187
1188 /**
1189  * Get metadata register index for specified steering domain.
1190  *
1191  * @param[in] dev
1192  *   Pointer to the rte_eth_dev structure.
1193  * @param[in] attr
1194  *   Attributes of flow to determine steering domain.
1195  * @param[out] error
1196  *   Pointer to the error structure.
1197  *
1198  * @return
1199  *   positive index on success, a negative errno value otherwise
1200  *   and rte_errno is set.
1201  */
1202 static enum modify_reg
1203 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1204                          const struct rte_flow_attr *attr,
1205                          struct rte_flow_error *error)
1206 {
1207         int reg =
1208                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1209                                           MLX5_METADATA_FDB :
1210                                             attr->egress ?
1211                                             MLX5_METADATA_TX :
1212                                             MLX5_METADATA_RX, 0, error);
1213         if (reg < 0)
1214                 return rte_flow_error_set(error,
1215                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1216                                           NULL, "unavailable "
1217                                           "metadata register");
1218         return reg;
1219 }
1220
1221 /**
1222  * Convert SET_META action to DV specification.
1223  *
1224  * @param[in] dev
1225  *   Pointer to the rte_eth_dev structure.
1226  * @param[in,out] resource
1227  *   Pointer to the modify-header resource.
1228  * @param[in] attr
1229  *   Attributes of flow that includes this item.
1230  * @param[in] conf
1231  *   Pointer to action specification.
1232  * @param[out] error
1233  *   Pointer to the error structure.
1234  *
1235  * @return
1236  *   0 on success, a negative errno value otherwise and rte_errno is set.
1237  */
1238 static int
1239 flow_dv_convert_action_set_meta
1240                         (struct rte_eth_dev *dev,
1241                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1242                          const struct rte_flow_attr *attr,
1243                          const struct rte_flow_action_set_meta *conf,
1244                          struct rte_flow_error *error)
1245 {
1246         uint32_t data = conf->data;
1247         uint32_t mask = conf->mask;
1248         struct rte_flow_item item = {
1249                 .spec = &data,
1250                 .mask = &mask,
1251         };
1252         struct field_modify_info reg_c_x[] = {
1253                 [1] = {0, 0, 0},
1254         };
1255         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1256
1257         if (reg < 0)
1258                 return reg;
1259         /*
1260          * In datapath code there is no endianness
1261          * coversions for perfromance reasons, all
1262          * pattern conversions are done in rte_flow.
1263          */
1264         if (reg == REG_C_0) {
1265                 struct mlx5_priv *priv = dev->data->dev_private;
1266                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1267                 uint32_t shl_c0;
1268
1269                 MLX5_ASSERT(msk_c0);
1270 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1271                 shl_c0 = rte_bsf32(msk_c0);
1272 #else
1273                 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1274 #endif
1275                 mask <<= shl_c0;
1276                 data <<= shl_c0;
1277                 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1278         }
1279         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1280         /* The routine expects parameters in memory as big-endian ones. */
1281         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1282                                              MLX5_MODIFICATION_TYPE_SET, error);
1283 }
1284
1285 /**
1286  * Convert modify-header set IPv4 DSCP action to DV specification.
1287  *
1288  * @param[in,out] resource
1289  *   Pointer to the modify-header resource.
1290  * @param[in] action
1291  *   Pointer to action specification.
1292  * @param[out] error
1293  *   Pointer to the error structure.
1294  *
1295  * @return
1296  *   0 on success, a negative errno value otherwise and rte_errno is set.
1297  */
1298 static int
1299 flow_dv_convert_action_modify_ipv4_dscp
1300                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1301                          const struct rte_flow_action *action,
1302                          struct rte_flow_error *error)
1303 {
1304         const struct rte_flow_action_set_dscp *conf =
1305                 (const struct rte_flow_action_set_dscp *)(action->conf);
1306         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1307         struct rte_flow_item_ipv4 ipv4;
1308         struct rte_flow_item_ipv4 ipv4_mask;
1309
1310         memset(&ipv4, 0, sizeof(ipv4));
1311         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1312         ipv4.hdr.type_of_service = conf->dscp;
1313         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1314         item.spec = &ipv4;
1315         item.mask = &ipv4_mask;
1316         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1317                                              MLX5_MODIFICATION_TYPE_SET, error);
1318 }
1319
1320 /**
1321  * Convert modify-header set IPv6 DSCP action to DV specification.
1322  *
1323  * @param[in,out] resource
1324  *   Pointer to the modify-header resource.
1325  * @param[in] action
1326  *   Pointer to action specification.
1327  * @param[out] error
1328  *   Pointer to the error structure.
1329  *
1330  * @return
1331  *   0 on success, a negative errno value otherwise and rte_errno is set.
1332  */
1333 static int
1334 flow_dv_convert_action_modify_ipv6_dscp
1335                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1336                          const struct rte_flow_action *action,
1337                          struct rte_flow_error *error)
1338 {
1339         const struct rte_flow_action_set_dscp *conf =
1340                 (const struct rte_flow_action_set_dscp *)(action->conf);
1341         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1342         struct rte_flow_item_ipv6 ipv6;
1343         struct rte_flow_item_ipv6 ipv6_mask;
1344
1345         memset(&ipv6, 0, sizeof(ipv6));
1346         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1347         /*
1348          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1349          * rdma-core only accept the DSCP bits byte aligned start from
1350          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1351          * bits in IPv6 case as rdma-core requires byte aligned value.
1352          */
1353         ipv6.hdr.vtc_flow = conf->dscp;
1354         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1355         item.spec = &ipv6;
1356         item.mask = &ipv6_mask;
1357         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1358                                              MLX5_MODIFICATION_TYPE_SET, error);
1359 }
1360
1361 /**
1362  * Validate MARK item.
1363  *
1364  * @param[in] dev
1365  *   Pointer to the rte_eth_dev structure.
1366  * @param[in] item
1367  *   Item specification.
1368  * @param[in] attr
1369  *   Attributes of flow that includes this item.
1370  * @param[out] error
1371  *   Pointer to error structure.
1372  *
1373  * @return
1374  *   0 on success, a negative errno value otherwise and rte_errno is set.
1375  */
1376 static int
1377 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1378                            const struct rte_flow_item *item,
1379                            const struct rte_flow_attr *attr __rte_unused,
1380                            struct rte_flow_error *error)
1381 {
1382         struct mlx5_priv *priv = dev->data->dev_private;
1383         struct mlx5_dev_config *config = &priv->config;
1384         const struct rte_flow_item_mark *spec = item->spec;
1385         const struct rte_flow_item_mark *mask = item->mask;
1386         const struct rte_flow_item_mark nic_mask = {
1387                 .id = priv->sh->dv_mark_mask,
1388         };
1389         int ret;
1390
1391         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1392                 return rte_flow_error_set(error, ENOTSUP,
1393                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1394                                           "extended metadata feature"
1395                                           " isn't enabled");
1396         if (!mlx5_flow_ext_mreg_supported(dev))
1397                 return rte_flow_error_set(error, ENOTSUP,
1398                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1399                                           "extended metadata register"
1400                                           " isn't supported");
1401         if (!nic_mask.id)
1402                 return rte_flow_error_set(error, ENOTSUP,
1403                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1404                                           "extended metadata register"
1405                                           " isn't available");
1406         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1407         if (ret < 0)
1408                 return ret;
1409         if (!spec)
1410                 return rte_flow_error_set(error, EINVAL,
1411                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1412                                           item->spec,
1413                                           "data cannot be empty");
1414         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1415                 return rte_flow_error_set(error, EINVAL,
1416                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1417                                           &spec->id,
1418                                           "mark id exceeds the limit");
1419         if (!mask)
1420                 mask = &nic_mask;
1421         if (!mask->id)
1422                 return rte_flow_error_set(error, EINVAL,
1423                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1424                                         "mask cannot be zero");
1425
1426         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1427                                         (const uint8_t *)&nic_mask,
1428                                         sizeof(struct rte_flow_item_mark),
1429                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1430         if (ret < 0)
1431                 return ret;
1432         return 0;
1433 }
1434
1435 /**
1436  * Validate META item.
1437  *
1438  * @param[in] dev
1439  *   Pointer to the rte_eth_dev structure.
1440  * @param[in] item
1441  *   Item specification.
1442  * @param[in] attr
1443  *   Attributes of flow that includes this item.
1444  * @param[out] error
1445  *   Pointer to error structure.
1446  *
1447  * @return
1448  *   0 on success, a negative errno value otherwise and rte_errno is set.
1449  */
1450 static int
1451 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1452                            const struct rte_flow_item *item,
1453                            const struct rte_flow_attr *attr,
1454                            struct rte_flow_error *error)
1455 {
1456         struct mlx5_priv *priv = dev->data->dev_private;
1457         struct mlx5_dev_config *config = &priv->config;
1458         const struct rte_flow_item_meta *spec = item->spec;
1459         const struct rte_flow_item_meta *mask = item->mask;
1460         struct rte_flow_item_meta nic_mask = {
1461                 .data = UINT32_MAX
1462         };
1463         int reg;
1464         int ret;
1465
1466         if (!spec)
1467                 return rte_flow_error_set(error, EINVAL,
1468                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1469                                           item->spec,
1470                                           "data cannot be empty");
1471         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1472                 if (!mlx5_flow_ext_mreg_supported(dev))
1473                         return rte_flow_error_set(error, ENOTSUP,
1474                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1475                                           "extended metadata register"
1476                                           " isn't supported");
1477                 reg = flow_dv_get_metadata_reg(dev, attr, error);
1478                 if (reg < 0)
1479                         return reg;
1480                 if (reg == REG_B)
1481                         return rte_flow_error_set(error, ENOTSUP,
1482                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1483                                           "match on reg_b "
1484                                           "isn't supported");
1485                 if (reg != REG_A)
1486                         nic_mask.data = priv->sh->dv_meta_mask;
1487         } else if (attr->transfer) {
1488                 return rte_flow_error_set(error, ENOTSUP,
1489                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1490                                         "extended metadata feature "
1491                                         "should be enabled when "
1492                                         "meta item is requested "
1493                                         "with e-switch mode ");
1494         }
1495         if (!mask)
1496                 mask = &rte_flow_item_meta_mask;
1497         if (!mask->data)
1498                 return rte_flow_error_set(error, EINVAL,
1499                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1500                                         "mask cannot be zero");
1501
1502         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1503                                         (const uint8_t *)&nic_mask,
1504                                         sizeof(struct rte_flow_item_meta),
1505                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1506         return ret;
1507 }
1508
1509 /**
1510  * Validate TAG item.
1511  *
1512  * @param[in] dev
1513  *   Pointer to the rte_eth_dev structure.
1514  * @param[in] item
1515  *   Item specification.
1516  * @param[in] attr
1517  *   Attributes of flow that includes this item.
1518  * @param[out] error
1519  *   Pointer to error structure.
1520  *
1521  * @return
1522  *   0 on success, a negative errno value otherwise and rte_errno is set.
1523  */
1524 static int
1525 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1526                           const struct rte_flow_item *item,
1527                           const struct rte_flow_attr *attr __rte_unused,
1528                           struct rte_flow_error *error)
1529 {
1530         const struct rte_flow_item_tag *spec = item->spec;
1531         const struct rte_flow_item_tag *mask = item->mask;
1532         const struct rte_flow_item_tag nic_mask = {
1533                 .data = RTE_BE32(UINT32_MAX),
1534                 .index = 0xff,
1535         };
1536         int ret;
1537
1538         if (!mlx5_flow_ext_mreg_supported(dev))
1539                 return rte_flow_error_set(error, ENOTSUP,
1540                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1541                                           "extensive metadata register"
1542                                           " isn't supported");
1543         if (!spec)
1544                 return rte_flow_error_set(error, EINVAL,
1545                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1546                                           item->spec,
1547                                           "data cannot be empty");
1548         if (!mask)
1549                 mask = &rte_flow_item_tag_mask;
1550         if (!mask->data)
1551                 return rte_flow_error_set(error, EINVAL,
1552                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1553                                         "mask cannot be zero");
1554
1555         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1556                                         (const uint8_t *)&nic_mask,
1557                                         sizeof(struct rte_flow_item_tag),
1558                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1559         if (ret < 0)
1560                 return ret;
1561         if (mask->index != 0xff)
1562                 return rte_flow_error_set(error, EINVAL,
1563                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1564                                           "partial mask for tag index"
1565                                           " is not supported");
1566         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1567         if (ret < 0)
1568                 return ret;
1569         MLX5_ASSERT(ret != REG_NON);
1570         return 0;
1571 }
1572
1573 /**
1574  * Validate vport item.
1575  *
1576  * @param[in] dev
1577  *   Pointer to the rte_eth_dev structure.
1578  * @param[in] item
1579  *   Item specification.
1580  * @param[in] attr
1581  *   Attributes of flow that includes this item.
1582  * @param[in] item_flags
1583  *   Bit-fields that holds the items detected until now.
1584  * @param[out] error
1585  *   Pointer to error structure.
1586  *
1587  * @return
1588  *   0 on success, a negative errno value otherwise and rte_errno is set.
1589  */
1590 static int
1591 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1592                               const struct rte_flow_item *item,
1593                               const struct rte_flow_attr *attr,
1594                               uint64_t item_flags,
1595                               struct rte_flow_error *error)
1596 {
1597         const struct rte_flow_item_port_id *spec = item->spec;
1598         const struct rte_flow_item_port_id *mask = item->mask;
1599         const struct rte_flow_item_port_id switch_mask = {
1600                         .id = 0xffffffff,
1601         };
1602         struct mlx5_priv *esw_priv;
1603         struct mlx5_priv *dev_priv;
1604         int ret;
1605
1606         if (!attr->transfer)
1607                 return rte_flow_error_set(error, EINVAL,
1608                                           RTE_FLOW_ERROR_TYPE_ITEM,
1609                                           NULL,
1610                                           "match on port id is valid only"
1611                                           " when transfer flag is enabled");
1612         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1613                 return rte_flow_error_set(error, ENOTSUP,
1614                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1615                                           "multiple source ports are not"
1616                                           " supported");
1617         if (!mask)
1618                 mask = &switch_mask;
1619         if (mask->id != 0xffffffff)
1620                 return rte_flow_error_set(error, ENOTSUP,
1621                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1622                                            mask,
1623                                            "no support for partial mask on"
1624                                            " \"id\" field");
1625         ret = mlx5_flow_item_acceptable
1626                                 (item, (const uint8_t *)mask,
1627                                  (const uint8_t *)&rte_flow_item_port_id_mask,
1628                                  sizeof(struct rte_flow_item_port_id),
1629                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1630         if (ret)
1631                 return ret;
1632         if (!spec)
1633                 return 0;
1634         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1635         if (!esw_priv)
1636                 return rte_flow_error_set(error, rte_errno,
1637                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1638                                           "failed to obtain E-Switch info for"
1639                                           " port");
1640         dev_priv = mlx5_dev_to_eswitch_info(dev);
1641         if (!dev_priv)
1642                 return rte_flow_error_set(error, rte_errno,
1643                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1644                                           NULL,
1645                                           "failed to obtain E-Switch info");
1646         if (esw_priv->domain_id != dev_priv->domain_id)
1647                 return rte_flow_error_set(error, EINVAL,
1648                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1649                                           "cannot match on a port from a"
1650                                           " different E-Switch");
1651         return 0;
1652 }
1653
1654 /**
1655  * Validate VLAN item.
1656  *
1657  * @param[in] item
1658  *   Item specification.
1659  * @param[in] item_flags
1660  *   Bit-fields that holds the items detected until now.
1661  * @param[in] dev
1662  *   Ethernet device flow is being created on.
1663  * @param[out] error
1664  *   Pointer to error structure.
1665  *
1666  * @return
1667  *   0 on success, a negative errno value otherwise and rte_errno is set.
1668  */
1669 static int
1670 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
1671                            uint64_t item_flags,
1672                            struct rte_eth_dev *dev,
1673                            struct rte_flow_error *error)
1674 {
1675         const struct rte_flow_item_vlan *mask = item->mask;
1676         const struct rte_flow_item_vlan nic_mask = {
1677                 .tci = RTE_BE16(UINT16_MAX),
1678                 .inner_type = RTE_BE16(UINT16_MAX),
1679         };
1680         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1681         int ret;
1682         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1683                                         MLX5_FLOW_LAYER_INNER_L4) :
1684                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1685                                         MLX5_FLOW_LAYER_OUTER_L4);
1686         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1687                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1688
1689         if (item_flags & vlanm)
1690                 return rte_flow_error_set(error, EINVAL,
1691                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1692                                           "multiple VLAN layers not supported");
1693         else if ((item_flags & l34m) != 0)
1694                 return rte_flow_error_set(error, EINVAL,
1695                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1696                                           "VLAN cannot follow L3/L4 layer");
1697         if (!mask)
1698                 mask = &rte_flow_item_vlan_mask;
1699         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1700                                         (const uint8_t *)&nic_mask,
1701                                         sizeof(struct rte_flow_item_vlan),
1702                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1703         if (ret)
1704                 return ret;
1705         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1706                 struct mlx5_priv *priv = dev->data->dev_private;
1707
1708                 if (priv->vmwa_context) {
1709                         /*
1710                          * Non-NULL context means we have a virtual machine
1711                          * and SR-IOV enabled, we have to create VLAN interface
1712                          * to make hypervisor to setup E-Switch vport
1713                          * context correctly. We avoid creating the multiple
1714                          * VLAN interfaces, so we cannot support VLAN tag mask.
1715                          */
1716                         return rte_flow_error_set(error, EINVAL,
1717                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1718                                                   item,
1719                                                   "VLAN tag mask is not"
1720                                                   " supported in virtual"
1721                                                   " environment");
1722                 }
1723         }
1724         return 0;
1725 }
1726
1727 /*
1728  * GTP flags are contained in 1 byte of the format:
1729  * -------------------------------------------
1730  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
1731  * |-----------------------------------------|
1732  * | value | Version | PT | Res | E | S | PN |
1733  * -------------------------------------------
1734  *
1735  * Matching is supported only for GTP flags E, S, PN.
1736  */
1737 #define MLX5_GTP_FLAGS_MASK     0x07
1738
1739 /**
1740  * Validate GTP item.
1741  *
1742  * @param[in] dev
1743  *   Pointer to the rte_eth_dev structure.
1744  * @param[in] item
1745  *   Item specification.
1746  * @param[in] item_flags
1747  *   Bit-fields that holds the items detected until now.
1748  * @param[out] error
1749  *   Pointer to error structure.
1750  *
1751  * @return
1752  *   0 on success, a negative errno value otherwise and rte_errno is set.
1753  */
1754 static int
1755 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
1756                           const struct rte_flow_item *item,
1757                           uint64_t item_flags,
1758                           struct rte_flow_error *error)
1759 {
1760         struct mlx5_priv *priv = dev->data->dev_private;
1761         const struct rte_flow_item_gtp *spec = item->spec;
1762         const struct rte_flow_item_gtp *mask = item->mask;
1763         const struct rte_flow_item_gtp nic_mask = {
1764                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
1765                 .msg_type = 0xff,
1766                 .teid = RTE_BE32(0xffffffff),
1767         };
1768
1769         if (!priv->config.hca_attr.tunnel_stateless_gtp)
1770                 return rte_flow_error_set(error, ENOTSUP,
1771                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1772                                           "GTP support is not enabled");
1773         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1774                 return rte_flow_error_set(error, ENOTSUP,
1775                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1776                                           "multiple tunnel layers not"
1777                                           " supported");
1778         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1779                 return rte_flow_error_set(error, EINVAL,
1780                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1781                                           "no outer UDP layer found");
1782         if (!mask)
1783                 mask = &rte_flow_item_gtp_mask;
1784         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
1785                 return rte_flow_error_set(error, ENOTSUP,
1786                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1787                                           "Match is supported for GTP"
1788                                           " flags only");
1789         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1790                                          (const uint8_t *)&nic_mask,
1791                                          sizeof(struct rte_flow_item_gtp),
1792                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1793 }
1794
1795 /**
1796  * Validate IPV4 item.
1797  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
1798  * add specific validation of fragment_offset field,
1799  *
1800  * @param[in] item
1801  *   Item specification.
1802  * @param[in] item_flags
1803  *   Bit-fields that holds the items detected until now.
1804  * @param[out] error
1805  *   Pointer to error structure.
1806  *
1807  * @return
1808  *   0 on success, a negative errno value otherwise and rte_errno is set.
1809  */
1810 static int
1811 flow_dv_validate_item_ipv4(const struct rte_flow_item *item,
1812                            uint64_t item_flags,
1813                            uint64_t last_item,
1814                            uint16_t ether_type,
1815                            struct rte_flow_error *error)
1816 {
1817         int ret;
1818         const struct rte_flow_item_ipv4 *spec = item->spec;
1819         const struct rte_flow_item_ipv4 *last = item->last;
1820         const struct rte_flow_item_ipv4 *mask = item->mask;
1821         rte_be16_t fragment_offset_spec = 0;
1822         rte_be16_t fragment_offset_last = 0;
1823         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
1824                 .hdr = {
1825                         .src_addr = RTE_BE32(0xffffffff),
1826                         .dst_addr = RTE_BE32(0xffffffff),
1827                         .type_of_service = 0xff,
1828                         .fragment_offset = RTE_BE16(0xffff),
1829                         .next_proto_id = 0xff,
1830                         .time_to_live = 0xff,
1831                 },
1832         };
1833
1834         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
1835                                            ether_type, &nic_ipv4_mask,
1836                                            MLX5_ITEM_RANGE_ACCEPTED, error);
1837         if (ret < 0)
1838                 return ret;
1839         if (spec && mask)
1840                 fragment_offset_spec = spec->hdr.fragment_offset &
1841                                        mask->hdr.fragment_offset;
1842         if (!fragment_offset_spec)
1843                 return 0;
1844         /*
1845          * spec and mask are valid, enforce using full mask to make sure the
1846          * complete value is used correctly.
1847          */
1848         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1849                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1850                 return rte_flow_error_set(error, EINVAL,
1851                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1852                                           item, "must use full mask for"
1853                                           " fragment_offset");
1854         /*
1855          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
1856          * indicating this is 1st fragment of fragmented packet.
1857          * This is not yet supported in MLX5, return appropriate error message.
1858          */
1859         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
1860                 return rte_flow_error_set(error, ENOTSUP,
1861                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1862                                           "match on first fragment not "
1863                                           "supported");
1864         if (fragment_offset_spec && !last)
1865                 return rte_flow_error_set(error, ENOTSUP,
1866                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1867                                           "specified value not supported");
1868         /* spec and last are valid, validate the specified range. */
1869         fragment_offset_last = last->hdr.fragment_offset &
1870                                mask->hdr.fragment_offset;
1871         /*
1872          * Match on fragment_offset spec 0x2001 and last 0x3fff
1873          * means MF is 1 and frag-offset is > 0.
1874          * This packet is fragment 2nd and onward, excluding last.
1875          * This is not yet supported in MLX5, return appropriate
1876          * error message.
1877          */
1878         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
1879             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1880                 return rte_flow_error_set(error, ENOTSUP,
1881                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1882                                           last, "match on following "
1883                                           "fragments not supported");
1884         /*
1885          * Match on fragment_offset spec 0x0001 and last 0x1fff
1886          * means MF is 0 and frag-offset is > 0.
1887          * This packet is last fragment of fragmented packet.
1888          * This is not yet supported in MLX5, return appropriate
1889          * error message.
1890          */
1891         if (fragment_offset_spec == RTE_BE16(1) &&
1892             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
1893                 return rte_flow_error_set(error, ENOTSUP,
1894                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1895                                           last, "match on last "
1896                                           "fragment not supported");
1897         /*
1898          * Match on fragment_offset spec 0x0001 and last 0x3fff
1899          * means MF and/or frag-offset is not 0.
1900          * This is a fragmented packet.
1901          * Other range values are invalid and rejected.
1902          */
1903         if (!(fragment_offset_spec == RTE_BE16(1) &&
1904               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
1905                 return rte_flow_error_set(error, ENOTSUP,
1906                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
1907                                           "specified range not supported");
1908         return 0;
1909 }
1910
1911 /**
1912  * Validate IPV6 fragment extension item.
1913  *
1914  * @param[in] item
1915  *   Item specification.
1916  * @param[in] item_flags
1917  *   Bit-fields that holds the items detected until now.
1918  * @param[out] error
1919  *   Pointer to error structure.
1920  *
1921  * @return
1922  *   0 on success, a negative errno value otherwise and rte_errno is set.
1923  */
1924 static int
1925 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
1926                                     uint64_t item_flags,
1927                                     struct rte_flow_error *error)
1928 {
1929         const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
1930         const struct rte_flow_item_ipv6_frag_ext *last = item->last;
1931         const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
1932         rte_be16_t frag_data_spec = 0;
1933         rte_be16_t frag_data_last = 0;
1934         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1935         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1936                                       MLX5_FLOW_LAYER_OUTER_L4;
1937         int ret = 0;
1938         struct rte_flow_item_ipv6_frag_ext nic_mask = {
1939                 .hdr = {
1940                         .next_header = 0xff,
1941                         .frag_data = RTE_BE16(0xffff),
1942                 },
1943         };
1944
1945         if (item_flags & l4m)
1946                 return rte_flow_error_set(error, EINVAL,
1947                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1948                                           "ipv6 fragment extension item cannot "
1949                                           "follow L4 item.");
1950         if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
1951             (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
1952                 return rte_flow_error_set(error, EINVAL,
1953                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1954                                           "ipv6 fragment extension item must "
1955                                           "follow ipv6 item");
1956         if (spec && mask)
1957                 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
1958         if (!frag_data_spec)
1959                 return 0;
1960         /*
1961          * spec and mask are valid, enforce using full mask to make sure the
1962          * complete value is used correctly.
1963          */
1964         if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
1965                                 RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
1966                 return rte_flow_error_set(error, EINVAL,
1967                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1968                                           item, "must use full mask for"
1969                                           " frag_data");
1970         /*
1971          * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
1972          * This is 1st fragment of fragmented packet.
1973          */
1974         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
1975                 return rte_flow_error_set(error, ENOTSUP,
1976                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1977                                           "match on first fragment not "
1978                                           "supported");
1979         if (frag_data_spec && !last)
1980                 return rte_flow_error_set(error, EINVAL,
1981                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1982                                           "specified value not supported");
1983         ret = mlx5_flow_item_acceptable
1984                                 (item, (const uint8_t *)mask,
1985                                  (const uint8_t *)&nic_mask,
1986                                  sizeof(struct rte_flow_item_ipv6_frag_ext),
1987                                  MLX5_ITEM_RANGE_ACCEPTED, error);
1988         if (ret)
1989                 return ret;
1990         /* spec and last are valid, validate the specified range. */
1991         frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
1992         /*
1993          * Match on frag_data spec 0x0009 and last 0xfff9
1994          * means M is 1 and frag-offset is > 0.
1995          * This packet is fragment 2nd and onward, excluding last.
1996          * This is not yet supported in MLX5, return appropriate
1997          * error message.
1998          */
1999         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
2000                                        RTE_IPV6_EHDR_MF_MASK) &&
2001             frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2002                 return rte_flow_error_set(error, ENOTSUP,
2003                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2004                                           last, "match on following "
2005                                           "fragments not supported");
2006         /*
2007          * Match on frag_data spec 0x0008 and last 0xfff8
2008          * means M is 0 and frag-offset is > 0.
2009          * This packet is last fragment of fragmented packet.
2010          * This is not yet supported in MLX5, return appropriate
2011          * error message.
2012          */
2013         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
2014             frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
2015                 return rte_flow_error_set(error, ENOTSUP,
2016                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2017                                           last, "match on last "
2018                                           "fragment not supported");
2019         /* Other range values are invalid and rejected. */
2020         return rte_flow_error_set(error, EINVAL,
2021                                   RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2022                                   "specified range not supported");
2023 }
2024
2025 /**
2026  * Validate the pop VLAN action.
2027  *
2028  * @param[in] dev
2029  *   Pointer to the rte_eth_dev structure.
2030  * @param[in] action_flags
2031  *   Holds the actions detected until now.
2032  * @param[in] action
2033  *   Pointer to the pop vlan action.
2034  * @param[in] item_flags
2035  *   The items found in this flow rule.
2036  * @param[in] attr
2037  *   Pointer to flow attributes.
2038  * @param[out] error
2039  *   Pointer to error structure.
2040  *
2041  * @return
2042  *   0 on success, a negative errno value otherwise and rte_errno is set.
2043  */
2044 static int
2045 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
2046                                  uint64_t action_flags,
2047                                  const struct rte_flow_action *action,
2048                                  uint64_t item_flags,
2049                                  const struct rte_flow_attr *attr,
2050                                  struct rte_flow_error *error)
2051 {
2052         const struct mlx5_priv *priv = dev->data->dev_private;
2053
2054         (void)action;
2055         (void)attr;
2056         if (!priv->sh->pop_vlan_action)
2057                 return rte_flow_error_set(error, ENOTSUP,
2058                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2059                                           NULL,
2060                                           "pop vlan action is not supported");
2061         if (attr->egress)
2062                 return rte_flow_error_set(error, ENOTSUP,
2063                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2064                                           NULL,
2065                                           "pop vlan action not supported for "
2066                                           "egress");
2067         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
2068                 return rte_flow_error_set(error, ENOTSUP,
2069                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2070                                           "no support for multiple VLAN "
2071                                           "actions");
2072         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
2073         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
2074             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
2075                 return rte_flow_error_set(error, ENOTSUP,
2076                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2077                                           NULL,
2078                                           "cannot pop vlan after decap without "
2079                                           "match on inner vlan in the flow");
2080         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
2081         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
2082             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2083                 return rte_flow_error_set(error, ENOTSUP,
2084                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2085                                           NULL,
2086                                           "cannot pop vlan without a "
2087                                           "match on (outer) vlan in the flow");
2088         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2089                 return rte_flow_error_set(error, EINVAL,
2090                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2091                                           "wrong action order, port_id should "
2092                                           "be after pop VLAN action");
2093         if (!attr->transfer && priv->representor)
2094                 return rte_flow_error_set(error, ENOTSUP,
2095                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2096                                           "pop vlan action for VF representor "
2097                                           "not supported on NIC table");
2098         return 0;
2099 }
2100
2101 /**
2102  * Get VLAN default info from vlan match info.
2103  *
2104  * @param[in] items
2105  *   the list of item specifications.
2106  * @param[out] vlan
2107  *   pointer VLAN info to fill to.
2108  *
2109  * @return
2110  *   0 on success, a negative errno value otherwise and rte_errno is set.
2111  */
2112 static void
2113 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2114                                   struct rte_vlan_hdr *vlan)
2115 {
2116         const struct rte_flow_item_vlan nic_mask = {
2117                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2118                                 MLX5DV_FLOW_VLAN_VID_MASK),
2119                 .inner_type = RTE_BE16(0xffff),
2120         };
2121
2122         if (items == NULL)
2123                 return;
2124         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2125                 int type = items->type;
2126
2127                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2128                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2129                         break;
2130         }
2131         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2132                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2133                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2134
2135                 /* If VLAN item in pattern doesn't contain data, return here. */
2136                 if (!vlan_v)
2137                         return;
2138                 if (!vlan_m)
2139                         vlan_m = &nic_mask;
2140                 /* Only full match values are accepted */
2141                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2142                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2143                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2144                         vlan->vlan_tci |=
2145                                 rte_be_to_cpu_16(vlan_v->tci &
2146                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2147                 }
2148                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2149                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2150                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2151                         vlan->vlan_tci |=
2152                                 rte_be_to_cpu_16(vlan_v->tci &
2153                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2154                 }
2155                 if (vlan_m->inner_type == nic_mask.inner_type)
2156                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2157                                                            vlan_m->inner_type);
2158         }
2159 }
2160
2161 /**
2162  * Validate the push VLAN action.
2163  *
2164  * @param[in] dev
2165  *   Pointer to the rte_eth_dev structure.
2166  * @param[in] action_flags
2167  *   Holds the actions detected until now.
2168  * @param[in] item_flags
2169  *   The items found in this flow rule.
2170  * @param[in] action
2171  *   Pointer to the action structure.
2172  * @param[in] attr
2173  *   Pointer to flow attributes
2174  * @param[out] error
2175  *   Pointer to error structure.
2176  *
2177  * @return
2178  *   0 on success, a negative errno value otherwise and rte_errno is set.
2179  */
2180 static int
2181 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2182                                   uint64_t action_flags,
2183                                   const struct rte_flow_item_vlan *vlan_m,
2184                                   const struct rte_flow_action *action,
2185                                   const struct rte_flow_attr *attr,
2186                                   struct rte_flow_error *error)
2187 {
2188         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2189         const struct mlx5_priv *priv = dev->data->dev_private;
2190
2191         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2192             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2193                 return rte_flow_error_set(error, EINVAL,
2194                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2195                                           "invalid vlan ethertype");
2196         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2197                 return rte_flow_error_set(error, EINVAL,
2198                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2199                                           "wrong action order, port_id should "
2200                                           "be after push VLAN");
2201         if (!attr->transfer && priv->representor)
2202                 return rte_flow_error_set(error, ENOTSUP,
2203                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2204                                           "push vlan action for VF representor "
2205                                           "not supported on NIC table");
2206         if (vlan_m &&
2207             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2208             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2209                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2210             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2211             !(mlx5_flow_find_action
2212                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2213                 return rte_flow_error_set(error, EINVAL,
2214                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2215                                           "not full match mask on VLAN PCP and "
2216                                           "there is no of_set_vlan_pcp action, "
2217                                           "push VLAN action cannot figure out "
2218                                           "PCP value");
2219         if (vlan_m &&
2220             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2221             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2222                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2223             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2224             !(mlx5_flow_find_action
2225                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2226                 return rte_flow_error_set(error, EINVAL,
2227                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2228                                           "not full match mask on VLAN VID and "
2229                                           "there is no of_set_vlan_vid action, "
2230                                           "push VLAN action cannot figure out "
2231                                           "VID value");
2232         (void)attr;
2233         return 0;
2234 }
2235
2236 /**
2237  * Validate the set VLAN PCP.
2238  *
2239  * @param[in] action_flags
2240  *   Holds the actions detected until now.
2241  * @param[in] actions
2242  *   Pointer to the list of actions remaining in the flow rule.
2243  * @param[out] error
2244  *   Pointer to error structure.
2245  *
2246  * @return
2247  *   0 on success, a negative errno value otherwise and rte_errno is set.
2248  */
2249 static int
2250 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2251                                      const struct rte_flow_action actions[],
2252                                      struct rte_flow_error *error)
2253 {
2254         const struct rte_flow_action *action = actions;
2255         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2256
2257         if (conf->vlan_pcp > 7)
2258                 return rte_flow_error_set(error, EINVAL,
2259                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2260                                           "VLAN PCP value is too big");
2261         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2262                 return rte_flow_error_set(error, ENOTSUP,
2263                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2264                                           "set VLAN PCP action must follow "
2265                                           "the push VLAN action");
2266         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2267                 return rte_flow_error_set(error, ENOTSUP,
2268                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2269                                           "Multiple VLAN PCP modification are "
2270                                           "not supported");
2271         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2272                 return rte_flow_error_set(error, EINVAL,
2273                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2274                                           "wrong action order, port_id should "
2275                                           "be after set VLAN PCP");
2276         return 0;
2277 }
2278
2279 /**
2280  * Validate the set VLAN VID.
2281  *
2282  * @param[in] item_flags
2283  *   Holds the items detected in this rule.
2284  * @param[in] action_flags
2285  *   Holds the actions detected until now.
2286  * @param[in] actions
2287  *   Pointer to the list of actions remaining in the flow rule.
2288  * @param[out] error
2289  *   Pointer to error structure.
2290  *
2291  * @return
2292  *   0 on success, a negative errno value otherwise and rte_errno is set.
2293  */
2294 static int
2295 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2296                                      uint64_t action_flags,
2297                                      const struct rte_flow_action actions[],
2298                                      struct rte_flow_error *error)
2299 {
2300         const struct rte_flow_action *action = actions;
2301         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2302
2303         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2304                 return rte_flow_error_set(error, EINVAL,
2305                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2306                                           "VLAN VID value is too big");
2307         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2308             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2309                 return rte_flow_error_set(error, ENOTSUP,
2310                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2311                                           "set VLAN VID action must follow push"
2312                                           " VLAN action or match on VLAN item");
2313         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2314                 return rte_flow_error_set(error, ENOTSUP,
2315                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2316                                           "Multiple VLAN VID modifications are "
2317                                           "not supported");
2318         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2319                 return rte_flow_error_set(error, EINVAL,
2320                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2321                                           "wrong action order, port_id should "
2322                                           "be after set VLAN VID");
2323         return 0;
2324 }
2325
2326 /*
2327  * Validate the FLAG action.
2328  *
2329  * @param[in] dev
2330  *   Pointer to the rte_eth_dev structure.
2331  * @param[in] action_flags
2332  *   Holds the actions detected until now.
2333  * @param[in] attr
2334  *   Pointer to flow attributes
2335  * @param[out] error
2336  *   Pointer to error structure.
2337  *
2338  * @return
2339  *   0 on success, a negative errno value otherwise and rte_errno is set.
2340  */
2341 static int
2342 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
2343                              uint64_t action_flags,
2344                              const struct rte_flow_attr *attr,
2345                              struct rte_flow_error *error)
2346 {
2347         struct mlx5_priv *priv = dev->data->dev_private;
2348         struct mlx5_dev_config *config = &priv->config;
2349         int ret;
2350
2351         /* Fall back if no extended metadata register support. */
2352         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2353                 return mlx5_flow_validate_action_flag(action_flags, attr,
2354                                                       error);
2355         /* Extensive metadata mode requires registers. */
2356         if (!mlx5_flow_ext_mreg_supported(dev))
2357                 return rte_flow_error_set(error, ENOTSUP,
2358                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2359                                           "no metadata registers "
2360                                           "to support flag action");
2361         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
2362                 return rte_flow_error_set(error, ENOTSUP,
2363                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2364                                           "extended metadata register"
2365                                           " isn't available");
2366         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2367         if (ret < 0)
2368                 return ret;
2369         MLX5_ASSERT(ret > 0);
2370         if (action_flags & MLX5_FLOW_ACTION_MARK)
2371                 return rte_flow_error_set(error, EINVAL,
2372                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2373                                           "can't mark and flag in same flow");
2374         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2375                 return rte_flow_error_set(error, EINVAL,
2376                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2377                                           "can't have 2 flag"
2378                                           " actions in same flow");
2379         return 0;
2380 }
2381
2382 /**
2383  * Validate MARK action.
2384  *
2385  * @param[in] dev
2386  *   Pointer to the rte_eth_dev structure.
2387  * @param[in] action
2388  *   Pointer to action.
2389  * @param[in] action_flags
2390  *   Holds the actions detected until now.
2391  * @param[in] attr
2392  *   Pointer to flow attributes
2393  * @param[out] error
2394  *   Pointer to error structure.
2395  *
2396  * @return
2397  *   0 on success, a negative errno value otherwise and rte_errno is set.
2398  */
2399 static int
2400 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
2401                              const struct rte_flow_action *action,
2402                              uint64_t action_flags,
2403                              const struct rte_flow_attr *attr,
2404                              struct rte_flow_error *error)
2405 {
2406         struct mlx5_priv *priv = dev->data->dev_private;
2407         struct mlx5_dev_config *config = &priv->config;
2408         const struct rte_flow_action_mark *mark = action->conf;
2409         int ret;
2410
2411         /* Fall back if no extended metadata register support. */
2412         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2413                 return mlx5_flow_validate_action_mark(action, action_flags,
2414                                                       attr, error);
2415         /* Extensive metadata mode requires registers. */
2416         if (!mlx5_flow_ext_mreg_supported(dev))
2417                 return rte_flow_error_set(error, ENOTSUP,
2418                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2419                                           "no metadata registers "
2420                                           "to support mark action");
2421         if (!priv->sh->dv_mark_mask)
2422                 return rte_flow_error_set(error, ENOTSUP,
2423                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2424                                           "extended metadata register"
2425                                           " isn't available");
2426         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2427         if (ret < 0)
2428                 return ret;
2429         MLX5_ASSERT(ret > 0);
2430         if (!mark)
2431                 return rte_flow_error_set(error, EINVAL,
2432                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2433                                           "configuration cannot be null");
2434         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
2435                 return rte_flow_error_set(error, EINVAL,
2436                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2437                                           &mark->id,
2438                                           "mark id exceeds the limit");
2439         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2440                 return rte_flow_error_set(error, EINVAL,
2441                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2442                                           "can't flag and mark in same flow");
2443         if (action_flags & MLX5_FLOW_ACTION_MARK)
2444                 return rte_flow_error_set(error, EINVAL,
2445                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2446                                           "can't have 2 mark actions in same"
2447                                           " flow");
2448         return 0;
2449 }
2450
2451 /**
2452  * Validate SET_META action.
2453  *
2454  * @param[in] dev
2455  *   Pointer to the rte_eth_dev structure.
2456  * @param[in] action
2457  *   Pointer to the action structure.
2458  * @param[in] action_flags
2459  *   Holds the actions detected until now.
2460  * @param[in] attr
2461  *   Pointer to flow attributes
2462  * @param[out] error
2463  *   Pointer to error structure.
2464  *
2465  * @return
2466  *   0 on success, a negative errno value otherwise and rte_errno is set.
2467  */
2468 static int
2469 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
2470                                  const struct rte_flow_action *action,
2471                                  uint64_t action_flags __rte_unused,
2472                                  const struct rte_flow_attr *attr,
2473                                  struct rte_flow_error *error)
2474 {
2475         const struct rte_flow_action_set_meta *conf;
2476         uint32_t nic_mask = UINT32_MAX;
2477         int reg;
2478
2479         if (!mlx5_flow_ext_mreg_supported(dev))
2480                 return rte_flow_error_set(error, ENOTSUP,
2481                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2482                                           "extended metadata register"
2483                                           " isn't supported");
2484         reg = flow_dv_get_metadata_reg(dev, attr, error);
2485         if (reg < 0)
2486                 return reg;
2487         if (reg != REG_A && reg != REG_B) {
2488                 struct mlx5_priv *priv = dev->data->dev_private;
2489
2490                 nic_mask = priv->sh->dv_meta_mask;
2491         }
2492         if (!(action->conf))
2493                 return rte_flow_error_set(error, EINVAL,
2494                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2495                                           "configuration cannot be null");
2496         conf = (const struct rte_flow_action_set_meta *)action->conf;
2497         if (!conf->mask)
2498                 return rte_flow_error_set(error, EINVAL,
2499                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2500                                           "zero mask doesn't have any effect");
2501         if (conf->mask & ~nic_mask)
2502                 return rte_flow_error_set(error, EINVAL,
2503                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2504                                           "meta data must be within reg C0");
2505         return 0;
2506 }
2507
2508 /**
2509  * Validate SET_TAG action.
2510  *
2511  * @param[in] dev
2512  *   Pointer to the rte_eth_dev structure.
2513  * @param[in] action
2514  *   Pointer to the action structure.
2515  * @param[in] action_flags
2516  *   Holds the actions detected until now.
2517  * @param[in] attr
2518  *   Pointer to flow attributes
2519  * @param[out] error
2520  *   Pointer to error structure.
2521  *
2522  * @return
2523  *   0 on success, a negative errno value otherwise and rte_errno is set.
2524  */
2525 static int
2526 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
2527                                 const struct rte_flow_action *action,
2528                                 uint64_t action_flags,
2529                                 const struct rte_flow_attr *attr,
2530                                 struct rte_flow_error *error)
2531 {
2532         const struct rte_flow_action_set_tag *conf;
2533         const uint64_t terminal_action_flags =
2534                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
2535                 MLX5_FLOW_ACTION_RSS;
2536         int ret;
2537
2538         if (!mlx5_flow_ext_mreg_supported(dev))
2539                 return rte_flow_error_set(error, ENOTSUP,
2540                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2541                                           "extensive metadata register"
2542                                           " isn't supported");
2543         if (!(action->conf))
2544                 return rte_flow_error_set(error, EINVAL,
2545                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2546                                           "configuration cannot be null");
2547         conf = (const struct rte_flow_action_set_tag *)action->conf;
2548         if (!conf->mask)
2549                 return rte_flow_error_set(error, EINVAL,
2550                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2551                                           "zero mask doesn't have any effect");
2552         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
2553         if (ret < 0)
2554                 return ret;
2555         if (!attr->transfer && attr->ingress &&
2556             (action_flags & terminal_action_flags))
2557                 return rte_flow_error_set(error, EINVAL,
2558                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2559                                           "set_tag has no effect"
2560                                           " with terminal actions");
2561         return 0;
2562 }
2563
2564 /**
2565  * Validate count action.
2566  *
2567  * @param[in] dev
2568  *   Pointer to rte_eth_dev structure.
2569  * @param[out] error
2570  *   Pointer to error structure.
2571  *
2572  * @return
2573  *   0 on success, a negative errno value otherwise and rte_errno is set.
2574  */
2575 static int
2576 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2577                               struct rte_flow_error *error)
2578 {
2579         struct mlx5_priv *priv = dev->data->dev_private;
2580
2581         if (!priv->config.devx)
2582                 goto notsup_err;
2583 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2584         return 0;
2585 #endif
2586 notsup_err:
2587         return rte_flow_error_set
2588                       (error, ENOTSUP,
2589                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2590                        NULL,
2591                        "count action not supported");
2592 }
2593
2594 /**
2595  * Validate the L2 encap action.
2596  *
2597  * @param[in] dev
2598  *   Pointer to the rte_eth_dev structure.
2599  * @param[in] action_flags
2600  *   Holds the actions detected until now.
2601  * @param[in] action
2602  *   Pointer to the action structure.
2603  * @param[in] attr
2604  *   Pointer to flow attributes.
2605  * @param[out] error
2606  *   Pointer to error structure.
2607  *
2608  * @return
2609  *   0 on success, a negative errno value otherwise and rte_errno is set.
2610  */
2611 static int
2612 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
2613                                  uint64_t action_flags,
2614                                  const struct rte_flow_action *action,
2615                                  const struct rte_flow_attr *attr,
2616                                  struct rte_flow_error *error)
2617 {
2618         const struct mlx5_priv *priv = dev->data->dev_private;
2619
2620         if (!(action->conf))
2621                 return rte_flow_error_set(error, EINVAL,
2622                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2623                                           "configuration cannot be null");
2624         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
2625                 return rte_flow_error_set(error, EINVAL,
2626                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2627                                           "can only have a single encap action "
2628                                           "in a flow");
2629         if (!attr->transfer && priv->representor)
2630                 return rte_flow_error_set(error, ENOTSUP,
2631                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2632                                           "encap action for VF representor "
2633                                           "not supported on NIC table");
2634         return 0;
2635 }
2636
2637 /**
2638  * Validate a decap action.
2639  *
2640  * @param[in] dev
2641  *   Pointer to the rte_eth_dev structure.
2642  * @param[in] action_flags
2643  *   Holds the actions detected until now.
2644  * @param[in] attr
2645  *   Pointer to flow attributes
2646  * @param[out] error
2647  *   Pointer to error structure.
2648  *
2649  * @return
2650  *   0 on success, a negative errno value otherwise and rte_errno is set.
2651  */
2652 static int
2653 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
2654                               uint64_t action_flags,
2655                               const struct rte_flow_attr *attr,
2656                               struct rte_flow_error *error)
2657 {
2658         const struct mlx5_priv *priv = dev->data->dev_private;
2659
2660         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
2661             !priv->config.decap_en)
2662                 return rte_flow_error_set(error, ENOTSUP,
2663                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2664                                           "decap is not enabled");
2665         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
2666                 return rte_flow_error_set(error, ENOTSUP,
2667                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2668                                           action_flags &
2669                                           MLX5_FLOW_ACTION_DECAP ? "can only "
2670                                           "have a single decap action" : "decap "
2671                                           "after encap is not supported");
2672         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2673                 return rte_flow_error_set(error, EINVAL,
2674                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2675                                           "can't have decap action after"
2676                                           " modify action");
2677         if (attr->egress)
2678                 return rte_flow_error_set(error, ENOTSUP,
2679                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2680                                           NULL,
2681                                           "decap action not supported for "
2682                                           "egress");
2683         if (!attr->transfer && priv->representor)
2684                 return rte_flow_error_set(error, ENOTSUP,
2685                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2686                                           "decap action for VF representor "
2687                                           "not supported on NIC table");
2688         return 0;
2689 }
2690
2691 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
2692
2693 /**
2694  * Validate the raw encap and decap actions.
2695  *
2696  * @param[in] dev
2697  *   Pointer to the rte_eth_dev structure.
2698  * @param[in] decap
2699  *   Pointer to the decap action.
2700  * @param[in] encap
2701  *   Pointer to the encap action.
2702  * @param[in] attr
2703  *   Pointer to flow attributes
2704  * @param[in/out] action_flags
2705  *   Holds the actions detected until now.
2706  * @param[out] actions_n
2707  *   pointer to the number of actions counter.
2708  * @param[out] error
2709  *   Pointer to error structure.
2710  *
2711  * @return
2712  *   0 on success, a negative errno value otherwise and rte_errno is set.
2713  */
2714 static int
2715 flow_dv_validate_action_raw_encap_decap
2716         (struct rte_eth_dev *dev,
2717          const struct rte_flow_action_raw_decap *decap,
2718          const struct rte_flow_action_raw_encap *encap,
2719          const struct rte_flow_attr *attr, uint64_t *action_flags,
2720          int *actions_n, struct rte_flow_error *error)
2721 {
2722         const struct mlx5_priv *priv = dev->data->dev_private;
2723         int ret;
2724
2725         if (encap && (!encap->size || !encap->data))
2726                 return rte_flow_error_set(error, EINVAL,
2727                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2728                                           "raw encap data cannot be empty");
2729         if (decap && encap) {
2730                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
2731                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2732                         /* L3 encap. */
2733                         decap = NULL;
2734                 else if (encap->size <=
2735                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2736                            decap->size >
2737                            MLX5_ENCAPSULATION_DECISION_SIZE)
2738                         /* L3 decap. */
2739                         encap = NULL;
2740                 else if (encap->size >
2741                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2742                            decap->size >
2743                            MLX5_ENCAPSULATION_DECISION_SIZE)
2744                         /* 2 L2 actions: encap and decap. */
2745                         ;
2746                 else
2747                         return rte_flow_error_set(error,
2748                                 ENOTSUP,
2749                                 RTE_FLOW_ERROR_TYPE_ACTION,
2750                                 NULL, "unsupported too small "
2751                                 "raw decap and too small raw "
2752                                 "encap combination");
2753         }
2754         if (decap) {
2755                 ret = flow_dv_validate_action_decap(dev, *action_flags, attr,
2756                                                     error);
2757                 if (ret < 0)
2758                         return ret;
2759                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
2760                 ++(*actions_n);
2761         }
2762         if (encap) {
2763                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
2764                         return rte_flow_error_set(error, ENOTSUP,
2765                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2766                                                   NULL,
2767                                                   "small raw encap size");
2768                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
2769                         return rte_flow_error_set(error, EINVAL,
2770                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2771                                                   NULL,
2772                                                   "more than one encap action");
2773                 if (!attr->transfer && priv->representor)
2774                         return rte_flow_error_set
2775                                         (error, ENOTSUP,
2776                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2777                                          "encap action for VF representor "
2778                                          "not supported on NIC table");
2779                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
2780                 ++(*actions_n);
2781         }
2782         return 0;
2783 }
2784
2785 /**
2786  * Match encap_decap resource.
2787  *
2788  * @param entry
2789  *   Pointer to exist resource entry object.
2790  * @param ctx
2791  *   Pointer to new encap_decap resource.
2792  *
2793  * @return
2794  *   0 on matching, -1 otherwise.
2795  */
2796 static int
2797 flow_dv_encap_decap_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
2798 {
2799         struct mlx5_flow_dv_encap_decap_resource *resource;
2800         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2801
2802         resource = (struct mlx5_flow_dv_encap_decap_resource *)ctx;
2803         cache_resource = container_of(entry,
2804                                       struct mlx5_flow_dv_encap_decap_resource,
2805                                       entry);
2806         if (resource->entry.key == cache_resource->entry.key &&
2807             resource->reformat_type == cache_resource->reformat_type &&
2808             resource->ft_type == cache_resource->ft_type &&
2809             resource->flags == cache_resource->flags &&
2810             resource->size == cache_resource->size &&
2811             !memcmp((const void *)resource->buf,
2812                     (const void *)cache_resource->buf,
2813                     resource->size))
2814                 return 0;
2815         return -1;
2816 }
2817
2818 /**
2819  * Find existing encap/decap resource or create and register a new one.
2820  *
2821  * @param[in, out] dev
2822  *   Pointer to rte_eth_dev structure.
2823  * @param[in, out] resource
2824  *   Pointer to encap/decap resource.
2825  * @parm[in, out] dev_flow
2826  *   Pointer to the dev_flow.
2827  * @param[out] error
2828  *   pointer to error structure.
2829  *
2830  * @return
2831  *   0 on success otherwise -errno and errno is set.
2832  */
2833 static int
2834 flow_dv_encap_decap_resource_register
2835                         (struct rte_eth_dev *dev,
2836                          struct mlx5_flow_dv_encap_decap_resource *resource,
2837                          struct mlx5_flow *dev_flow,
2838                          struct rte_flow_error *error)
2839 {
2840         struct mlx5_priv *priv = dev->data->dev_private;
2841         struct mlx5_dev_ctx_shared *sh = priv->sh;
2842         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2843         struct mlx5dv_dr_domain *domain;
2844         struct mlx5_hlist_entry *entry;
2845         union mlx5_flow_encap_decap_key encap_decap_key = {
2846                 {
2847                         .ft_type = resource->ft_type,
2848                         .refmt_type = resource->reformat_type,
2849                         .buf_size = resource->size,
2850                         .table_level = !!dev_flow->dv.group,
2851                         .cksum = 0,
2852                 }
2853         };
2854         int ret;
2855
2856         resource->flags = dev_flow->dv.group ? 0 : 1;
2857         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2858                 domain = sh->fdb_domain;
2859         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2860                 domain = sh->rx_domain;
2861         else
2862                 domain = sh->tx_domain;
2863         encap_decap_key.cksum = __rte_raw_cksum(resource->buf,
2864                                                 resource->size, 0);
2865         resource->entry.key = encap_decap_key.v64;
2866         /* Lookup a matching resource from cache. */
2867         entry = mlx5_hlist_lookup_ex(sh->encaps_decaps, resource->entry.key,
2868                                      flow_dv_encap_decap_resource_match,
2869                                      (void *)resource);
2870         if (entry) {
2871                 cache_resource = container_of(entry,
2872                         struct mlx5_flow_dv_encap_decap_resource, entry);
2873                 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2874                         (void *)cache_resource,
2875                         rte_atomic32_read(&cache_resource->refcnt));
2876                 rte_atomic32_inc(&cache_resource->refcnt);
2877                 dev_flow->handle->dvh.rix_encap_decap = cache_resource->idx;
2878                 dev_flow->dv.encap_decap = cache_resource;
2879                 return 0;
2880         }
2881         /* Register new encap/decap resource. */
2882         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2883                                        &dev_flow->handle->dvh.rix_encap_decap);
2884         if (!cache_resource)
2885                 return rte_flow_error_set(error, ENOMEM,
2886                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2887                                           "cannot allocate resource memory");
2888         *cache_resource = *resource;
2889         cache_resource->idx = dev_flow->handle->dvh.rix_encap_decap;
2890         ret = mlx5_flow_os_create_flow_action_packet_reformat
2891                                         (sh->ctx, domain, cache_resource,
2892                                          &cache_resource->action);
2893         if (ret) {
2894                 mlx5_free(cache_resource);
2895                 return rte_flow_error_set(error, ENOMEM,
2896                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2897                                           NULL, "cannot create action");
2898         }
2899         rte_atomic32_init(&cache_resource->refcnt);
2900         rte_atomic32_inc(&cache_resource->refcnt);
2901         if (mlx5_hlist_insert_ex(sh->encaps_decaps, &cache_resource->entry,
2902                                  flow_dv_encap_decap_resource_match,
2903                                  (void *)cache_resource)) {
2904                 claim_zero(mlx5_flow_os_destroy_flow_action
2905                                                 (cache_resource->action));
2906                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2907                                 cache_resource->idx);
2908                 return rte_flow_error_set(error, EEXIST,
2909                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2910                                           NULL, "action exist");
2911         }
2912         dev_flow->dv.encap_decap = cache_resource;
2913         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2914                 (void *)cache_resource,
2915                 rte_atomic32_read(&cache_resource->refcnt));
2916         return 0;
2917 }
2918
2919 /**
2920  * Find existing table jump resource or create and register a new one.
2921  *
2922  * @param[in, out] dev
2923  *   Pointer to rte_eth_dev structure.
2924  * @param[in, out] tbl
2925  *   Pointer to flow table resource.
2926  * @parm[in, out] dev_flow
2927  *   Pointer to the dev_flow.
2928  * @param[out] error
2929  *   pointer to error structure.
2930  *
2931  * @return
2932  *   0 on success otherwise -errno and errno is set.
2933  */
2934 static int
2935 flow_dv_jump_tbl_resource_register
2936                         (struct rte_eth_dev *dev __rte_unused,
2937                          struct mlx5_flow_tbl_resource *tbl,
2938                          struct mlx5_flow *dev_flow,
2939                          struct rte_flow_error *error)
2940 {
2941         struct mlx5_flow_tbl_data_entry *tbl_data =
2942                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2943         int cnt, ret;
2944
2945         MLX5_ASSERT(tbl);
2946         cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
2947         if (!cnt) {
2948                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
2949                                 (tbl->obj, &tbl_data->jump.action);
2950                 if (ret)
2951                         return rte_flow_error_set(error, ENOMEM,
2952                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2953                                         NULL, "cannot create jump action");
2954                 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2955                         (void *)&tbl_data->jump, cnt);
2956         } else {
2957                 /* old jump should not make the table ref++. */
2958                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
2959                 MLX5_ASSERT(tbl_data->jump.action);
2960                 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2961                         (void *)&tbl_data->jump, cnt);
2962         }
2963         rte_atomic32_inc(&tbl_data->jump.refcnt);
2964         dev_flow->handle->rix_jump = tbl_data->idx;
2965         dev_flow->dv.jump = &tbl_data->jump;
2966         return 0;
2967 }
2968
2969 /**
2970  * Find existing default miss resource or create and register a new one.
2971  *
2972  * @param[in, out] dev
2973  *   Pointer to rte_eth_dev structure.
2974  * @param[out] error
2975  *   pointer to error structure.
2976  *
2977  * @return
2978  *   0 on success otherwise -errno and errno is set.
2979  */
2980 static int
2981 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
2982                 struct rte_flow_error *error)
2983 {
2984         struct mlx5_priv *priv = dev->data->dev_private;
2985         struct mlx5_dev_ctx_shared *sh = priv->sh;
2986         struct mlx5_flow_default_miss_resource *cache_resource =
2987                         &sh->default_miss;
2988         int cnt = rte_atomic32_read(&cache_resource->refcnt);
2989
2990         if (!cnt) {
2991                 MLX5_ASSERT(cache_resource->action);
2992                 cache_resource->action =
2993                 mlx5_glue->dr_create_flow_action_default_miss();
2994                 if (!cache_resource->action)
2995                         return rte_flow_error_set(error, ENOMEM,
2996                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2997                                         "cannot create default miss action");
2998                 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++",
2999                                 (void *)cache_resource->action, cnt);
3000         }
3001         rte_atomic32_inc(&cache_resource->refcnt);
3002         return 0;
3003 }
3004
3005 /**
3006  * Find existing table port ID resource or create and register a new one.
3007  *
3008  * @param[in, out] dev
3009  *   Pointer to rte_eth_dev structure.
3010  * @param[in, out] resource
3011  *   Pointer to port ID action resource.
3012  * @parm[in, out] dev_flow
3013  *   Pointer to the dev_flow.
3014  * @param[out] error
3015  *   pointer to error structure.
3016  *
3017  * @return
3018  *   0 on success otherwise -errno and errno is set.
3019  */
3020 static int
3021 flow_dv_port_id_action_resource_register
3022                         (struct rte_eth_dev *dev,
3023                          struct mlx5_flow_dv_port_id_action_resource *resource,
3024                          struct mlx5_flow *dev_flow,
3025                          struct rte_flow_error *error)
3026 {
3027         struct mlx5_priv *priv = dev->data->dev_private;
3028         struct mlx5_dev_ctx_shared *sh = priv->sh;
3029         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
3030         uint32_t idx = 0;
3031         int ret;
3032
3033         /* Lookup a matching resource from cache. */
3034         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list,
3035                       idx, cache_resource, next) {
3036                 if (resource->port_id == cache_resource->port_id) {
3037                         DRV_LOG(DEBUG, "port id action resource resource %p: "
3038                                 "refcnt %d++",
3039                                 (void *)cache_resource,
3040                                 rte_atomic32_read(&cache_resource->refcnt));
3041                         rte_atomic32_inc(&cache_resource->refcnt);
3042                         dev_flow->handle->rix_port_id_action = idx;
3043                         dev_flow->dv.port_id_action = cache_resource;
3044                         return 0;
3045                 }
3046         }
3047         /* Register new port id action resource. */
3048         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID],
3049                                        &dev_flow->handle->rix_port_id_action);
3050         if (!cache_resource)
3051                 return rte_flow_error_set(error, ENOMEM,
3052                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3053                                           "cannot allocate resource memory");
3054         *cache_resource = *resource;
3055         ret = mlx5_flow_os_create_flow_action_dest_port
3056                                 (priv->sh->fdb_domain, resource->port_id,
3057                                  &cache_resource->action);
3058         if (ret) {
3059                 mlx5_free(cache_resource);
3060                 return rte_flow_error_set(error, ENOMEM,
3061                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3062                                           NULL, "cannot create action");
3063         }
3064         rte_atomic32_init(&cache_resource->refcnt);
3065         rte_atomic32_inc(&cache_resource->refcnt);
3066         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
3067                      dev_flow->handle->rix_port_id_action, cache_resource,
3068                      next);
3069         dev_flow->dv.port_id_action = cache_resource;
3070         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
3071                 (void *)cache_resource,
3072                 rte_atomic32_read(&cache_resource->refcnt));
3073         return 0;
3074 }
3075
3076 /**
3077  * Find existing push vlan resource or create and register a new one.
3078  *
3079  * @param [in, out] dev
3080  *   Pointer to rte_eth_dev structure.
3081  * @param[in, out] resource
3082  *   Pointer to port ID action resource.
3083  * @parm[in, out] dev_flow
3084  *   Pointer to the dev_flow.
3085  * @param[out] error
3086  *   pointer to error structure.
3087  *
3088  * @return
3089  *   0 on success otherwise -errno and errno is set.
3090  */
3091 static int
3092 flow_dv_push_vlan_action_resource_register
3093                        (struct rte_eth_dev *dev,
3094                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
3095                         struct mlx5_flow *dev_flow,
3096                         struct rte_flow_error *error)
3097 {
3098         struct mlx5_priv *priv = dev->data->dev_private;
3099         struct mlx5_dev_ctx_shared *sh = priv->sh;
3100         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
3101         struct mlx5dv_dr_domain *domain;
3102         uint32_t idx = 0;
3103         int ret;
3104
3105         /* Lookup a matching resource from cache. */
3106         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3107                       sh->push_vlan_action_list, idx, cache_resource, next) {
3108                 if (resource->vlan_tag == cache_resource->vlan_tag &&
3109                     resource->ft_type == cache_resource->ft_type) {
3110                         DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
3111                                 "refcnt %d++",
3112                                 (void *)cache_resource,
3113                                 rte_atomic32_read(&cache_resource->refcnt));
3114                         rte_atomic32_inc(&cache_resource->refcnt);
3115                         dev_flow->handle->dvh.rix_push_vlan = idx;
3116                         dev_flow->dv.push_vlan_res = cache_resource;
3117                         return 0;
3118                 }
3119         }
3120         /* Register new push_vlan action resource. */
3121         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3122                                        &dev_flow->handle->dvh.rix_push_vlan);
3123         if (!cache_resource)
3124                 return rte_flow_error_set(error, ENOMEM,
3125                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3126                                           "cannot allocate resource memory");
3127         *cache_resource = *resource;
3128         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3129                 domain = sh->fdb_domain;
3130         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3131                 domain = sh->rx_domain;
3132         else
3133                 domain = sh->tx_domain;
3134         ret = mlx5_flow_os_create_flow_action_push_vlan
3135                                         (domain, resource->vlan_tag,
3136                                          &cache_resource->action);
3137         if (ret) {
3138                 mlx5_free(cache_resource);
3139                 return rte_flow_error_set(error, ENOMEM,
3140                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3141                                           NULL, "cannot create action");
3142         }
3143         rte_atomic32_init(&cache_resource->refcnt);
3144         rte_atomic32_inc(&cache_resource->refcnt);
3145         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3146                      &sh->push_vlan_action_list,
3147                      dev_flow->handle->dvh.rix_push_vlan,
3148                      cache_resource, next);
3149         dev_flow->dv.push_vlan_res = cache_resource;
3150         DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
3151                 (void *)cache_resource,
3152                 rte_atomic32_read(&cache_resource->refcnt));
3153         return 0;
3154 }
3155 /**
3156  * Get the size of specific rte_flow_item_type hdr size
3157  *
3158  * @param[in] item_type
3159  *   Tested rte_flow_item_type.
3160  *
3161  * @return
3162  *   sizeof struct item_type, 0 if void or irrelevant.
3163  */
3164 static size_t
3165 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
3166 {
3167         size_t retval;
3168
3169         switch (item_type) {
3170         case RTE_FLOW_ITEM_TYPE_ETH:
3171                 retval = sizeof(struct rte_ether_hdr);
3172                 break;
3173         case RTE_FLOW_ITEM_TYPE_VLAN:
3174                 retval = sizeof(struct rte_vlan_hdr);
3175                 break;
3176         case RTE_FLOW_ITEM_TYPE_IPV4:
3177                 retval = sizeof(struct rte_ipv4_hdr);
3178                 break;
3179         case RTE_FLOW_ITEM_TYPE_IPV6:
3180                 retval = sizeof(struct rte_ipv6_hdr);
3181                 break;
3182         case RTE_FLOW_ITEM_TYPE_UDP:
3183                 retval = sizeof(struct rte_udp_hdr);
3184                 break;
3185         case RTE_FLOW_ITEM_TYPE_TCP:
3186                 retval = sizeof(struct rte_tcp_hdr);
3187                 break;
3188         case RTE_FLOW_ITEM_TYPE_VXLAN:
3189         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3190                 retval = sizeof(struct rte_vxlan_hdr);
3191                 break;
3192         case RTE_FLOW_ITEM_TYPE_GRE:
3193         case RTE_FLOW_ITEM_TYPE_NVGRE:
3194                 retval = sizeof(struct rte_gre_hdr);
3195                 break;
3196         case RTE_FLOW_ITEM_TYPE_MPLS:
3197                 retval = sizeof(struct rte_mpls_hdr);
3198                 break;
3199         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
3200         default:
3201                 retval = 0;
3202                 break;
3203         }
3204         return retval;
3205 }
3206
3207 #define MLX5_ENCAP_IPV4_VERSION         0x40
3208 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
3209 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
3210 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
3211 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
3212 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
3213 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
3214
3215 /**
3216  * Convert the encap action data from list of rte_flow_item to raw buffer
3217  *
3218  * @param[in] items
3219  *   Pointer to rte_flow_item objects list.
3220  * @param[out] buf
3221  *   Pointer to the output buffer.
3222  * @param[out] size
3223  *   Pointer to the output buffer size.
3224  * @param[out] error
3225  *   Pointer to the error structure.
3226  *
3227  * @return
3228  *   0 on success, a negative errno value otherwise and rte_errno is set.
3229  */
3230 static int
3231 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
3232                            size_t *size, struct rte_flow_error *error)
3233 {
3234         struct rte_ether_hdr *eth = NULL;
3235         struct rte_vlan_hdr *vlan = NULL;
3236         struct rte_ipv4_hdr *ipv4 = NULL;
3237         struct rte_ipv6_hdr *ipv6 = NULL;
3238         struct rte_udp_hdr *udp = NULL;
3239         struct rte_vxlan_hdr *vxlan = NULL;
3240         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
3241         struct rte_gre_hdr *gre = NULL;
3242         size_t len;
3243         size_t temp_size = 0;
3244
3245         if (!items)
3246                 return rte_flow_error_set(error, EINVAL,
3247                                           RTE_FLOW_ERROR_TYPE_ACTION,
3248                                           NULL, "invalid empty data");
3249         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3250                 len = flow_dv_get_item_hdr_len(items->type);
3251                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
3252                         return rte_flow_error_set(error, EINVAL,
3253                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3254                                                   (void *)items->type,
3255                                                   "items total size is too big"
3256                                                   " for encap action");
3257                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
3258                 switch (items->type) {
3259                 case RTE_FLOW_ITEM_TYPE_ETH:
3260                         eth = (struct rte_ether_hdr *)&buf[temp_size];
3261                         break;
3262                 case RTE_FLOW_ITEM_TYPE_VLAN:
3263                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
3264                         if (!eth)
3265                                 return rte_flow_error_set(error, EINVAL,
3266                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3267                                                 (void *)items->type,
3268                                                 "eth header not found");
3269                         if (!eth->ether_type)
3270                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
3271                         break;
3272                 case RTE_FLOW_ITEM_TYPE_IPV4:
3273                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
3274                         if (!vlan && !eth)
3275                                 return rte_flow_error_set(error, EINVAL,
3276                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3277                                                 (void *)items->type,
3278                                                 "neither eth nor vlan"
3279                                                 " header found");
3280                         if (vlan && !vlan->eth_proto)
3281                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3282                         else if (eth && !eth->ether_type)
3283                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3284                         if (!ipv4->version_ihl)
3285                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
3286                                                     MLX5_ENCAP_IPV4_IHL_MIN;
3287                         if (!ipv4->time_to_live)
3288                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
3289                         break;
3290                 case RTE_FLOW_ITEM_TYPE_IPV6:
3291                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
3292                         if (!vlan && !eth)
3293                                 return rte_flow_error_set(error, EINVAL,
3294                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3295                                                 (void *)items->type,
3296                                                 "neither eth nor vlan"
3297                                                 " header found");
3298                         if (vlan && !vlan->eth_proto)
3299                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3300                         else if (eth && !eth->ether_type)
3301                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3302                         if (!ipv6->vtc_flow)
3303                                 ipv6->vtc_flow =
3304                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
3305                         if (!ipv6->hop_limits)
3306                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
3307                         break;
3308                 case RTE_FLOW_ITEM_TYPE_UDP:
3309                         udp = (struct rte_udp_hdr *)&buf[temp_size];
3310                         if (!ipv4 && !ipv6)
3311                                 return rte_flow_error_set(error, EINVAL,
3312                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3313                                                 (void *)items->type,
3314                                                 "ip header not found");
3315                         if (ipv4 && !ipv4->next_proto_id)
3316                                 ipv4->next_proto_id = IPPROTO_UDP;
3317                         else if (ipv6 && !ipv6->proto)
3318                                 ipv6->proto = IPPROTO_UDP;
3319                         break;
3320                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3321                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
3322                         if (!udp)
3323                                 return rte_flow_error_set(error, EINVAL,
3324                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3325                                                 (void *)items->type,
3326                                                 "udp header not found");
3327                         if (!udp->dst_port)
3328                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
3329                         if (!vxlan->vx_flags)
3330                                 vxlan->vx_flags =
3331                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
3332                         break;
3333                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3334                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
3335                         if (!udp)
3336                                 return rte_flow_error_set(error, EINVAL,
3337                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3338                                                 (void *)items->type,
3339                                                 "udp header not found");
3340                         if (!vxlan_gpe->proto)
3341                                 return rte_flow_error_set(error, EINVAL,
3342                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3343                                                 (void *)items->type,
3344                                                 "next protocol not found");
3345                         if (!udp->dst_port)
3346                                 udp->dst_port =
3347                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
3348                         if (!vxlan_gpe->vx_flags)
3349                                 vxlan_gpe->vx_flags =
3350                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
3351                         break;
3352                 case RTE_FLOW_ITEM_TYPE_GRE:
3353                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3354                         gre = (struct rte_gre_hdr *)&buf[temp_size];
3355                         if (!gre->proto)
3356                                 return rte_flow_error_set(error, EINVAL,
3357                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3358                                                 (void *)items->type,
3359                                                 "next protocol not found");
3360                         if (!ipv4 && !ipv6)
3361                                 return rte_flow_error_set(error, EINVAL,
3362                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3363                                                 (void *)items->type,
3364                                                 "ip header not found");
3365                         if (ipv4 && !ipv4->next_proto_id)
3366                                 ipv4->next_proto_id = IPPROTO_GRE;
3367                         else if (ipv6 && !ipv6->proto)
3368                                 ipv6->proto = IPPROTO_GRE;
3369                         break;
3370                 case RTE_FLOW_ITEM_TYPE_VOID:
3371                         break;
3372                 default:
3373                         return rte_flow_error_set(error, EINVAL,
3374                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3375                                                   (void *)items->type,
3376                                                   "unsupported item type");
3377                         break;
3378                 }
3379                 temp_size += len;
3380         }
3381         *size = temp_size;
3382         return 0;
3383 }
3384
3385 static int
3386 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
3387 {
3388         struct rte_ether_hdr *eth = NULL;
3389         struct rte_vlan_hdr *vlan = NULL;
3390         struct rte_ipv6_hdr *ipv6 = NULL;
3391         struct rte_udp_hdr *udp = NULL;
3392         char *next_hdr;
3393         uint16_t proto;
3394
3395         eth = (struct rte_ether_hdr *)data;
3396         next_hdr = (char *)(eth + 1);
3397         proto = RTE_BE16(eth->ether_type);
3398
3399         /* VLAN skipping */
3400         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
3401                 vlan = (struct rte_vlan_hdr *)next_hdr;
3402                 proto = RTE_BE16(vlan->eth_proto);
3403                 next_hdr += sizeof(struct rte_vlan_hdr);
3404         }
3405
3406         /* HW calculates IPv4 csum. no need to proceed */
3407         if (proto == RTE_ETHER_TYPE_IPV4)
3408                 return 0;
3409
3410         /* non IPv4/IPv6 header. not supported */
3411         if (proto != RTE_ETHER_TYPE_IPV6) {
3412                 return rte_flow_error_set(error, ENOTSUP,
3413                                           RTE_FLOW_ERROR_TYPE_ACTION,
3414                                           NULL, "Cannot offload non IPv4/IPv6");
3415         }
3416
3417         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
3418
3419         /* ignore non UDP */
3420         if (ipv6->proto != IPPROTO_UDP)
3421                 return 0;
3422
3423         udp = (struct rte_udp_hdr *)(ipv6 + 1);
3424         udp->dgram_cksum = 0;
3425
3426         return 0;
3427 }
3428
3429 /**
3430  * Convert L2 encap action to DV specification.
3431  *
3432  * @param[in] dev
3433  *   Pointer to rte_eth_dev structure.
3434  * @param[in] action
3435  *   Pointer to action structure.
3436  * @param[in, out] dev_flow
3437  *   Pointer to the mlx5_flow.
3438  * @param[in] transfer
3439  *   Mark if the flow is E-Switch flow.
3440  * @param[out] error
3441  *   Pointer to the error structure.
3442  *
3443  * @return
3444  *   0 on success, a negative errno value otherwise and rte_errno is set.
3445  */
3446 static int
3447 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
3448                                const struct rte_flow_action *action,
3449                                struct mlx5_flow *dev_flow,
3450                                uint8_t transfer,
3451                                struct rte_flow_error *error)
3452 {
3453         const struct rte_flow_item *encap_data;
3454         const struct rte_flow_action_raw_encap *raw_encap_data;
3455         struct mlx5_flow_dv_encap_decap_resource res = {
3456                 .reformat_type =
3457                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
3458                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3459                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
3460         };
3461
3462         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3463                 raw_encap_data =
3464                         (const struct rte_flow_action_raw_encap *)action->conf;
3465                 res.size = raw_encap_data->size;
3466                 memcpy(res.buf, raw_encap_data->data, res.size);
3467         } else {
3468                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
3469                         encap_data =
3470                                 ((const struct rte_flow_action_vxlan_encap *)
3471                                                 action->conf)->definition;
3472                 else
3473                         encap_data =
3474                                 ((const struct rte_flow_action_nvgre_encap *)
3475                                                 action->conf)->definition;
3476                 if (flow_dv_convert_encap_data(encap_data, res.buf,
3477                                                &res.size, error))
3478                         return -rte_errno;
3479         }
3480         if (flow_dv_zero_encap_udp_csum(res.buf, error))
3481                 return -rte_errno;
3482         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3483                 return rte_flow_error_set(error, EINVAL,
3484                                           RTE_FLOW_ERROR_TYPE_ACTION,
3485                                           NULL, "can't create L2 encap action");
3486         return 0;
3487 }
3488
3489 /**
3490  * Convert L2 decap action to DV specification.
3491  *
3492  * @param[in] dev
3493  *   Pointer to rte_eth_dev structure.
3494  * @param[in, out] dev_flow
3495  *   Pointer to the mlx5_flow.
3496  * @param[in] transfer
3497  *   Mark if the flow is E-Switch flow.
3498  * @param[out] error
3499  *   Pointer to the error structure.
3500  *
3501  * @return
3502  *   0 on success, a negative errno value otherwise and rte_errno is set.
3503  */
3504 static int
3505 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
3506                                struct mlx5_flow *dev_flow,
3507                                uint8_t transfer,
3508                                struct rte_flow_error *error)
3509 {
3510         struct mlx5_flow_dv_encap_decap_resource res = {
3511                 .size = 0,
3512                 .reformat_type =
3513                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
3514                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3515                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
3516         };
3517
3518         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3519                 return rte_flow_error_set(error, EINVAL,
3520                                           RTE_FLOW_ERROR_TYPE_ACTION,
3521                                           NULL, "can't create L2 decap action");
3522         return 0;
3523 }
3524
3525 /**
3526  * Convert raw decap/encap (L3 tunnel) action to DV specification.
3527  *
3528  * @param[in] dev
3529  *   Pointer to rte_eth_dev structure.
3530  * @param[in] action
3531  *   Pointer to action structure.
3532  * @param[in, out] dev_flow
3533  *   Pointer to the mlx5_flow.
3534  * @param[in] attr
3535  *   Pointer to the flow attributes.
3536  * @param[out] error
3537  *   Pointer to the error structure.
3538  *
3539  * @return
3540  *   0 on success, a negative errno value otherwise and rte_errno is set.
3541  */
3542 static int
3543 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
3544                                 const struct rte_flow_action *action,
3545                                 struct mlx5_flow *dev_flow,
3546                                 const struct rte_flow_attr *attr,
3547                                 struct rte_flow_error *error)
3548 {
3549         const struct rte_flow_action_raw_encap *encap_data;
3550         struct mlx5_flow_dv_encap_decap_resource res;
3551
3552         memset(&res, 0, sizeof(res));
3553         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
3554         res.size = encap_data->size;
3555         memcpy(res.buf, encap_data->data, res.size);
3556         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
3557                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
3558                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3559         if (attr->transfer)
3560                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3561         else
3562                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3563                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3564         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3565                 return rte_flow_error_set(error, EINVAL,
3566                                           RTE_FLOW_ERROR_TYPE_ACTION,
3567                                           NULL, "can't create encap action");
3568         return 0;
3569 }
3570
3571 /**
3572  * Create action push VLAN.
3573  *
3574  * @param[in] dev
3575  *   Pointer to rte_eth_dev structure.
3576  * @param[in] attr
3577  *   Pointer to the flow attributes.
3578  * @param[in] vlan
3579  *   Pointer to the vlan to push to the Ethernet header.
3580  * @param[in, out] dev_flow
3581  *   Pointer to the mlx5_flow.
3582  * @param[out] error
3583  *   Pointer to the error structure.
3584  *
3585  * @return
3586  *   0 on success, a negative errno value otherwise and rte_errno is set.
3587  */
3588 static int
3589 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
3590                                 const struct rte_flow_attr *attr,
3591                                 const struct rte_vlan_hdr *vlan,
3592                                 struct mlx5_flow *dev_flow,
3593                                 struct rte_flow_error *error)
3594 {
3595         struct mlx5_flow_dv_push_vlan_action_resource res;
3596
3597         memset(&res, 0, sizeof(res));
3598         res.vlan_tag =
3599                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
3600                                  vlan->vlan_tci);
3601         if (attr->transfer)
3602                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3603         else
3604                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3605                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3606         return flow_dv_push_vlan_action_resource_register
3607                                             (dev, &res, dev_flow, error);
3608 }
3609
3610 static int fdb_mirror;
3611
3612 /**
3613  * Validate the modify-header actions.
3614  *
3615  * @param[in] action_flags
3616  *   Holds the actions detected until now.
3617  * @param[in] action
3618  *   Pointer to the modify action.
3619  * @param[out] error
3620  *   Pointer to error structure.
3621  *
3622  * @return
3623  *   0 on success, a negative errno value otherwise and rte_errno is set.
3624  */
3625 static int
3626 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
3627                                    const struct rte_flow_action *action,
3628                                    struct rte_flow_error *error)
3629 {
3630         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
3631                 return rte_flow_error_set(error, EINVAL,
3632                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3633                                           NULL, "action configuration not set");
3634         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3635                 return rte_flow_error_set(error, EINVAL,
3636                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3637                                           "can't have encap action before"
3638                                           " modify action");
3639         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror)
3640                 return rte_flow_error_set(error, EINVAL,
3641                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3642                                           "can't support sample action before"
3643                                           " modify action for E-Switch"
3644                                           " mirroring");
3645         return 0;
3646 }
3647
3648 /**
3649  * Validate the modify-header MAC address actions.
3650  *
3651  * @param[in] action_flags
3652  *   Holds the actions detected until now.
3653  * @param[in] action
3654  *   Pointer to the modify action.
3655  * @param[in] item_flags
3656  *   Holds the items detected.
3657  * @param[out] error
3658  *   Pointer to error structure.
3659  *
3660  * @return
3661  *   0 on success, a negative errno value otherwise and rte_errno is set.
3662  */
3663 static int
3664 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
3665                                    const struct rte_flow_action *action,
3666                                    const uint64_t item_flags,
3667                                    struct rte_flow_error *error)
3668 {
3669         int ret = 0;
3670
3671         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3672         if (!ret) {
3673                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
3674                         return rte_flow_error_set(error, EINVAL,
3675                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3676                                                   NULL,
3677                                                   "no L2 item in pattern");
3678         }
3679         return ret;
3680 }
3681
3682 /**
3683  * Validate the modify-header IPv4 address actions.
3684  *
3685  * @param[in] action_flags
3686  *   Holds the actions detected until now.
3687  * @param[in] action
3688  *   Pointer to the modify action.
3689  * @param[in] item_flags
3690  *   Holds the items detected.
3691  * @param[out] error
3692  *   Pointer to error structure.
3693  *
3694  * @return
3695  *   0 on success, a negative errno value otherwise and rte_errno is set.
3696  */
3697 static int
3698 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3699                                     const struct rte_flow_action *action,
3700                                     const uint64_t item_flags,
3701                                     struct rte_flow_error *error)
3702 {
3703         int ret = 0;
3704         uint64_t layer;
3705
3706         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3707         if (!ret) {
3708                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3709                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3710                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3711                 if (!(item_flags & layer))
3712                         return rte_flow_error_set(error, EINVAL,
3713                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3714                                                   NULL,
3715                                                   "no ipv4 item in pattern");
3716         }
3717         return ret;
3718 }
3719
3720 /**
3721  * Validate the modify-header IPv6 address actions.
3722  *
3723  * @param[in] action_flags
3724  *   Holds the actions detected until now.
3725  * @param[in] action
3726  *   Pointer to the modify action.
3727  * @param[in] item_flags
3728  *   Holds the items detected.
3729  * @param[out] error
3730  *   Pointer to error structure.
3731  *
3732  * @return
3733  *   0 on success, a negative errno value otherwise and rte_errno is set.
3734  */
3735 static int
3736 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3737                                     const struct rte_flow_action *action,
3738                                     const uint64_t item_flags,
3739                                     struct rte_flow_error *error)
3740 {
3741         int ret = 0;
3742         uint64_t layer;
3743
3744         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3745         if (!ret) {
3746                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3747                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3748                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3749                 if (!(item_flags & layer))
3750                         return rte_flow_error_set(error, EINVAL,
3751                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3752                                                   NULL,
3753                                                   "no ipv6 item in pattern");
3754         }
3755         return ret;
3756 }
3757
3758 /**
3759  * Validate the modify-header TP actions.
3760  *
3761  * @param[in] action_flags
3762  *   Holds the actions detected until now.
3763  * @param[in] action
3764  *   Pointer to the modify action.
3765  * @param[in] item_flags
3766  *   Holds the items detected.
3767  * @param[out] error
3768  *   Pointer to error structure.
3769  *
3770  * @return
3771  *   0 on success, a negative errno value otherwise and rte_errno is set.
3772  */
3773 static int
3774 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3775                                   const struct rte_flow_action *action,
3776                                   const uint64_t item_flags,
3777                                   struct rte_flow_error *error)
3778 {
3779         int ret = 0;
3780         uint64_t layer;
3781
3782         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3783         if (!ret) {
3784                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3785                                  MLX5_FLOW_LAYER_INNER_L4 :
3786                                  MLX5_FLOW_LAYER_OUTER_L4;
3787                 if (!(item_flags & layer))
3788                         return rte_flow_error_set(error, EINVAL,
3789                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3790                                                   NULL, "no transport layer "
3791                                                   "in pattern");
3792         }
3793         return ret;
3794 }
3795
3796 /**
3797  * Validate the modify-header actions of increment/decrement
3798  * TCP Sequence-number.
3799  *
3800  * @param[in] action_flags
3801  *   Holds the actions detected until now.
3802  * @param[in] action
3803  *   Pointer to the modify action.
3804  * @param[in] item_flags
3805  *   Holds the items detected.
3806  * @param[out] error
3807  *   Pointer to error structure.
3808  *
3809  * @return
3810  *   0 on success, a negative errno value otherwise and rte_errno is set.
3811  */
3812 static int
3813 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3814                                        const struct rte_flow_action *action,
3815                                        const uint64_t item_flags,
3816                                        struct rte_flow_error *error)
3817 {
3818         int ret = 0;
3819         uint64_t layer;
3820
3821         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3822         if (!ret) {
3823                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3824                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3825                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3826                 if (!(item_flags & layer))
3827                         return rte_flow_error_set(error, EINVAL,
3828                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3829                                                   NULL, "no TCP item in"
3830                                                   " pattern");
3831                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3832                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3833                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3834                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3835                         return rte_flow_error_set(error, EINVAL,
3836                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3837                                                   NULL,
3838                                                   "cannot decrease and increase"
3839                                                   " TCP sequence number"
3840                                                   " at the same time");
3841         }
3842         return ret;
3843 }
3844
3845 /**
3846  * Validate the modify-header actions of increment/decrement
3847  * TCP Acknowledgment number.
3848  *
3849  * @param[in] action_flags
3850  *   Holds the actions detected until now.
3851  * @param[in] action
3852  *   Pointer to the modify action.
3853  * @param[in] item_flags
3854  *   Holds the items detected.
3855  * @param[out] error
3856  *   Pointer to error structure.
3857  *
3858  * @return
3859  *   0 on success, a negative errno value otherwise and rte_errno is set.
3860  */
3861 static int
3862 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3863                                        const struct rte_flow_action *action,
3864                                        const uint64_t item_flags,
3865                                        struct rte_flow_error *error)
3866 {
3867         int ret = 0;
3868         uint64_t layer;
3869
3870         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3871         if (!ret) {
3872                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3873                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3874                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3875                 if (!(item_flags & layer))
3876                         return rte_flow_error_set(error, EINVAL,
3877                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3878                                                   NULL, "no TCP item in"
3879                                                   " pattern");
3880                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3881                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3882                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3883                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3884                         return rte_flow_error_set(error, EINVAL,
3885                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3886                                                   NULL,
3887                                                   "cannot decrease and increase"
3888                                                   " TCP acknowledgment number"
3889                                                   " at the same time");
3890         }
3891         return ret;
3892 }
3893
3894 /**
3895  * Validate the modify-header TTL actions.
3896  *
3897  * @param[in] action_flags
3898  *   Holds the actions detected until now.
3899  * @param[in] action
3900  *   Pointer to the modify action.
3901  * @param[in] item_flags
3902  *   Holds the items detected.
3903  * @param[out] error
3904  *   Pointer to error structure.
3905  *
3906  * @return
3907  *   0 on success, a negative errno value otherwise and rte_errno is set.
3908  */
3909 static int
3910 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3911                                    const struct rte_flow_action *action,
3912                                    const uint64_t item_flags,
3913                                    struct rte_flow_error *error)
3914 {
3915         int ret = 0;
3916         uint64_t layer;
3917
3918         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3919         if (!ret) {
3920                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3921                                  MLX5_FLOW_LAYER_INNER_L3 :
3922                                  MLX5_FLOW_LAYER_OUTER_L3;
3923                 if (!(item_flags & layer))
3924                         return rte_flow_error_set(error, EINVAL,
3925                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3926                                                   NULL,
3927                                                   "no IP protocol in pattern");
3928         }
3929         return ret;
3930 }
3931
3932 /**
3933  * Validate jump action.
3934  *
3935  * @param[in] action
3936  *   Pointer to the jump action.
3937  * @param[in] action_flags
3938  *   Holds the actions detected until now.
3939  * @param[in] attributes
3940  *   Pointer to flow attributes
3941  * @param[in] external
3942  *   Action belongs to flow rule created by request external to PMD.
3943  * @param[out] error
3944  *   Pointer to error structure.
3945  *
3946  * @return
3947  *   0 on success, a negative errno value otherwise and rte_errno is set.
3948  */
3949 static int
3950 flow_dv_validate_action_jump(const struct rte_flow_action *action,
3951                              uint64_t action_flags,
3952                              const struct rte_flow_attr *attributes,
3953                              bool external, struct rte_flow_error *error)
3954 {
3955         uint32_t target_group, table;
3956         int ret = 0;
3957
3958         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3959                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3960                 return rte_flow_error_set(error, EINVAL,
3961                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3962                                           "can't have 2 fate actions in"
3963                                           " same flow");
3964         if (action_flags & MLX5_FLOW_ACTION_METER)
3965                 return rte_flow_error_set(error, ENOTSUP,
3966                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3967                                           "jump with meter not support");
3968         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror)
3969                 return rte_flow_error_set(error, EINVAL,
3970                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3971                                           "E-Switch mirroring can't support"
3972                                           " Sample action and jump action in"
3973                                           " same flow now");
3974         if (!action->conf)
3975                 return rte_flow_error_set(error, EINVAL,
3976                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3977                                           NULL, "action configuration not set");
3978         target_group =
3979                 ((const struct rte_flow_action_jump *)action->conf)->group;
3980         ret = mlx5_flow_group_to_table(attributes, external, target_group,
3981                                        true, &table, error);
3982         if (ret)
3983                 return ret;
3984         if (attributes->group == target_group)
3985                 return rte_flow_error_set(error, EINVAL,
3986                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3987                                           "target group must be other than"
3988                                           " the current flow group");
3989         return 0;
3990 }
3991
3992 /*
3993  * Validate the port_id action.
3994  *
3995  * @param[in] dev
3996  *   Pointer to rte_eth_dev structure.
3997  * @param[in] action_flags
3998  *   Bit-fields that holds the actions detected until now.
3999  * @param[in] action
4000  *   Port_id RTE action structure.
4001  * @param[in] attr
4002  *   Attributes of flow that includes this action.
4003  * @param[out] error
4004  *   Pointer to error structure.
4005  *
4006  * @return
4007  *   0 on success, a negative errno value otherwise and rte_errno is set.
4008  */
4009 static int
4010 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
4011                                 uint64_t action_flags,
4012                                 const struct rte_flow_action *action,
4013                                 const struct rte_flow_attr *attr,
4014                                 struct rte_flow_error *error)
4015 {
4016         const struct rte_flow_action_port_id *port_id;
4017         struct mlx5_priv *act_priv;
4018         struct mlx5_priv *dev_priv;
4019         uint16_t port;
4020
4021         if (!attr->transfer)
4022                 return rte_flow_error_set(error, ENOTSUP,
4023                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4024                                           NULL,
4025                                           "port id action is valid in transfer"
4026                                           " mode only");
4027         if (!action || !action->conf)
4028                 return rte_flow_error_set(error, ENOTSUP,
4029                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4030                                           NULL,
4031                                           "port id action parameters must be"
4032                                           " specified");
4033         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
4034                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4035                 return rte_flow_error_set(error, EINVAL,
4036                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4037                                           "can have only one fate actions in"
4038                                           " a flow");
4039         dev_priv = mlx5_dev_to_eswitch_info(dev);
4040         if (!dev_priv)
4041                 return rte_flow_error_set(error, rte_errno,
4042                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4043                                           NULL,
4044                                           "failed to obtain E-Switch info");
4045         port_id = action->conf;
4046         port = port_id->original ? dev->data->port_id : port_id->id;
4047         act_priv = mlx5_port_to_eswitch_info(port, false);
4048         if (!act_priv)
4049                 return rte_flow_error_set
4050                                 (error, rte_errno,
4051                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
4052                                  "failed to obtain E-Switch port id for port");
4053         if (act_priv->domain_id != dev_priv->domain_id)
4054                 return rte_flow_error_set
4055                                 (error, EINVAL,
4056                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4057                                  "port does not belong to"
4058                                  " E-Switch being configured");
4059         return 0;
4060 }
4061
4062 /**
4063  * Get the maximum number of modify header actions.
4064  *
4065  * @param dev
4066  *   Pointer to rte_eth_dev structure.
4067  * @param flags
4068  *   Flags bits to check if root level.
4069  *
4070  * @return
4071  *   Max number of modify header actions device can support.
4072  */
4073 static inline unsigned int
4074 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
4075                               uint64_t flags)
4076 {
4077         /*
4078          * There's no way to directly query the max capacity from FW.
4079          * The maximal value on root table should be assumed to be supported.
4080          */
4081         if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
4082                 return MLX5_MAX_MODIFY_NUM;
4083         else
4084                 return MLX5_ROOT_TBL_MODIFY_NUM;
4085 }
4086
4087 /**
4088  * Validate the meter action.
4089  *
4090  * @param[in] dev
4091  *   Pointer to rte_eth_dev structure.
4092  * @param[in] action_flags
4093  *   Bit-fields that holds the actions detected until now.
4094  * @param[in] action
4095  *   Pointer to the meter action.
4096  * @param[in] attr
4097  *   Attributes of flow that includes this action.
4098  * @param[out] error
4099  *   Pointer to error structure.
4100  *
4101  * @return
4102  *   0 on success, a negative errno value otherwise and rte_ernno is set.
4103  */
4104 static int
4105 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
4106                                 uint64_t action_flags,
4107                                 const struct rte_flow_action *action,
4108                                 const struct rte_flow_attr *attr,
4109                                 struct rte_flow_error *error)
4110 {
4111         struct mlx5_priv *priv = dev->data->dev_private;
4112         const struct rte_flow_action_meter *am = action->conf;
4113         struct mlx5_flow_meter *fm;
4114
4115         if (!am)
4116                 return rte_flow_error_set(error, EINVAL,
4117                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4118                                           "meter action conf is NULL");
4119
4120         if (action_flags & MLX5_FLOW_ACTION_METER)
4121                 return rte_flow_error_set(error, ENOTSUP,
4122                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4123                                           "meter chaining not support");
4124         if (action_flags & MLX5_FLOW_ACTION_JUMP)
4125                 return rte_flow_error_set(error, ENOTSUP,
4126                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4127                                           "meter with jump not support");
4128         if (!priv->mtr_en)
4129                 return rte_flow_error_set(error, ENOTSUP,
4130                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4131                                           NULL,
4132                                           "meter action not supported");
4133         fm = mlx5_flow_meter_find(priv, am->mtr_id);
4134         if (!fm)
4135                 return rte_flow_error_set(error, EINVAL,
4136                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4137                                           "Meter not found");
4138         if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
4139               (!fm->ingress && !attr->ingress && attr->egress) ||
4140               (!fm->egress && !attr->egress && attr->ingress))))
4141                 return rte_flow_error_set(error, EINVAL,
4142                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4143                                           "Flow attributes are either invalid "
4144                                           "or have a conflict with current "
4145                                           "meter attributes");
4146         return 0;
4147 }
4148
4149 /**
4150  * Validate the age action.
4151  *
4152  * @param[in] action_flags
4153  *   Holds the actions detected until now.
4154  * @param[in] action
4155  *   Pointer to the age action.
4156  * @param[in] dev
4157  *   Pointer to the Ethernet device structure.
4158  * @param[out] error
4159  *   Pointer to error structure.
4160  *
4161  * @return
4162  *   0 on success, a negative errno value otherwise and rte_errno is set.
4163  */
4164 static int
4165 flow_dv_validate_action_age(uint64_t action_flags,
4166                             const struct rte_flow_action *action,
4167                             struct rte_eth_dev *dev,
4168                             struct rte_flow_error *error)
4169 {
4170         struct mlx5_priv *priv = dev->data->dev_private;
4171         const struct rte_flow_action_age *age = action->conf;
4172
4173         if (!priv->config.devx || priv->sh->cmng.counter_fallback)
4174                 return rte_flow_error_set(error, ENOTSUP,
4175                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4176                                           NULL,
4177                                           "age action not supported");
4178         if (!(action->conf))
4179                 return rte_flow_error_set(error, EINVAL,
4180                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4181                                           "configuration cannot be null");
4182         if (!(age->timeout))
4183                 return rte_flow_error_set(error, EINVAL,
4184                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4185                                           "invalid timeout value 0");
4186         if (action_flags & MLX5_FLOW_ACTION_AGE)
4187                 return rte_flow_error_set(error, EINVAL,
4188                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4189                                           "duplicate age actions set");
4190         return 0;
4191 }
4192
4193 /**
4194  * Validate the modify-header IPv4 DSCP actions.
4195  *
4196  * @param[in] action_flags
4197  *   Holds the actions detected until now.
4198  * @param[in] action
4199  *   Pointer to the modify action.
4200  * @param[in] item_flags
4201  *   Holds the items detected.
4202  * @param[out] error
4203  *   Pointer to error structure.
4204  *
4205  * @return
4206  *   0 on success, a negative errno value otherwise and rte_errno is set.
4207  */
4208 static int
4209 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
4210                                          const struct rte_flow_action *action,
4211                                          const uint64_t item_flags,
4212                                          struct rte_flow_error *error)
4213 {
4214         int ret = 0;
4215
4216         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4217         if (!ret) {
4218                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
4219                         return rte_flow_error_set(error, EINVAL,
4220                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4221                                                   NULL,
4222                                                   "no ipv4 item in pattern");
4223         }
4224         return ret;
4225 }
4226
4227 /**
4228  * Validate the modify-header IPv6 DSCP actions.
4229  *
4230  * @param[in] action_flags
4231  *   Holds the actions detected until now.
4232  * @param[in] action
4233  *   Pointer to the modify action.
4234  * @param[in] item_flags
4235  *   Holds the items detected.
4236  * @param[out] error
4237  *   Pointer to error structure.
4238  *
4239  * @return
4240  *   0 on success, a negative errno value otherwise and rte_errno is set.
4241  */
4242 static int
4243 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
4244                                          const struct rte_flow_action *action,
4245                                          const uint64_t item_flags,
4246                                          struct rte_flow_error *error)
4247 {
4248         int ret = 0;
4249
4250         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4251         if (!ret) {
4252                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
4253                         return rte_flow_error_set(error, EINVAL,
4254                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4255                                                   NULL,
4256                                                   "no ipv6 item in pattern");
4257         }
4258         return ret;
4259 }
4260
4261 /**
4262  * Match modify-header resource.
4263  *
4264  * @param entry
4265  *   Pointer to exist resource entry object.
4266  * @param ctx
4267  *   Pointer to new modify-header resource.
4268  *
4269  * @return
4270  *   0 on matching, -1 otherwise.
4271  */
4272 static int
4273 flow_dv_modify_hdr_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
4274 {
4275         struct mlx5_flow_dv_modify_hdr_resource *resource;
4276         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4277         uint32_t actions_len;
4278
4279         resource = (struct mlx5_flow_dv_modify_hdr_resource *)ctx;
4280         cache_resource = container_of(entry,
4281                                       struct mlx5_flow_dv_modify_hdr_resource,
4282                                       entry);
4283         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4284         if (resource->entry.key == cache_resource->entry.key &&
4285             resource->ft_type == cache_resource->ft_type &&
4286             resource->actions_num == cache_resource->actions_num &&
4287             resource->flags == cache_resource->flags &&
4288             !memcmp((const void *)resource->actions,
4289                     (const void *)cache_resource->actions,
4290                     actions_len))
4291                 return 0;
4292         return -1;
4293 }
4294
4295 /**
4296  * Validate the sample action.
4297  *
4298  * @param[in] action_flags
4299  *   Holds the actions detected until now.
4300  * @param[in] action
4301  *   Pointer to the sample action.
4302  * @param[in] dev
4303  *   Pointer to the Ethernet device structure.
4304  * @param[in] attr
4305  *   Attributes of flow that includes this action.
4306  * @param[out] error
4307  *   Pointer to error structure.
4308  *
4309  * @return
4310  *   0 on success, a negative errno value otherwise and rte_errno is set.
4311  */
4312 static int
4313 flow_dv_validate_action_sample(uint64_t action_flags,
4314                                const struct rte_flow_action *action,
4315                                struct rte_eth_dev *dev,
4316                                const struct rte_flow_attr *attr,
4317                                struct rte_flow_error *error)
4318 {
4319         struct mlx5_priv *priv = dev->data->dev_private;
4320         struct mlx5_dev_config *dev_conf = &priv->config;
4321         const struct rte_flow_action_sample *sample = action->conf;
4322         const struct rte_flow_action *act;
4323         uint64_t sub_action_flags = 0;
4324         uint16_t queue_index = 0xFFFF;
4325         int actions_n = 0;
4326         int ret;
4327         fdb_mirror = 0;
4328
4329         if (!sample)
4330                 return rte_flow_error_set(error, EINVAL,
4331                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4332                                           "configuration cannot be NULL");
4333         if (sample->ratio == 0)
4334                 return rte_flow_error_set(error, EINVAL,
4335                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4336                                           "ratio value starts from 1");
4337         if (!priv->config.devx || (sample->ratio > 0 && !priv->sampler_en))
4338                 return rte_flow_error_set(error, ENOTSUP,
4339                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4340                                           NULL,
4341                                           "sample action not supported");
4342         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
4343                 return rte_flow_error_set(error, EINVAL,
4344                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4345                                           "Multiple sample actions not "
4346                                           "supported");
4347         if (action_flags & MLX5_FLOW_ACTION_METER)
4348                 return rte_flow_error_set(error, EINVAL,
4349                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4350                                           "wrong action order, meter should "
4351                                           "be after sample action");
4352         if (action_flags & MLX5_FLOW_ACTION_JUMP)
4353                 return rte_flow_error_set(error, EINVAL,
4354                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4355                                           "wrong action order, jump should "
4356                                           "be after sample action");
4357         act = sample->actions;
4358         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
4359                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
4360                         return rte_flow_error_set(error, ENOTSUP,
4361                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4362                                                   act, "too many actions");
4363                 switch (act->type) {
4364                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4365                         ret = mlx5_flow_validate_action_queue(act,
4366                                                               sub_action_flags,
4367                                                               dev,
4368                                                               attr, error);
4369                         if (ret < 0)
4370                                 return ret;
4371                         queue_index = ((const struct rte_flow_action_queue *)
4372                                                         (act->conf))->index;
4373                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
4374                         ++actions_n;
4375                         break;
4376                 case RTE_FLOW_ACTION_TYPE_MARK:
4377                         ret = flow_dv_validate_action_mark(dev, act,
4378                                                            sub_action_flags,
4379                                                            attr, error);
4380                         if (ret < 0)
4381                                 return ret;
4382                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
4383                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
4384                                                 MLX5_FLOW_ACTION_MARK_EXT;
4385                         else
4386                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
4387                         ++actions_n;
4388                         break;
4389                 case RTE_FLOW_ACTION_TYPE_COUNT:
4390                         ret = flow_dv_validate_action_count(dev, error);
4391                         if (ret < 0)
4392                                 return ret;
4393                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
4394                         ++actions_n;
4395                         break;
4396                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
4397                         ret = flow_dv_validate_action_port_id(dev,
4398                                                               sub_action_flags,
4399                                                               act,
4400                                                               attr,
4401                                                               error);
4402                         if (ret)
4403                                 return ret;
4404                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
4405                         ++actions_n;
4406                         break;
4407                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4408                         ret = flow_dv_validate_action_raw_encap_decap
4409                                 (dev, NULL, act->conf, attr, &sub_action_flags,
4410                                  &actions_n, error);
4411                         if (ret < 0)
4412                                 return ret;
4413                         ++actions_n;
4414                         break;
4415                 default:
4416                         return rte_flow_error_set(error, ENOTSUP,
4417                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4418                                                   NULL,
4419                                                   "Doesn't support optional "
4420                                                   "action");
4421                 }
4422         }
4423         if (attr->ingress && !attr->transfer) {
4424                 if (!(sub_action_flags & MLX5_FLOW_ACTION_QUEUE))
4425                         return rte_flow_error_set(error, EINVAL,
4426                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4427                                                   NULL,
4428                                                   "Ingress must has a dest "
4429                                                   "QUEUE for Sample");
4430         } else if (attr->egress && !attr->transfer) {
4431                 return rte_flow_error_set(error, ENOTSUP,
4432                                           RTE_FLOW_ERROR_TYPE_ACTION,
4433                                           NULL,
4434                                           "Sample Only support Ingress "
4435                                           "or E-Switch");
4436         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
4437                 MLX5_ASSERT(attr->transfer);
4438                 if (sample->ratio > 1)
4439                         return rte_flow_error_set(error, ENOTSUP,
4440                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4441                                                   NULL,
4442                                                   "E-Switch doesn't support "
4443                                                   "any optional action "
4444                                                   "for sampling");
4445                 fdb_mirror = 1;
4446                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
4447                         return rte_flow_error_set(error, ENOTSUP,
4448                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4449                                                   NULL,
4450                                                   "unsupported action QUEUE");
4451                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
4452                         return rte_flow_error_set(error, EINVAL,
4453                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4454                                                   NULL,
4455                                                   "E-Switch must has a dest "
4456                                                   "port for mirroring");
4457         }
4458         /* Continue validation for Xcap actions.*/
4459         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
4460             (queue_index == 0xFFFF ||
4461              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
4462                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
4463                      MLX5_FLOW_XCAP_ACTIONS)
4464                         return rte_flow_error_set(error, ENOTSUP,
4465                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4466                                                   NULL, "encap and decap "
4467                                                   "combination aren't "
4468                                                   "supported");
4469                 if (!attr->transfer && attr->ingress && (sub_action_flags &
4470                                                         MLX5_FLOW_ACTION_ENCAP))
4471                         return rte_flow_error_set(error, ENOTSUP,
4472                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4473                                                   NULL, "encap is not supported"
4474                                                   " for ingress traffic");
4475         }
4476         return 0;
4477 }
4478
4479 /**
4480  * Find existing modify-header resource or create and register a new one.
4481  *
4482  * @param dev[in, out]
4483  *   Pointer to rte_eth_dev structure.
4484  * @param[in, out] resource
4485  *   Pointer to modify-header resource.
4486  * @parm[in, out] dev_flow
4487  *   Pointer to the dev_flow.
4488  * @param[out] error
4489  *   pointer to error structure.
4490  *
4491  * @return
4492  *   0 on success otherwise -errno and errno is set.
4493  */
4494 static int
4495 flow_dv_modify_hdr_resource_register
4496                         (struct rte_eth_dev *dev,
4497                          struct mlx5_flow_dv_modify_hdr_resource *resource,
4498                          struct mlx5_flow *dev_flow,
4499                          struct rte_flow_error *error)
4500 {
4501         struct mlx5_priv *priv = dev->data->dev_private;
4502         struct mlx5_dev_ctx_shared *sh = priv->sh;
4503         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4504         struct mlx5dv_dr_domain *ns;
4505         uint32_t actions_len;
4506         struct mlx5_hlist_entry *entry;
4507         union mlx5_flow_modify_hdr_key hdr_mod_key = {
4508                 {
4509                         .ft_type = resource->ft_type,
4510                         .actions_num = resource->actions_num,
4511                         .group = dev_flow->dv.group,
4512                         .cksum = 0,
4513                 }
4514         };
4515         int ret;
4516
4517         resource->flags = dev_flow->dv.group ? 0 :
4518                           MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4519         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
4520                                     resource->flags))
4521                 return rte_flow_error_set(error, EOVERFLOW,
4522                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4523                                           "too many modify header items");
4524         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
4525                 ns = sh->fdb_domain;
4526         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
4527                 ns = sh->tx_domain;
4528         else
4529                 ns = sh->rx_domain;
4530         /* Lookup a matching resource from cache. */
4531         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4532         hdr_mod_key.cksum = __rte_raw_cksum(resource->actions, actions_len, 0);
4533         resource->entry.key = hdr_mod_key.v64;
4534         entry = mlx5_hlist_lookup_ex(sh->modify_cmds, resource->entry.key,
4535                                      flow_dv_modify_hdr_resource_match,
4536                                      (void *)resource);
4537         if (entry) {
4538                 cache_resource = container_of(entry,
4539                                         struct mlx5_flow_dv_modify_hdr_resource,
4540                                         entry);
4541                 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
4542                         (void *)cache_resource,
4543                         rte_atomic32_read(&cache_resource->refcnt));
4544                 rte_atomic32_inc(&cache_resource->refcnt);
4545                 dev_flow->handle->dvh.modify_hdr = cache_resource;
4546                 return 0;
4547
4548         }
4549         /* Register new modify-header resource. */
4550         cache_resource = mlx5_malloc(MLX5_MEM_ZERO,
4551                                     sizeof(*cache_resource) + actions_len, 0,
4552                                     SOCKET_ID_ANY);
4553         if (!cache_resource)
4554                 return rte_flow_error_set(error, ENOMEM,
4555                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4556                                           "cannot allocate resource memory");
4557         *cache_resource = *resource;
4558         rte_memcpy(cache_resource->actions, resource->actions, actions_len);
4559         ret = mlx5_flow_os_create_flow_action_modify_header
4560                                         (sh->ctx, ns, cache_resource,
4561                                          actions_len, &cache_resource->action);
4562         if (ret) {
4563                 mlx5_free(cache_resource);
4564                 return rte_flow_error_set(error, ENOMEM,
4565                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4566                                           NULL, "cannot create action");
4567         }
4568         rte_atomic32_init(&cache_resource->refcnt);
4569         rte_atomic32_inc(&cache_resource->refcnt);
4570         if (mlx5_hlist_insert_ex(sh->modify_cmds, &cache_resource->entry,
4571                                  flow_dv_modify_hdr_resource_match,
4572                                  (void *)cache_resource)) {
4573                 claim_zero(mlx5_flow_os_destroy_flow_action
4574                                                 (cache_resource->action));
4575                 mlx5_free(cache_resource);
4576                 return rte_flow_error_set(error, EEXIST,
4577                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4578                                           NULL, "action exist");
4579         }
4580         dev_flow->handle->dvh.modify_hdr = cache_resource;
4581         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
4582                 (void *)cache_resource,
4583                 rte_atomic32_read(&cache_resource->refcnt));
4584         return 0;
4585 }
4586
4587 /**
4588  * Get DV flow counter by index.
4589  *
4590  * @param[in] dev
4591  *   Pointer to the Ethernet device structure.
4592  * @param[in] idx
4593  *   mlx5 flow counter index in the container.
4594  * @param[out] ppool
4595  *   mlx5 flow counter pool in the container,
4596  *
4597  * @return
4598  *   Pointer to the counter, NULL otherwise.
4599  */
4600 static struct mlx5_flow_counter *
4601 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
4602                            uint32_t idx,
4603                            struct mlx5_flow_counter_pool **ppool)
4604 {
4605         struct mlx5_priv *priv = dev->data->dev_private;
4606         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4607         struct mlx5_flow_counter_pool *pool;
4608
4609         /* Decrease to original index and clear shared bit. */
4610         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
4611         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cmng->n);
4612         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
4613         MLX5_ASSERT(pool);
4614         if (ppool)
4615                 *ppool = pool;
4616         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
4617 }
4618
4619 /**
4620  * Check the devx counter belongs to the pool.
4621  *
4622  * @param[in] pool
4623  *   Pointer to the counter pool.
4624  * @param[in] id
4625  *   The counter devx ID.
4626  *
4627  * @return
4628  *   True if counter belongs to the pool, false otherwise.
4629  */
4630 static bool
4631 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
4632 {
4633         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
4634                    MLX5_COUNTERS_PER_POOL;
4635
4636         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
4637                 return true;
4638         return false;
4639 }
4640
4641 /**
4642  * Get a pool by devx counter ID.
4643  *
4644  * @param[in] cmng
4645  *   Pointer to the counter management.
4646  * @param[in] id
4647  *   The counter devx ID.
4648  *
4649  * @return
4650  *   The counter pool pointer if exists, NULL otherwise,
4651  */
4652 static struct mlx5_flow_counter_pool *
4653 flow_dv_find_pool_by_id(struct mlx5_flow_counter_mng *cmng, int id)
4654 {
4655         uint32_t i;
4656         struct mlx5_flow_counter_pool *pool = NULL;
4657
4658         rte_spinlock_lock(&cmng->pool_update_sl);
4659         /* Check last used pool. */
4660         if (cmng->last_pool_idx != POOL_IDX_INVALID &&
4661             flow_dv_is_counter_in_pool(cmng->pools[cmng->last_pool_idx], id)) {
4662                 pool = cmng->pools[cmng->last_pool_idx];
4663                 goto out;
4664         }
4665         /* ID out of range means no suitable pool in the container. */
4666         if (id > cmng->max_id || id < cmng->min_id)
4667                 goto out;
4668         /*
4669          * Find the pool from the end of the container, since mostly counter
4670          * ID is sequence increasing, and the last pool should be the needed
4671          * one.
4672          */
4673         i = cmng->n_valid;
4674         while (i--) {
4675                 struct mlx5_flow_counter_pool *pool_tmp = cmng->pools[i];
4676
4677                 if (flow_dv_is_counter_in_pool(pool_tmp, id)) {
4678                         pool = pool_tmp;
4679                         break;
4680                 }
4681         }
4682 out:
4683         rte_spinlock_unlock(&cmng->pool_update_sl);
4684         return pool;
4685 }
4686
4687 /**
4688  * Resize a counter container.
4689  *
4690  * @param[in] dev
4691  *   Pointer to the Ethernet device structure.
4692  *
4693  * @return
4694  *   0 on success, otherwise negative errno value and rte_errno is set.
4695  */
4696 static int
4697 flow_dv_container_resize(struct rte_eth_dev *dev)
4698 {
4699         struct mlx5_priv *priv = dev->data->dev_private;
4700         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4701         void *old_pools = cmng->pools;
4702         uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
4703         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
4704         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
4705
4706         if (!pools) {
4707                 rte_errno = ENOMEM;
4708                 return -ENOMEM;
4709         }
4710         if (old_pools)
4711                 memcpy(pools, old_pools, cmng->n *
4712                                        sizeof(struct mlx5_flow_counter_pool *));
4713         cmng->n = resize;
4714         cmng->pools = pools;
4715         if (old_pools)
4716                 mlx5_free(old_pools);
4717         return 0;
4718 }
4719
4720 /**
4721  * Query a devx flow counter.
4722  *
4723  * @param[in] dev
4724  *   Pointer to the Ethernet device structure.
4725  * @param[in] cnt
4726  *   Index to the flow counter.
4727  * @param[out] pkts
4728  *   The statistics value of packets.
4729  * @param[out] bytes
4730  *   The statistics value of bytes.
4731  *
4732  * @return
4733  *   0 on success, otherwise a negative errno value and rte_errno is set.
4734  */
4735 static inline int
4736 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
4737                      uint64_t *bytes)
4738 {
4739         struct mlx5_priv *priv = dev->data->dev_private;
4740         struct mlx5_flow_counter_pool *pool = NULL;
4741         struct mlx5_flow_counter *cnt;
4742         int offset;
4743
4744         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4745         MLX5_ASSERT(pool);
4746         if (priv->sh->cmng.counter_fallback)
4747                 return mlx5_devx_cmd_flow_counter_query(cnt->dcs_when_active, 0,
4748                                         0, pkts, bytes, 0, NULL, NULL, 0);
4749         rte_spinlock_lock(&pool->sl);
4750         if (!pool->raw) {
4751                 *pkts = 0;
4752                 *bytes = 0;
4753         } else {
4754                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
4755                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
4756                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
4757         }
4758         rte_spinlock_unlock(&pool->sl);
4759         return 0;
4760 }
4761
4762 /**
4763  * Create and initialize a new counter pool.
4764  *
4765  * @param[in] dev
4766  *   Pointer to the Ethernet device structure.
4767  * @param[out] dcs
4768  *   The devX counter handle.
4769  * @param[in] age
4770  *   Whether the pool is for counter that was allocated for aging.
4771  * @param[in/out] cont_cur
4772  *   Pointer to the container pointer, it will be update in pool resize.
4773  *
4774  * @return
4775  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
4776  */
4777 static struct mlx5_flow_counter_pool *
4778 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
4779                     uint32_t age)
4780 {
4781         struct mlx5_priv *priv = dev->data->dev_private;
4782         struct mlx5_flow_counter_pool *pool;
4783         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4784         bool fallback = priv->sh->cmng.counter_fallback;
4785         uint32_t size = sizeof(*pool);
4786
4787         size += MLX5_COUNTERS_PER_POOL * MLX5_CNT_SIZE;
4788         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * MLX5_AGE_SIZE);
4789         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
4790         if (!pool) {
4791                 rte_errno = ENOMEM;
4792                 return NULL;
4793         }
4794         pool->raw = NULL;
4795         pool->is_aged = !!age;
4796         pool->query_gen = 0;
4797         pool->min_dcs = dcs;
4798         rte_spinlock_init(&pool->sl);
4799         rte_spinlock_init(&pool->csl);
4800         TAILQ_INIT(&pool->counters[0]);
4801         TAILQ_INIT(&pool->counters[1]);
4802         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
4803         rte_spinlock_lock(&cmng->pool_update_sl);
4804         pool->index = cmng->n_valid;
4805         if (pool->index == cmng->n && flow_dv_container_resize(dev)) {
4806                 mlx5_free(pool);
4807                 rte_spinlock_unlock(&cmng->pool_update_sl);
4808                 return NULL;
4809         }
4810         cmng->pools[pool->index] = pool;
4811         cmng->n_valid++;
4812         if (unlikely(fallback)) {
4813                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
4814
4815                 if (base < cmng->min_id)
4816                         cmng->min_id = base;
4817                 if (base > cmng->max_id)
4818                         cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
4819                 cmng->last_pool_idx = pool->index;
4820         }
4821         rte_spinlock_unlock(&cmng->pool_update_sl);
4822         return pool;
4823 }
4824
4825 /**
4826  * Prepare a new counter and/or a new counter pool.
4827  *
4828  * @param[in] dev
4829  *   Pointer to the Ethernet device structure.
4830  * @param[out] cnt_free
4831  *   Where to put the pointer of a new counter.
4832  * @param[in] age
4833  *   Whether the pool is for counter that was allocated for aging.
4834  *
4835  * @return
4836  *   The counter pool pointer and @p cnt_free is set on success,
4837  *   NULL otherwise and rte_errno is set.
4838  */
4839 static struct mlx5_flow_counter_pool *
4840 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
4841                              struct mlx5_flow_counter **cnt_free,
4842                              uint32_t age)
4843 {
4844         struct mlx5_priv *priv = dev->data->dev_private;
4845         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4846         struct mlx5_flow_counter_pool *pool;
4847         struct mlx5_counters tmp_tq;
4848         struct mlx5_devx_obj *dcs = NULL;
4849         struct mlx5_flow_counter *cnt;
4850         enum mlx5_counter_type cnt_type =
4851                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
4852         bool fallback = priv->sh->cmng.counter_fallback;
4853         uint32_t i;
4854
4855         if (fallback) {
4856                 /* bulk_bitmap must be 0 for single counter allocation. */
4857                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
4858                 if (!dcs)
4859                         return NULL;
4860                 pool = flow_dv_find_pool_by_id(cmng, dcs->id);
4861                 if (!pool) {
4862                         pool = flow_dv_pool_create(dev, dcs, age);
4863                         if (!pool) {
4864                                 mlx5_devx_cmd_destroy(dcs);
4865                                 return NULL;
4866                         }
4867                 }
4868                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
4869                 cnt = MLX5_POOL_GET_CNT(pool, i);
4870                 cnt->pool = pool;
4871                 cnt->dcs_when_free = dcs;
4872                 *cnt_free = cnt;
4873                 return pool;
4874         }
4875         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
4876         if (!dcs) {
4877                 rte_errno = ENODATA;
4878                 return NULL;
4879         }
4880         pool = flow_dv_pool_create(dev, dcs, age);
4881         if (!pool) {
4882                 mlx5_devx_cmd_destroy(dcs);
4883                 return NULL;
4884         }
4885         TAILQ_INIT(&tmp_tq);
4886         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
4887                 cnt = MLX5_POOL_GET_CNT(pool, i);
4888                 cnt->pool = pool;
4889                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
4890         }
4891         rte_spinlock_lock(&cmng->csl[cnt_type]);
4892         TAILQ_CONCAT(&cmng->counters[cnt_type], &tmp_tq, next);
4893         rte_spinlock_unlock(&cmng->csl[cnt_type]);
4894         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
4895         (*cnt_free)->pool = pool;
4896         return pool;
4897 }
4898
4899 /**
4900  * Allocate a flow counter.
4901  *
4902  * @param[in] dev
4903  *   Pointer to the Ethernet device structure.
4904  * @param[in] age
4905  *   Whether the counter was allocated for aging.
4906  *
4907  * @return
4908  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
4909  */
4910 static uint32_t
4911 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t age)
4912 {
4913         struct mlx5_priv *priv = dev->data->dev_private;
4914         struct mlx5_flow_counter_pool *pool = NULL;
4915         struct mlx5_flow_counter *cnt_free = NULL;
4916         bool fallback = priv->sh->cmng.counter_fallback;
4917         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4918         enum mlx5_counter_type cnt_type =
4919                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
4920         uint32_t cnt_idx;
4921
4922         if (!priv->config.devx) {
4923                 rte_errno = ENOTSUP;
4924                 return 0;
4925         }
4926         /* Get free counters from container. */
4927         rte_spinlock_lock(&cmng->csl[cnt_type]);
4928         cnt_free = TAILQ_FIRST(&cmng->counters[cnt_type]);
4929         if (cnt_free)
4930                 TAILQ_REMOVE(&cmng->counters[cnt_type], cnt_free, next);
4931         rte_spinlock_unlock(&cmng->csl[cnt_type]);
4932         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, age))
4933                 goto err;
4934         pool = cnt_free->pool;
4935         if (fallback)
4936                 cnt_free->dcs_when_active = cnt_free->dcs_when_free;
4937         /* Create a DV counter action only in the first time usage. */
4938         if (!cnt_free->action) {
4939                 uint16_t offset;
4940                 struct mlx5_devx_obj *dcs;
4941                 int ret;
4942
4943                 if (!fallback) {
4944                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
4945                         dcs = pool->min_dcs;
4946                 } else {
4947                         offset = 0;
4948                         dcs = cnt_free->dcs_when_free;
4949                 }
4950                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
4951                                                             &cnt_free->action);
4952                 if (ret) {
4953                         rte_errno = errno;
4954                         goto err;
4955                 }
4956         }
4957         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
4958                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
4959         /* Update the counter reset values. */
4960         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
4961                                  &cnt_free->bytes))
4962                 goto err;
4963         if (!fallback && !priv->sh->cmng.query_thread_on)
4964                 /* Start the asynchronous batch query by the host thread. */
4965                 mlx5_set_query_alarm(priv->sh);
4966         return cnt_idx;
4967 err:
4968         if (cnt_free) {
4969                 cnt_free->pool = pool;
4970                 if (fallback)
4971                         cnt_free->dcs_when_free = cnt_free->dcs_when_active;
4972                 rte_spinlock_lock(&cmng->csl[cnt_type]);
4973                 TAILQ_INSERT_TAIL(&cmng->counters[cnt_type], cnt_free, next);
4974                 rte_spinlock_unlock(&cmng->csl[cnt_type]);
4975         }
4976         return 0;
4977 }
4978
4979 /**
4980  * Allocate a shared flow counter.
4981  *
4982  * @param[in] ctx
4983  *   Pointer to the shared counter configuration.
4984  * @param[in] data
4985  *   Pointer to save the allocated counter index.
4986  *
4987  * @return
4988  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
4989  */
4990
4991 static int32_t
4992 flow_dv_counter_alloc_shared_cb(void *ctx, union mlx5_l3t_data *data)
4993 {
4994         struct mlx5_shared_counter_conf *conf = ctx;
4995         struct rte_eth_dev *dev = conf->dev;
4996         struct mlx5_flow_counter *cnt;
4997
4998         data->dword = flow_dv_counter_alloc(dev, 0);
4999         data->dword |= MLX5_CNT_SHARED_OFFSET;
5000         cnt = flow_dv_counter_get_by_idx(dev, data->dword, NULL);
5001         cnt->shared_info.id = conf->id;
5002         return 0;
5003 }
5004
5005 /**
5006  * Get a shared flow counter.
5007  *
5008  * @param[in] dev
5009  *   Pointer to the Ethernet device structure.
5010  * @param[in] id
5011  *   Counter identifier.
5012  *
5013  * @return
5014  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
5015  */
5016 static uint32_t
5017 flow_dv_counter_get_shared(struct rte_eth_dev *dev, uint32_t id)
5018 {
5019         struct mlx5_priv *priv = dev->data->dev_private;
5020         struct mlx5_shared_counter_conf conf = {
5021                 .dev = dev,
5022                 .id = id,
5023         };
5024         union mlx5_l3t_data data = {
5025                 .dword = 0,
5026         };
5027
5028         mlx5_l3t_prepare_entry(priv->sh->cnt_id_tbl, id, &data,
5029                                flow_dv_counter_alloc_shared_cb, &conf);
5030         return data.dword;
5031 }
5032
5033 /**
5034  * Get age param from counter index.
5035  *
5036  * @param[in] dev
5037  *   Pointer to the Ethernet device structure.
5038  * @param[in] counter
5039  *   Index to the counter handler.
5040  *
5041  * @return
5042  *   The aging parameter specified for the counter index.
5043  */
5044 static struct mlx5_age_param*
5045 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
5046                                 uint32_t counter)
5047 {
5048         struct mlx5_flow_counter *cnt;
5049         struct mlx5_flow_counter_pool *pool = NULL;
5050
5051         flow_dv_counter_get_by_idx(dev, counter, &pool);
5052         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
5053         cnt = MLX5_POOL_GET_CNT(pool, counter);
5054         return MLX5_CNT_TO_AGE(cnt);
5055 }
5056
5057 /**
5058  * Remove a flow counter from aged counter list.
5059  *
5060  * @param[in] dev
5061  *   Pointer to the Ethernet device structure.
5062  * @param[in] counter
5063  *   Index to the counter handler.
5064  * @param[in] cnt
5065  *   Pointer to the counter handler.
5066  */
5067 static void
5068 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
5069                                 uint32_t counter, struct mlx5_flow_counter *cnt)
5070 {
5071         struct mlx5_age_info *age_info;
5072         struct mlx5_age_param *age_param;
5073         struct mlx5_priv *priv = dev->data->dev_private;
5074         uint16_t expected = AGE_CANDIDATE;
5075
5076         age_info = GET_PORT_AGE_INFO(priv);
5077         age_param = flow_dv_counter_idx_get_age(dev, counter);
5078         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
5079                                          AGE_FREE, false, __ATOMIC_RELAXED,
5080                                          __ATOMIC_RELAXED)) {
5081                 /**
5082                  * We need the lock even it is age timeout,
5083                  * since counter may still in process.
5084                  */
5085                 rte_spinlock_lock(&age_info->aged_sl);
5086                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
5087                 rte_spinlock_unlock(&age_info->aged_sl);
5088                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
5089         }
5090 }
5091
5092 /**
5093  * Release a flow counter.
5094  *
5095  * @param[in] dev
5096  *   Pointer to the Ethernet device structure.
5097  * @param[in] counter
5098  *   Index to the counter handler.
5099  */
5100 static void
5101 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
5102 {
5103         struct mlx5_priv *priv = dev->data->dev_private;
5104         struct mlx5_flow_counter_pool *pool = NULL;
5105         struct mlx5_flow_counter *cnt;
5106         enum mlx5_counter_type cnt_type;
5107
5108         if (!counter)
5109                 return;
5110         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
5111         MLX5_ASSERT(pool);
5112         if (IS_SHARED_CNT(counter) &&
5113             mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, cnt->shared_info.id))
5114                 return;
5115         if (pool->is_aged)
5116                 flow_dv_counter_remove_from_age(dev, counter, cnt);
5117         cnt->pool = pool;
5118         /*
5119          * Put the counter back to list to be updated in none fallback mode.
5120          * Currently, we are using two list alternately, while one is in query,
5121          * add the freed counter to the other list based on the pool query_gen
5122          * value. After query finishes, add counter the list to the global
5123          * container counter list. The list changes while query starts. In
5124          * this case, lock will not be needed as query callback and release
5125          * function both operate with the different list.
5126          *
5127          */
5128         if (!priv->sh->cmng.counter_fallback) {
5129                 rte_spinlock_lock(&pool->csl);
5130                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
5131                 rte_spinlock_unlock(&pool->csl);
5132         } else {
5133                 cnt->dcs_when_free = cnt->dcs_when_active;
5134                 cnt_type = pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
5135                                            MLX5_COUNTER_TYPE_ORIGIN;
5136                 rte_spinlock_lock(&priv->sh->cmng.csl[cnt_type]);
5137                 TAILQ_INSERT_TAIL(&priv->sh->cmng.counters[cnt_type],
5138                                   cnt, next);
5139                 rte_spinlock_unlock(&priv->sh->cmng.csl[cnt_type]);
5140         }
5141 }
5142
5143 /**
5144  * Verify the @p attributes will be correctly understood by the NIC and store
5145  * them in the @p flow if everything is correct.
5146  *
5147  * @param[in] dev
5148  *   Pointer to dev struct.
5149  * @param[in] attributes
5150  *   Pointer to flow attributes
5151  * @param[in] external
5152  *   This flow rule is created by request external to PMD.
5153  * @param[out] error
5154  *   Pointer to error structure.
5155  *
5156  * @return
5157  *   - 0 on success and non root table.
5158  *   - 1 on success and root table.
5159  *   - a negative errno value otherwise and rte_errno is set.
5160  */
5161 static int
5162 flow_dv_validate_attributes(struct rte_eth_dev *dev,
5163                             const struct rte_flow_attr *attributes,
5164                             bool external __rte_unused,
5165                             struct rte_flow_error *error)
5166 {
5167         struct mlx5_priv *priv = dev->data->dev_private;
5168         uint32_t priority_max = priv->config.flow_prio - 1;
5169         int ret = 0;
5170
5171 #ifndef HAVE_MLX5DV_DR
5172         if (attributes->group)
5173                 return rte_flow_error_set(error, ENOTSUP,
5174                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
5175                                           NULL,
5176                                           "groups are not supported");
5177 #else
5178         uint32_t table = 0;
5179
5180         ret = mlx5_flow_group_to_table(attributes, external,
5181                                        attributes->group, !!priv->fdb_def_rule,
5182                                        &table, error);
5183         if (ret)
5184                 return ret;
5185         if (!table)
5186                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
5187 #endif
5188         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
5189             attributes->priority >= priority_max)
5190                 return rte_flow_error_set(error, ENOTSUP,
5191                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
5192                                           NULL,
5193                                           "priority out of range");
5194         if (attributes->transfer) {
5195                 if (!priv->config.dv_esw_en)
5196                         return rte_flow_error_set
5197                                 (error, ENOTSUP,
5198                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5199                                  "E-Switch dr is not supported");
5200                 if (!(priv->representor || priv->master))
5201                         return rte_flow_error_set
5202                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5203                                  NULL, "E-Switch configuration can only be"
5204                                  " done by a master or a representor device");
5205                 if (attributes->egress)
5206                         return rte_flow_error_set
5207                                 (error, ENOTSUP,
5208                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
5209                                  "egress is not supported");
5210         }
5211         if (!(attributes->egress ^ attributes->ingress))
5212                 return rte_flow_error_set(error, ENOTSUP,
5213                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
5214                                           "must specify exactly one of "
5215                                           "ingress or egress");
5216         return ret;
5217 }
5218
5219 /**
5220  * Internal validation function. For validating both actions and items.
5221  *
5222  * @param[in] dev
5223  *   Pointer to the rte_eth_dev structure.
5224  * @param[in] attr
5225  *   Pointer to the flow attributes.
5226  * @param[in] items
5227  *   Pointer to the list of items.
5228  * @param[in] actions
5229  *   Pointer to the list of actions.
5230  * @param[in] external
5231  *   This flow rule is created by request external to PMD.
5232  * @param[in] hairpin
5233  *   Number of hairpin TX actions, 0 means classic flow.
5234  * @param[out] error
5235  *   Pointer to the error structure.
5236  *
5237  * @return
5238  *   0 on success, a negative errno value otherwise and rte_errno is set.
5239  */
5240 static int
5241 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
5242                  const struct rte_flow_item items[],
5243                  const struct rte_flow_action actions[],
5244                  bool external, int hairpin, struct rte_flow_error *error)
5245 {
5246         int ret;
5247         uint64_t action_flags = 0;
5248         uint64_t item_flags = 0;
5249         uint64_t last_item = 0;
5250         uint8_t next_protocol = 0xff;
5251         uint16_t ether_type = 0;
5252         int actions_n = 0;
5253         uint8_t item_ipv6_proto = 0;
5254         const struct rte_flow_item *gre_item = NULL;
5255         const struct rte_flow_action_raw_decap *decap;
5256         const struct rte_flow_action_raw_encap *encap;
5257         const struct rte_flow_action_rss *rss;
5258         const struct rte_flow_item_tcp nic_tcp_mask = {
5259                 .hdr = {
5260                         .tcp_flags = 0xFF,
5261                         .src_port = RTE_BE16(UINT16_MAX),
5262                         .dst_port = RTE_BE16(UINT16_MAX),
5263                 }
5264         };
5265         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
5266                 .hdr = {
5267                         .src_addr =
5268                         "\xff\xff\xff\xff\xff\xff\xff\xff"
5269                         "\xff\xff\xff\xff\xff\xff\xff\xff",
5270                         .dst_addr =
5271                         "\xff\xff\xff\xff\xff\xff\xff\xff"
5272                         "\xff\xff\xff\xff\xff\xff\xff\xff",
5273                         .vtc_flow = RTE_BE32(0xffffffff),
5274                         .proto = 0xff,
5275                         .hop_limits = 0xff,
5276                 },
5277                 .has_frag_ext = 1,
5278         };
5279         const struct rte_flow_item_ecpri nic_ecpri_mask = {
5280                 .hdr = {
5281                         .common = {
5282                                 .u32 =
5283                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
5284                                         .type = 0xFF,
5285                                         }).u32),
5286                         },
5287                         .dummy[0] = 0xffffffff,
5288                 },
5289         };
5290         struct mlx5_priv *priv = dev->data->dev_private;
5291         struct mlx5_dev_config *dev_conf = &priv->config;
5292         uint16_t queue_index = 0xFFFF;
5293         const struct rte_flow_item_vlan *vlan_m = NULL;
5294         int16_t rw_act_num = 0;
5295         uint64_t is_root;
5296
5297         if (items == NULL)
5298                 return -1;
5299         ret = flow_dv_validate_attributes(dev, attr, external, error);
5300         if (ret < 0)
5301                 return ret;
5302         is_root = (uint64_t)ret;
5303         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5304                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
5305                 int type = items->type;
5306
5307                 if (!mlx5_flow_os_item_supported(type))
5308                         return rte_flow_error_set(error, ENOTSUP,
5309                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5310                                                   NULL, "item not supported");
5311                 switch (type) {
5312                 case RTE_FLOW_ITEM_TYPE_VOID:
5313                         break;
5314                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
5315                         ret = flow_dv_validate_item_port_id
5316                                         (dev, items, attr, item_flags, error);
5317                         if (ret < 0)
5318                                 return ret;
5319                         last_item = MLX5_FLOW_ITEM_PORT_ID;
5320                         break;
5321                 case RTE_FLOW_ITEM_TYPE_ETH:
5322                         ret = mlx5_flow_validate_item_eth(items, item_flags,
5323                                                           error);
5324                         if (ret < 0)
5325                                 return ret;
5326                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
5327                                              MLX5_FLOW_LAYER_OUTER_L2;
5328                         if (items->mask != NULL && items->spec != NULL) {
5329                                 ether_type =
5330                                         ((const struct rte_flow_item_eth *)
5331                                          items->spec)->type;
5332                                 ether_type &=
5333                                         ((const struct rte_flow_item_eth *)
5334                                          items->mask)->type;
5335                                 ether_type = rte_be_to_cpu_16(ether_type);
5336                         } else {
5337                                 ether_type = 0;
5338                         }
5339                         break;
5340                 case RTE_FLOW_ITEM_TYPE_VLAN:
5341                         ret = flow_dv_validate_item_vlan(items, item_flags,
5342                                                          dev, error);
5343                         if (ret < 0)
5344                                 return ret;
5345                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
5346                                              MLX5_FLOW_LAYER_OUTER_VLAN;
5347                         if (items->mask != NULL && items->spec != NULL) {
5348                                 ether_type =
5349                                         ((const struct rte_flow_item_vlan *)
5350                                          items->spec)->inner_type;
5351                                 ether_type &=
5352                                         ((const struct rte_flow_item_vlan *)
5353                                          items->mask)->inner_type;
5354                                 ether_type = rte_be_to_cpu_16(ether_type);
5355                         } else {
5356                                 ether_type = 0;
5357                         }
5358                         /* Store outer VLAN mask for of_push_vlan action. */
5359                         if (!tunnel)
5360                                 vlan_m = items->mask;
5361                         break;
5362                 case RTE_FLOW_ITEM_TYPE_IPV4:
5363                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5364                                                   &item_flags, &tunnel);
5365                         ret = flow_dv_validate_item_ipv4(items, item_flags,
5366                                                          last_item, ether_type,
5367                                                          error);
5368                         if (ret < 0)
5369                                 return ret;
5370                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5371                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5372                         if (items->mask != NULL &&
5373                             ((const struct rte_flow_item_ipv4 *)
5374                              items->mask)->hdr.next_proto_id) {
5375                                 next_protocol =
5376                                         ((const struct rte_flow_item_ipv4 *)
5377                                          (items->spec))->hdr.next_proto_id;
5378                                 next_protocol &=
5379                                         ((const struct rte_flow_item_ipv4 *)
5380                                          (items->mask))->hdr.next_proto_id;
5381                         } else {
5382                                 /* Reset for inner layer. */
5383                                 next_protocol = 0xff;
5384                         }
5385                         break;
5386                 case RTE_FLOW_ITEM_TYPE_IPV6:
5387                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5388                                                   &item_flags, &tunnel);
5389                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
5390                                                            last_item,
5391                                                            ether_type,
5392                                                            &nic_ipv6_mask,
5393                                                            error);
5394                         if (ret < 0)
5395                                 return ret;
5396                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5397                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5398                         if (items->mask != NULL &&
5399                             ((const struct rte_flow_item_ipv6 *)
5400                              items->mask)->hdr.proto) {
5401                                 item_ipv6_proto =
5402                                         ((const struct rte_flow_item_ipv6 *)
5403                                          items->spec)->hdr.proto;
5404                                 next_protocol =
5405                                         ((const struct rte_flow_item_ipv6 *)
5406                                          items->spec)->hdr.proto;
5407                                 next_protocol &=
5408                                         ((const struct rte_flow_item_ipv6 *)
5409                                          items->mask)->hdr.proto;
5410                         } else {
5411                                 /* Reset for inner layer. */
5412                                 next_protocol = 0xff;
5413                         }
5414                         break;
5415                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
5416                         ret = flow_dv_validate_item_ipv6_frag_ext(items,
5417                                                                   item_flags,
5418                                                                   error);
5419                         if (ret < 0)
5420                                 return ret;
5421                         last_item = tunnel ?
5422                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
5423                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
5424                         if (items->mask != NULL &&
5425                             ((const struct rte_flow_item_ipv6_frag_ext *)
5426                              items->mask)->hdr.next_header) {
5427                                 next_protocol =
5428                                 ((const struct rte_flow_item_ipv6_frag_ext *)
5429                                  items->spec)->hdr.next_header;
5430                                 next_protocol &=
5431                                 ((const struct rte_flow_item_ipv6_frag_ext *)
5432                                  items->mask)->hdr.next_header;
5433                         } else {
5434                                 /* Reset for inner layer. */
5435                                 next_protocol = 0xff;
5436                         }
5437                         break;
5438                 case RTE_FLOW_ITEM_TYPE_TCP:
5439                         ret = mlx5_flow_validate_item_tcp
5440                                                 (items, item_flags,
5441                                                  next_protocol,
5442                                                  &nic_tcp_mask,
5443                                                  error);
5444                         if (ret < 0)
5445                                 return ret;
5446                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5447                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5448                         break;
5449                 case RTE_FLOW_ITEM_TYPE_UDP:
5450                         ret = mlx5_flow_validate_item_udp(items, item_flags,
5451                                                           next_protocol,
5452                                                           error);
5453                         if (ret < 0)
5454                                 return ret;
5455                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5456                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5457                         break;
5458                 case RTE_FLOW_ITEM_TYPE_GRE:
5459                         ret = mlx5_flow_validate_item_gre(items, item_flags,
5460                                                           next_protocol, error);
5461                         if (ret < 0)
5462                                 return ret;
5463                         gre_item = items;
5464                         last_item = MLX5_FLOW_LAYER_GRE;
5465                         break;
5466                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5467                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
5468                                                             next_protocol,
5469                                                             error);
5470                         if (ret < 0)
5471                                 return ret;
5472                         last_item = MLX5_FLOW_LAYER_NVGRE;
5473                         break;
5474                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5475                         ret = mlx5_flow_validate_item_gre_key
5476                                 (items, item_flags, gre_item, error);
5477                         if (ret < 0)
5478                                 return ret;
5479                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5480                         break;
5481                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5482                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
5483                                                             error);
5484                         if (ret < 0)
5485                                 return ret;
5486                         last_item = MLX5_FLOW_LAYER_VXLAN;
5487                         break;
5488                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5489                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
5490                                                                 item_flags, dev,
5491                                                                 error);
5492                         if (ret < 0)
5493                                 return ret;
5494                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5495                         break;
5496                 case RTE_FLOW_ITEM_TYPE_GENEVE:
5497                         ret = mlx5_flow_validate_item_geneve(items,
5498                                                              item_flags, dev,
5499                                                              error);
5500                         if (ret < 0)
5501                                 return ret;
5502                         last_item = MLX5_FLOW_LAYER_GENEVE;
5503                         break;
5504                 case RTE_FLOW_ITEM_TYPE_MPLS:
5505                         ret = mlx5_flow_validate_item_mpls(dev, items,
5506                                                            item_flags,
5507                                                            last_item, error);
5508                         if (ret < 0)
5509                                 return ret;
5510                         last_item = MLX5_FLOW_LAYER_MPLS;
5511                         break;
5512
5513                 case RTE_FLOW_ITEM_TYPE_MARK:
5514                         ret = flow_dv_validate_item_mark(dev, items, attr,
5515                                                          error);
5516                         if (ret < 0)
5517                                 return ret;
5518                         last_item = MLX5_FLOW_ITEM_MARK;
5519                         break;
5520                 case RTE_FLOW_ITEM_TYPE_META:
5521                         ret = flow_dv_validate_item_meta(dev, items, attr,
5522                                                          error);
5523                         if (ret < 0)
5524                                 return ret;
5525                         last_item = MLX5_FLOW_ITEM_METADATA;
5526                         break;
5527                 case RTE_FLOW_ITEM_TYPE_ICMP:
5528                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
5529                                                            next_protocol,
5530                                                            error);
5531                         if (ret < 0)
5532                                 return ret;
5533                         last_item = MLX5_FLOW_LAYER_ICMP;
5534                         break;
5535                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5536                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
5537                                                             next_protocol,
5538                                                             error);
5539                         if (ret < 0)
5540                                 return ret;
5541                         item_ipv6_proto = IPPROTO_ICMPV6;
5542                         last_item = MLX5_FLOW_LAYER_ICMP6;
5543                         break;
5544                 case RTE_FLOW_ITEM_TYPE_TAG:
5545                         ret = flow_dv_validate_item_tag(dev, items,
5546                                                         attr, error);
5547                         if (ret < 0)
5548                                 return ret;
5549                         last_item = MLX5_FLOW_ITEM_TAG;
5550                         break;
5551                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
5552                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
5553                         break;
5554                 case RTE_FLOW_ITEM_TYPE_GTP:
5555                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
5556                                                         error);
5557                         if (ret < 0)
5558                                 return ret;
5559                         last_item = MLX5_FLOW_LAYER_GTP;
5560                         break;
5561                 case RTE_FLOW_ITEM_TYPE_ECPRI:
5562                         /* Capacity will be checked in the translate stage. */
5563                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
5564                                                             last_item,
5565                                                             ether_type,
5566                                                             &nic_ecpri_mask,
5567                                                             error);
5568                         if (ret < 0)
5569                                 return ret;
5570                         last_item = MLX5_FLOW_LAYER_ECPRI;
5571                         break;
5572                 default:
5573                         return rte_flow_error_set(error, ENOTSUP,
5574                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5575                                                   NULL, "item not supported");
5576                 }
5577                 item_flags |= last_item;
5578         }
5579         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5580                 int type = actions->type;
5581
5582                 if (!mlx5_flow_os_action_supported(type))
5583                         return rte_flow_error_set(error, ENOTSUP,
5584                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5585                                                   actions,
5586                                                   "action not supported");
5587                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5588                         return rte_flow_error_set(error, ENOTSUP,
5589                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5590                                                   actions, "too many actions");
5591                 switch (type) {
5592                 case RTE_FLOW_ACTION_TYPE_VOID:
5593                         break;
5594                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5595                         ret = flow_dv_validate_action_port_id(dev,
5596                                                               action_flags,
5597                                                               actions,
5598                                                               attr,
5599                                                               error);
5600                         if (ret)
5601                                 return ret;
5602                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5603                         ++actions_n;
5604                         break;
5605                 case RTE_FLOW_ACTION_TYPE_FLAG:
5606                         ret = flow_dv_validate_action_flag(dev, action_flags,
5607                                                            attr, error);
5608                         if (ret < 0)
5609                                 return ret;
5610                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5611                                 /* Count all modify-header actions as one. */
5612                                 if (!(action_flags &
5613                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5614                                         ++actions_n;
5615                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
5616                                                 MLX5_FLOW_ACTION_MARK_EXT;
5617                         } else {
5618                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
5619                                 ++actions_n;
5620                         }
5621                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5622                         break;
5623                 case RTE_FLOW_ACTION_TYPE_MARK:
5624                         ret = flow_dv_validate_action_mark(dev, actions,
5625                                                            action_flags,
5626                                                            attr, error);
5627                         if (ret < 0)
5628                                 return ret;
5629                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5630                                 /* Count all modify-header actions as one. */
5631                                 if (!(action_flags &
5632                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5633                                         ++actions_n;
5634                                 action_flags |= MLX5_FLOW_ACTION_MARK |
5635                                                 MLX5_FLOW_ACTION_MARK_EXT;
5636                         } else {
5637                                 action_flags |= MLX5_FLOW_ACTION_MARK;
5638                                 ++actions_n;
5639                         }
5640                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5641                         break;
5642                 case RTE_FLOW_ACTION_TYPE_SET_META:
5643                         ret = flow_dv_validate_action_set_meta(dev, actions,
5644                                                                action_flags,
5645                                                                attr, error);
5646                         if (ret < 0)
5647                                 return ret;
5648                         /* Count all modify-header actions as one action. */
5649                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5650                                 ++actions_n;
5651                         action_flags |= MLX5_FLOW_ACTION_SET_META;
5652                         rw_act_num += MLX5_ACT_NUM_SET_META;
5653                         break;
5654                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5655                         ret = flow_dv_validate_action_set_tag(dev, actions,
5656                                                               action_flags,
5657                                                               attr, error);
5658                         if (ret < 0)
5659                                 return ret;
5660                         /* Count all modify-header actions as one action. */
5661                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5662                                 ++actions_n;
5663                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
5664                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5665                         break;
5666                 case RTE_FLOW_ACTION_TYPE_DROP:
5667                         ret = mlx5_flow_validate_action_drop(action_flags,
5668                                                              attr, error);
5669                         if (ret < 0)
5670                                 return ret;
5671                         action_flags |= MLX5_FLOW_ACTION_DROP;
5672                         ++actions_n;
5673                         break;
5674                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5675                         ret = mlx5_flow_validate_action_queue(actions,
5676                                                               action_flags, dev,
5677                                                               attr, error);
5678                         if (ret < 0)
5679                                 return ret;
5680                         queue_index = ((const struct rte_flow_action_queue *)
5681                                                         (actions->conf))->index;
5682                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
5683                         ++actions_n;
5684                         break;
5685                 case RTE_FLOW_ACTION_TYPE_RSS:
5686                         rss = actions->conf;
5687                         ret = mlx5_flow_validate_action_rss(actions,
5688                                                             action_flags, dev,
5689                                                             attr, item_flags,
5690                                                             error);
5691                         if (ret < 0)
5692                                 return ret;
5693                         if (rss != NULL && rss->queue_num)
5694                                 queue_index = rss->queue[0];
5695                         action_flags |= MLX5_FLOW_ACTION_RSS;
5696                         ++actions_n;
5697                         break;
5698                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
5699                         ret =
5700                         mlx5_flow_validate_action_default_miss(action_flags,
5701                                         attr, error);
5702                         if (ret < 0)
5703                                 return ret;
5704                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
5705                         ++actions_n;
5706                         break;
5707                 case RTE_FLOW_ACTION_TYPE_COUNT:
5708                         ret = flow_dv_validate_action_count(dev, error);
5709                         if (ret < 0)
5710                                 return ret;
5711                         action_flags |= MLX5_FLOW_ACTION_COUNT;
5712                         ++actions_n;
5713                         break;
5714                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5715                         if (flow_dv_validate_action_pop_vlan(dev,
5716                                                              action_flags,
5717                                                              actions,
5718                                                              item_flags, attr,
5719                                                              error))
5720                                 return -rte_errno;
5721                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5722                         ++actions_n;
5723                         break;
5724                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5725                         ret = flow_dv_validate_action_push_vlan(dev,
5726                                                                 action_flags,
5727                                                                 vlan_m,
5728                                                                 actions, attr,
5729                                                                 error);
5730                         if (ret < 0)
5731                                 return ret;
5732                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5733                         ++actions_n;
5734                         break;
5735                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5736                         ret = flow_dv_validate_action_set_vlan_pcp
5737                                                 (action_flags, actions, error);
5738                         if (ret < 0)
5739                                 return ret;
5740                         /* Count PCP with push_vlan command. */
5741                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
5742                         break;
5743                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5744                         ret = flow_dv_validate_action_set_vlan_vid
5745                                                 (item_flags, action_flags,
5746                                                  actions, error);
5747                         if (ret < 0)
5748                                 return ret;
5749                         /* Count VID with push_vlan command. */
5750                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
5751                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
5752                         break;
5753                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5754                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5755                         ret = flow_dv_validate_action_l2_encap(dev,
5756                                                                action_flags,
5757                                                                actions, attr,
5758                                                                error);
5759                         if (ret < 0)
5760                                 return ret;
5761                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
5762                         ++actions_n;
5763                         break;
5764                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5765                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5766                         ret = flow_dv_validate_action_decap(dev, action_flags,
5767                                                             attr, error);
5768                         if (ret < 0)
5769                                 return ret;
5770                         action_flags |= MLX5_FLOW_ACTION_DECAP;
5771                         ++actions_n;
5772                         break;
5773                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5774                         ret = flow_dv_validate_action_raw_encap_decap
5775                                 (dev, NULL, actions->conf, attr, &action_flags,
5776                                  &actions_n, error);
5777                         if (ret < 0)
5778                                 return ret;
5779                         break;
5780                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5781                         decap = actions->conf;
5782                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5783                                 ;
5784                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5785                                 encap = NULL;
5786                                 actions--;
5787                         } else {
5788                                 encap = actions->conf;
5789                         }
5790                         ret = flow_dv_validate_action_raw_encap_decap
5791                                            (dev,
5792                                             decap ? decap : &empty_decap, encap,
5793                                             attr, &action_flags, &actions_n,
5794                                             error);
5795                         if (ret < 0)
5796                                 return ret;
5797                         break;
5798                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5799                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5800                         ret = flow_dv_validate_action_modify_mac(action_flags,
5801                                                                  actions,
5802                                                                  item_flags,
5803                                                                  error);
5804                         if (ret < 0)
5805                                 return ret;
5806                         /* Count all modify-header actions as one action. */
5807                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5808                                 ++actions_n;
5809                         action_flags |= actions->type ==
5810                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5811                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
5812                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
5813                         /*
5814                          * Even if the source and destination MAC addresses have
5815                          * overlap in the header with 4B alignment, the convert
5816                          * function will handle them separately and 4 SW actions
5817                          * will be created. And 2 actions will be added each
5818                          * time no matter how many bytes of address will be set.
5819                          */
5820                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
5821                         break;
5822                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5823                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5824                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
5825                                                                   actions,
5826                                                                   item_flags,
5827                                                                   error);
5828                         if (ret < 0)
5829                                 return ret;
5830                         /* Count all modify-header actions as one action. */
5831                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5832                                 ++actions_n;
5833                         action_flags |= actions->type ==
5834                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5835                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
5836                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
5837                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
5838                         break;
5839                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5840                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5841                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
5842                                                                   actions,
5843                                                                   item_flags,
5844                                                                   error);
5845                         if (ret < 0)
5846                                 return ret;
5847                         if (item_ipv6_proto == IPPROTO_ICMPV6)
5848                                 return rte_flow_error_set(error, ENOTSUP,
5849                                         RTE_FLOW_ERROR_TYPE_ACTION,
5850                                         actions,
5851                                         "Can't change header "
5852                                         "with ICMPv6 proto");
5853                         /* Count all modify-header actions as one action. */
5854                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5855                                 ++actions_n;
5856                         action_flags |= actions->type ==
5857                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5858                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
5859                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
5860                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
5861                         break;
5862                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5863                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5864                         ret = flow_dv_validate_action_modify_tp(action_flags,
5865                                                                 actions,
5866                                                                 item_flags,
5867                                                                 error);
5868                         if (ret < 0)
5869                                 return ret;
5870                         /* Count all modify-header actions as one action. */
5871                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5872                                 ++actions_n;
5873                         action_flags |= actions->type ==
5874                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5875                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
5876                                                 MLX5_FLOW_ACTION_SET_TP_DST;
5877                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
5878                         break;
5879                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5880                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5881                         ret = flow_dv_validate_action_modify_ttl(action_flags,
5882                                                                  actions,
5883                                                                  item_flags,
5884                                                                  error);
5885                         if (ret < 0)
5886                                 return ret;
5887                         /* Count all modify-header actions as one action. */
5888                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5889                                 ++actions_n;
5890                         action_flags |= actions->type ==
5891                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
5892                                                 MLX5_FLOW_ACTION_SET_TTL :
5893                                                 MLX5_FLOW_ACTION_DEC_TTL;
5894                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
5895                         break;
5896                 case RTE_FLOW_ACTION_TYPE_JUMP:
5897                         ret = flow_dv_validate_action_jump(actions,
5898                                                            action_flags,
5899                                                            attr, external,
5900                                                            error);
5901                         if (ret)
5902                                 return ret;
5903                         ++actions_n;
5904                         action_flags |= MLX5_FLOW_ACTION_JUMP;
5905                         break;
5906                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5907                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5908                         ret = flow_dv_validate_action_modify_tcp_seq
5909                                                                 (action_flags,
5910                                                                  actions,
5911                                                                  item_flags,
5912                                                                  error);
5913                         if (ret < 0)
5914                                 return ret;
5915                         /* Count all modify-header actions as one action. */
5916                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5917                                 ++actions_n;
5918                         action_flags |= actions->type ==
5919                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5920                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
5921                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5922                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
5923                         break;
5924                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5925                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5926                         ret = flow_dv_validate_action_modify_tcp_ack
5927                                                                 (action_flags,
5928                                                                  actions,
5929                                                                  item_flags,
5930                                                                  error);
5931                         if (ret < 0)
5932                                 return ret;
5933                         /* Count all modify-header actions as one action. */
5934                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5935                                 ++actions_n;
5936                         action_flags |= actions->type ==
5937                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
5938                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
5939                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
5940                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
5941                         break;
5942                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
5943                         break;
5944                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
5945                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
5946                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5947                         break;
5948                 case RTE_FLOW_ACTION_TYPE_METER:
5949                         ret = mlx5_flow_validate_action_meter(dev,
5950                                                               action_flags,
5951                                                               actions, attr,
5952                                                               error);
5953                         if (ret < 0)
5954                                 return ret;
5955                         action_flags |= MLX5_FLOW_ACTION_METER;
5956                         ++actions_n;
5957                         /* Meter action will add one more TAG action. */
5958                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5959                         break;
5960                 case RTE_FLOW_ACTION_TYPE_AGE:
5961                         ret = flow_dv_validate_action_age(action_flags,
5962                                                           actions, dev,
5963                                                           error);
5964                         if (ret < 0)
5965                                 return ret;
5966                         action_flags |= MLX5_FLOW_ACTION_AGE;
5967                         ++actions_n;
5968                         break;
5969                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5970                         ret = flow_dv_validate_action_modify_ipv4_dscp
5971                                                          (action_flags,
5972                                                           actions,
5973                                                           item_flags,
5974                                                           error);
5975                         if (ret < 0)
5976                                 return ret;
5977                         /* Count all modify-header actions as one action. */
5978                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5979                                 ++actions_n;
5980                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
5981                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5982                         break;
5983                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5984                         ret = flow_dv_validate_action_modify_ipv6_dscp
5985                                                                 (action_flags,
5986                                                                  actions,
5987                                                                  item_flags,
5988                                                                  error);
5989                         if (ret < 0)
5990                                 return ret;
5991                         /* Count all modify-header actions as one action. */
5992                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5993                                 ++actions_n;
5994                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
5995                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5996                         break;
5997                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
5998                         ret = flow_dv_validate_action_sample(action_flags,
5999                                                              actions, dev,
6000                                                              attr, error);
6001                         if (ret < 0)
6002                                 return ret;
6003                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
6004                         ++actions_n;
6005                         break;
6006                 default:
6007                         return rte_flow_error_set(error, ENOTSUP,
6008                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6009                                                   actions,
6010                                                   "action not supported");
6011                 }
6012         }
6013         /*
6014          * Validate the drop action mutual exclusion with other actions.
6015          * Drop action is mutually-exclusive with any other action, except for
6016          * Count action.
6017          */
6018         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
6019             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
6020                 return rte_flow_error_set(error, EINVAL,
6021                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
6022                                           "Drop action is mutually-exclusive "
6023                                           "with any other action, except for "
6024                                           "Count action");
6025         /* Eswitch has few restrictions on using items and actions */
6026         if (attr->transfer) {
6027                 if (!mlx5_flow_ext_mreg_supported(dev) &&
6028                     action_flags & MLX5_FLOW_ACTION_FLAG)
6029                         return rte_flow_error_set(error, ENOTSUP,
6030                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6031                                                   NULL,
6032                                                   "unsupported action FLAG");
6033                 if (!mlx5_flow_ext_mreg_supported(dev) &&
6034                     action_flags & MLX5_FLOW_ACTION_MARK)
6035                         return rte_flow_error_set(error, ENOTSUP,
6036                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6037                                                   NULL,
6038                                                   "unsupported action MARK");
6039                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
6040                         return rte_flow_error_set(error, ENOTSUP,
6041                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6042                                                   NULL,
6043                                                   "unsupported action QUEUE");
6044                 if (action_flags & MLX5_FLOW_ACTION_RSS)
6045                         return rte_flow_error_set(error, ENOTSUP,
6046                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6047                                                   NULL,
6048                                                   "unsupported action RSS");
6049                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
6050                         return rte_flow_error_set(error, EINVAL,
6051                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6052                                                   actions,
6053                                                   "no fate action is found");
6054         } else {
6055                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
6056                         return rte_flow_error_set(error, EINVAL,
6057                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6058                                                   actions,
6059                                                   "no fate action is found");
6060         }
6061         /* Continue validation for Xcap and VLAN actions.*/
6062         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
6063                              MLX5_FLOW_VLAN_ACTIONS)) &&
6064             (queue_index == 0xFFFF ||
6065              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
6066                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
6067                     MLX5_FLOW_XCAP_ACTIONS)
6068                         return rte_flow_error_set(error, ENOTSUP,
6069                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6070                                                   NULL, "encap and decap "
6071                                                   "combination aren't supported");
6072                 if (!attr->transfer && attr->ingress) {
6073                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
6074                                 return rte_flow_error_set
6075                                                 (error, ENOTSUP,
6076                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6077                                                  NULL, "encap is not supported"
6078                                                  " for ingress traffic");
6079                         else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
6080                                 return rte_flow_error_set
6081                                                 (error, ENOTSUP,
6082                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6083                                                  NULL, "push VLAN action not "
6084                                                  "supported for ingress");
6085                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
6086                                         MLX5_FLOW_VLAN_ACTIONS)
6087                                 return rte_flow_error_set
6088                                                 (error, ENOTSUP,
6089                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6090                                                  NULL, "no support for "
6091                                                  "multiple VLAN actions");
6092                 }
6093         }
6094         /* Hairpin flow will add one more TAG action. */
6095         if (hairpin > 0)
6096                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6097         /* extra metadata enabled: one more TAG action will be add. */
6098         if (dev_conf->dv_flow_en &&
6099             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
6100             mlx5_flow_ext_mreg_supported(dev))
6101                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6102         if ((uint32_t)rw_act_num >
6103                         flow_dv_modify_hdr_action_max(dev, is_root)) {
6104                 return rte_flow_error_set(error, ENOTSUP,
6105                                           RTE_FLOW_ERROR_TYPE_ACTION,
6106                                           NULL, "too many header modify"
6107                                           " actions to support");
6108         }
6109         return 0;
6110 }
6111
6112 /**
6113  * Internal preparation function. Allocates the DV flow size,
6114  * this size is constant.
6115  *
6116  * @param[in] dev
6117  *   Pointer to the rte_eth_dev structure.
6118  * @param[in] attr
6119  *   Pointer to the flow attributes.
6120  * @param[in] items
6121  *   Pointer to the list of items.
6122  * @param[in] actions
6123  *   Pointer to the list of actions.
6124  * @param[out] error
6125  *   Pointer to the error structure.
6126  *
6127  * @return
6128  *   Pointer to mlx5_flow object on success,
6129  *   otherwise NULL and rte_errno is set.
6130  */
6131 static struct mlx5_flow *
6132 flow_dv_prepare(struct rte_eth_dev *dev,
6133                 const struct rte_flow_attr *attr __rte_unused,
6134                 const struct rte_flow_item items[] __rte_unused,
6135                 const struct rte_flow_action actions[] __rte_unused,
6136                 struct rte_flow_error *error)
6137 {
6138         uint32_t handle_idx = 0;
6139         struct mlx5_flow *dev_flow;
6140         struct mlx5_flow_handle *dev_handle;
6141         struct mlx5_priv *priv = dev->data->dev_private;
6142
6143         /* In case of corrupting the memory. */
6144         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
6145                 rte_flow_error_set(error, ENOSPC,
6146                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6147                                    "not free temporary device flow");
6148                 return NULL;
6149         }
6150         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
6151                                    &handle_idx);
6152         if (!dev_handle) {
6153                 rte_flow_error_set(error, ENOMEM,
6154                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6155                                    "not enough memory to create flow handle");
6156                 return NULL;
6157         }
6158         /* No multi-thread supporting. */
6159         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
6160         dev_flow->handle = dev_handle;
6161         dev_flow->handle_idx = handle_idx;
6162         /*
6163          * In some old rdma-core releases, before continuing, a check of the
6164          * length of matching parameter will be done at first. It needs to use
6165          * the length without misc4 param. If the flow has misc4 support, then
6166          * the length needs to be adjusted accordingly. Each param member is
6167          * aligned with a 64B boundary naturally.
6168          */
6169         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) -
6170                                   MLX5_ST_SZ_BYTES(fte_match_set_misc4);
6171         /*
6172          * The matching value needs to be cleared to 0 before using. In the
6173          * past, it will be automatically cleared when using rte_*alloc
6174          * API. The time consumption will be almost the same as before.
6175          */
6176         memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
6177         dev_flow->ingress = attr->ingress;
6178         dev_flow->dv.transfer = attr->transfer;
6179         return dev_flow;
6180 }
6181
6182 #ifdef RTE_LIBRTE_MLX5_DEBUG
6183 /**
6184  * Sanity check for match mask and value. Similar to check_valid_spec() in
6185  * kernel driver. If unmasked bit is present in value, it returns failure.
6186  *
6187  * @param match_mask
6188  *   pointer to match mask buffer.
6189  * @param match_value
6190  *   pointer to match value buffer.
6191  *
6192  * @return
6193  *   0 if valid, -EINVAL otherwise.
6194  */
6195 static int
6196 flow_dv_check_valid_spec(void *match_mask, void *match_value)
6197 {
6198         uint8_t *m = match_mask;
6199         uint8_t *v = match_value;
6200         unsigned int i;
6201
6202         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
6203                 if (v[i] & ~m[i]) {
6204                         DRV_LOG(ERR,
6205                                 "match_value differs from match_criteria"
6206                                 " %p[%u] != %p[%u]",
6207                                 match_value, i, match_mask, i);
6208                         return -EINVAL;
6209                 }
6210         }
6211         return 0;
6212 }
6213 #endif
6214
6215 /**
6216  * Add match of ip_version.
6217  *
6218  * @param[in] group
6219  *   Flow group.
6220  * @param[in] headers_v
6221  *   Values header pointer.
6222  * @param[in] headers_m
6223  *   Masks header pointer.
6224  * @param[in] ip_version
6225  *   The IP version to set.
6226  */
6227 static inline void
6228 flow_dv_set_match_ip_version(uint32_t group,
6229                              void *headers_v,
6230                              void *headers_m,
6231                              uint8_t ip_version)
6232 {
6233         if (group == 0)
6234                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
6235         else
6236                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
6237                          ip_version);
6238         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
6239         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
6240         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
6241 }
6242
6243 /**
6244  * Add Ethernet item to matcher and to the value.
6245  *
6246  * @param[in, out] matcher
6247  *   Flow matcher.
6248  * @param[in, out] key
6249  *   Flow matcher value.
6250  * @param[in] item
6251  *   Flow pattern to translate.
6252  * @param[in] inner
6253  *   Item is inner pattern.
6254  */
6255 static void
6256 flow_dv_translate_item_eth(void *matcher, void *key,
6257                            const struct rte_flow_item *item, int inner,
6258                            uint32_t group)
6259 {
6260         const struct rte_flow_item_eth *eth_m = item->mask;
6261         const struct rte_flow_item_eth *eth_v = item->spec;
6262         const struct rte_flow_item_eth nic_mask = {
6263                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6264                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6265                 .type = RTE_BE16(0xffff),
6266         };
6267         void *headers_m;
6268         void *headers_v;
6269         char *l24_v;
6270         unsigned int i;
6271
6272         if (!eth_v)
6273                 return;
6274         if (!eth_m)
6275                 eth_m = &nic_mask;
6276         if (inner) {
6277                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6278                                          inner_headers);
6279                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6280         } else {
6281                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6282                                          outer_headers);
6283                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6284         }
6285         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
6286                &eth_m->dst, sizeof(eth_m->dst));
6287         /* The value must be in the range of the mask. */
6288         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
6289         for (i = 0; i < sizeof(eth_m->dst); ++i)
6290                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
6291         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
6292                &eth_m->src, sizeof(eth_m->src));
6293         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
6294         /* The value must be in the range of the mask. */
6295         for (i = 0; i < sizeof(eth_m->dst); ++i)
6296                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
6297         if (eth_v->type) {
6298                 /* When ethertype is present set mask for tagged VLAN. */
6299                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6300                 /* Set value for tagged VLAN if ethertype is 802.1Q. */
6301                 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
6302                     eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
6303                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag,
6304                                  1);
6305                         /* Return here to avoid setting match on ethertype. */
6306                         return;
6307                 }
6308         }
6309         /*
6310          * HW supports match on one Ethertype, the Ethertype following the last
6311          * VLAN tag of the packet (see PRM).
6312          * Set match on ethertype only if ETH header is not followed by VLAN.
6313          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6314          * ethertype, and use ip_version field instead.
6315          * eCPRI over Ether layer will use type value 0xAEFE.
6316          */
6317         if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6318             eth_m->type == 0xFFFF) {
6319                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6320         } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6321                    eth_m->type == 0xFFFF) {
6322                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6323         } else {
6324                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6325                          rte_be_to_cpu_16(eth_m->type));
6326                 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6327                                      ethertype);
6328                 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
6329         }
6330 }
6331
6332 /**
6333  * Add VLAN item to matcher and to the value.
6334  *
6335  * @param[in, out] dev_flow
6336  *   Flow descriptor.
6337  * @param[in, out] matcher
6338  *   Flow matcher.
6339  * @param[in, out] key
6340  *   Flow matcher value.
6341  * @param[in] item
6342  *   Flow pattern to translate.
6343  * @param[in] inner
6344  *   Item is inner pattern.
6345  */
6346 static void
6347 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
6348                             void *matcher, void *key,
6349                             const struct rte_flow_item *item,
6350                             int inner, uint32_t group)
6351 {
6352         const struct rte_flow_item_vlan *vlan_m = item->mask;
6353         const struct rte_flow_item_vlan *vlan_v = item->spec;
6354         void *headers_m;
6355         void *headers_v;
6356         uint16_t tci_m;
6357         uint16_t tci_v;
6358
6359         if (inner) {
6360                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6361                                          inner_headers);
6362                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6363         } else {
6364                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6365                                          outer_headers);
6366                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6367                 /*
6368                  * This is workaround, masks are not supported,
6369                  * and pre-validated.
6370                  */
6371                 if (vlan_v)
6372                         dev_flow->handle->vf_vlan.tag =
6373                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
6374         }
6375         /*
6376          * When VLAN item exists in flow, mark packet as tagged,
6377          * even if TCI is not specified.
6378          */
6379         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6380         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
6381         if (!vlan_v)
6382                 return;
6383         if (!vlan_m)
6384                 vlan_m = &rte_flow_item_vlan_mask;
6385         tci_m = rte_be_to_cpu_16(vlan_m->tci);
6386         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
6387         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
6388         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
6389         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
6390         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
6391         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
6392         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
6393         /*
6394          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6395          * ethertype, and use ip_version field instead.
6396          */
6397         if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6398             vlan_m->inner_type == 0xFFFF) {
6399                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6400         } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6401                    vlan_m->inner_type == 0xFFFF) {
6402                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6403         } else {
6404                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6405                          rte_be_to_cpu_16(vlan_m->inner_type));
6406                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
6407                          rte_be_to_cpu_16(vlan_m->inner_type &
6408                                           vlan_v->inner_type));
6409         }
6410 }
6411
6412 /**
6413  * Add IPV4 item to matcher and to the value.
6414  *
6415  * @param[in, out] matcher
6416  *   Flow matcher.
6417  * @param[in, out] key
6418  *   Flow matcher value.
6419  * @param[in] item
6420  *   Flow pattern to translate.
6421  * @param[in] item_flags
6422  *   Bit-fields that holds the items detected until now.
6423  * @param[in] inner
6424  *   Item is inner pattern.
6425  * @param[in] group
6426  *   The group to insert the rule.
6427  */
6428 static void
6429 flow_dv_translate_item_ipv4(void *matcher, void *key,
6430                             const struct rte_flow_item *item,
6431                             const uint64_t item_flags,
6432                             int inner, uint32_t group)
6433 {
6434         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
6435         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
6436         const struct rte_flow_item_ipv4 nic_mask = {
6437                 .hdr = {
6438                         .src_addr = RTE_BE32(0xffffffff),
6439                         .dst_addr = RTE_BE32(0xffffffff),
6440                         .type_of_service = 0xff,
6441                         .next_proto_id = 0xff,
6442                         .time_to_live = 0xff,
6443                 },
6444         };
6445         void *headers_m;
6446         void *headers_v;
6447         char *l24_m;
6448         char *l24_v;
6449         uint8_t tos;
6450
6451         if (inner) {
6452                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6453                                          inner_headers);
6454                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6455         } else {
6456                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6457                                          outer_headers);
6458                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6459         }
6460         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6461         /*
6462          * On outer header (which must contains L2), or inner header with L2,
6463          * set cvlan_tag mask bit to mark this packet as untagged.
6464          * This should be done even if item->spec is empty.
6465          */
6466         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6467                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6468         if (!ipv4_v)
6469                 return;
6470         if (!ipv4_m)
6471                 ipv4_m = &nic_mask;
6472         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6473                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6474         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6475                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6476         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
6477         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
6478         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6479                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6480         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6481                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6482         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
6483         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
6484         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
6485         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
6486                  ipv4_m->hdr.type_of_service);
6487         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
6488         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
6489                  ipv4_m->hdr.type_of_service >> 2);
6490         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
6491         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6492                  ipv4_m->hdr.next_proto_id);
6493         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6494                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
6495         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6496                  ipv4_m->hdr.time_to_live);
6497         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6498                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
6499         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
6500                  !!(ipv4_m->hdr.fragment_offset));
6501         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
6502                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
6503 }
6504
6505 /**
6506  * Add IPV6 item to matcher and to the value.
6507  *
6508  * @param[in, out] matcher
6509  *   Flow matcher.
6510  * @param[in, out] key
6511  *   Flow matcher value.
6512  * @param[in] item
6513  *   Flow pattern to translate.
6514  * @param[in] item_flags
6515  *   Bit-fields that holds the items detected until now.
6516  * @param[in] inner
6517  *   Item is inner pattern.
6518  * @param[in] group
6519  *   The group to insert the rule.
6520  */
6521 static void
6522 flow_dv_translate_item_ipv6(void *matcher, void *key,
6523                             const struct rte_flow_item *item,
6524                             const uint64_t item_flags,
6525                             int inner, uint32_t group)
6526 {
6527         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
6528         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
6529         const struct rte_flow_item_ipv6 nic_mask = {
6530                 .hdr = {
6531                         .src_addr =
6532                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6533                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6534                         .dst_addr =
6535                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6536                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6537                         .vtc_flow = RTE_BE32(0xffffffff),
6538                         .proto = 0xff,
6539                         .hop_limits = 0xff,
6540                 },
6541         };
6542         void *headers_m;
6543         void *headers_v;
6544         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6545         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6546         char *l24_m;
6547         char *l24_v;
6548         uint32_t vtc_m;
6549         uint32_t vtc_v;
6550         int i;
6551         int size;
6552
6553         if (inner) {
6554                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6555                                          inner_headers);
6556                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6557         } else {
6558                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6559                                          outer_headers);
6560                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6561         }
6562         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6563         /*
6564          * On outer header (which must contains L2), or inner header with L2,
6565          * set cvlan_tag mask bit to mark this packet as untagged.
6566          * This should be done even if item->spec is empty.
6567          */
6568         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6569                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6570         if (!ipv6_v)
6571                 return;
6572         if (!ipv6_m)
6573                 ipv6_m = &nic_mask;
6574         size = sizeof(ipv6_m->hdr.dst_addr);
6575         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6576                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6577         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6578                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6579         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
6580         for (i = 0; i < size; ++i)
6581                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
6582         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6583                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6584         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6585                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6586         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
6587         for (i = 0; i < size; ++i)
6588                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
6589         /* TOS. */
6590         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
6591         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
6592         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
6593         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
6594         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
6595         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
6596         /* Label. */
6597         if (inner) {
6598                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
6599                          vtc_m);
6600                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
6601                          vtc_v);
6602         } else {
6603                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
6604                          vtc_m);
6605                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
6606                          vtc_v);
6607         }
6608         /* Protocol. */
6609         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6610                  ipv6_m->hdr.proto);
6611         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6612                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
6613         /* Hop limit. */
6614         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6615                  ipv6_m->hdr.hop_limits);
6616         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6617                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
6618         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
6619                  !!(ipv6_m->has_frag_ext));
6620         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
6621                  !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext));
6622 }
6623
6624 /**
6625  * Add IPV6 fragment extension item to matcher and to the value.
6626  *
6627  * @param[in, out] matcher
6628  *   Flow matcher.
6629  * @param[in, out] key
6630  *   Flow matcher value.
6631  * @param[in] item
6632  *   Flow pattern to translate.
6633  * @param[in] inner
6634  *   Item is inner pattern.
6635  */
6636 static void
6637 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key,
6638                                      const struct rte_flow_item *item,
6639                                      int inner)
6640 {
6641         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask;
6642         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec;
6643         const struct rte_flow_item_ipv6_frag_ext nic_mask = {
6644                 .hdr = {
6645                         .next_header = 0xff,
6646                         .frag_data = RTE_BE16(0xffff),
6647                 },
6648         };
6649         void *headers_m;
6650         void *headers_v;
6651
6652         if (inner) {
6653                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6654                                          inner_headers);
6655                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6656         } else {
6657                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6658                                          outer_headers);
6659                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6660         }
6661         /* IPv6 fragment extension item exists, so packet is IP fragment. */
6662         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
6663         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1);
6664         if (!ipv6_frag_ext_v)
6665                 return;
6666         if (!ipv6_frag_ext_m)
6667                 ipv6_frag_ext_m = &nic_mask;
6668         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6669                  ipv6_frag_ext_m->hdr.next_header);
6670         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6671                  ipv6_frag_ext_v->hdr.next_header &
6672                  ipv6_frag_ext_m->hdr.next_header);
6673 }
6674
6675 /**
6676  * Add TCP item to matcher and to the value.
6677  *
6678  * @param[in, out] matcher
6679  *   Flow matcher.
6680  * @param[in, out] key
6681  *   Flow matcher value.
6682  * @param[in] item
6683  *   Flow pattern to translate.
6684  * @param[in] inner
6685  *   Item is inner pattern.
6686  */
6687 static void
6688 flow_dv_translate_item_tcp(void *matcher, void *key,
6689                            const struct rte_flow_item *item,
6690                            int inner)
6691 {
6692         const struct rte_flow_item_tcp *tcp_m = item->mask;
6693         const struct rte_flow_item_tcp *tcp_v = item->spec;
6694         void *headers_m;
6695         void *headers_v;
6696
6697         if (inner) {
6698                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6699                                          inner_headers);
6700                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6701         } else {
6702                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6703                                          outer_headers);
6704                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6705         }
6706         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6707         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
6708         if (!tcp_v)
6709                 return;
6710         if (!tcp_m)
6711                 tcp_m = &rte_flow_item_tcp_mask;
6712         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
6713                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
6714         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
6715                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
6716         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
6717                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
6718         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
6719                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
6720         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
6721                  tcp_m->hdr.tcp_flags);
6722         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
6723                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
6724 }
6725
6726 /**
6727  * Add UDP item to matcher and to the value.
6728  *
6729  * @param[in, out] matcher
6730  *   Flow matcher.
6731  * @param[in, out] key
6732  *   Flow matcher value.
6733  * @param[in] item
6734  *   Flow pattern to translate.
6735  * @param[in] inner
6736  *   Item is inner pattern.
6737  */
6738 static void
6739 flow_dv_translate_item_udp(void *matcher, void *key,
6740                            const struct rte_flow_item *item,
6741                            int inner)
6742 {
6743         const struct rte_flow_item_udp *udp_m = item->mask;
6744         const struct rte_flow_item_udp *udp_v = item->spec;
6745         void *headers_m;
6746         void *headers_v;
6747
6748         if (inner) {
6749                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6750                                          inner_headers);
6751                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6752         } else {
6753                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6754                                          outer_headers);
6755                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6756         }
6757         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6758         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
6759         if (!udp_v)
6760                 return;
6761         if (!udp_m)
6762                 udp_m = &rte_flow_item_udp_mask;
6763         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
6764                  rte_be_to_cpu_16(udp_m->hdr.src_port));
6765         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
6766                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
6767         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
6768                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
6769         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6770                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
6771 }
6772
6773 /**
6774  * Add GRE optional Key item to matcher and to the value.
6775  *
6776  * @param[in, out] matcher
6777  *   Flow matcher.
6778  * @param[in, out] key
6779  *   Flow matcher value.
6780  * @param[in] item
6781  *   Flow pattern to translate.
6782  * @param[in] inner
6783  *   Item is inner pattern.
6784  */
6785 static void
6786 flow_dv_translate_item_gre_key(void *matcher, void *key,
6787                                    const struct rte_flow_item *item)
6788 {
6789         const rte_be32_t *key_m = item->mask;
6790         const rte_be32_t *key_v = item->spec;
6791         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6792         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6793         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
6794
6795         /* GRE K bit must be on and should already be validated */
6796         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
6797         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
6798         if (!key_v)
6799                 return;
6800         if (!key_m)
6801                 key_m = &gre_key_default_mask;
6802         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
6803                  rte_be_to_cpu_32(*key_m) >> 8);
6804         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
6805                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
6806         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
6807                  rte_be_to_cpu_32(*key_m) & 0xFF);
6808         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
6809                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
6810 }
6811
6812 /**
6813  * Add GRE item to matcher and to the value.
6814  *
6815  * @param[in, out] matcher
6816  *   Flow matcher.
6817  * @param[in, out] key
6818  *   Flow matcher value.
6819  * @param[in] item
6820  *   Flow pattern to translate.
6821  * @param[in] inner
6822  *   Item is inner pattern.
6823  */
6824 static void
6825 flow_dv_translate_item_gre(void *matcher, void *key,
6826                            const struct rte_flow_item *item,
6827                            int inner)
6828 {
6829         const struct rte_flow_item_gre *gre_m = item->mask;
6830         const struct rte_flow_item_gre *gre_v = item->spec;
6831         void *headers_m;
6832         void *headers_v;
6833         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6834         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6835         struct {
6836                 union {
6837                         __extension__
6838                         struct {
6839                                 uint16_t version:3;
6840                                 uint16_t rsvd0:9;
6841                                 uint16_t s_present:1;
6842                                 uint16_t k_present:1;
6843                                 uint16_t rsvd_bit1:1;
6844                                 uint16_t c_present:1;
6845                         };
6846                         uint16_t value;
6847                 };
6848         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
6849
6850         if (inner) {
6851                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6852                                          inner_headers);
6853                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6854         } else {
6855                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6856                                          outer_headers);
6857                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6858         }
6859         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6860         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
6861         if (!gre_v)
6862                 return;
6863         if (!gre_m)
6864                 gre_m = &rte_flow_item_gre_mask;
6865         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
6866                  rte_be_to_cpu_16(gre_m->protocol));
6867         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6868                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
6869         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
6870         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
6871         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
6872                  gre_crks_rsvd0_ver_m.c_present);
6873         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
6874                  gre_crks_rsvd0_ver_v.c_present &
6875                  gre_crks_rsvd0_ver_m.c_present);
6876         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
6877                  gre_crks_rsvd0_ver_m.k_present);
6878         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
6879                  gre_crks_rsvd0_ver_v.k_present &
6880                  gre_crks_rsvd0_ver_m.k_present);
6881         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
6882                  gre_crks_rsvd0_ver_m.s_present);
6883         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
6884                  gre_crks_rsvd0_ver_v.s_present &
6885                  gre_crks_rsvd0_ver_m.s_present);
6886 }
6887
6888 /**
6889  * Add NVGRE item to matcher and to the value.
6890  *
6891  * @param[in, out] matcher
6892  *   Flow matcher.
6893  * @param[in, out] key
6894  *   Flow matcher value.
6895  * @param[in] item
6896  *   Flow pattern to translate.
6897  * @param[in] inner
6898  *   Item is inner pattern.
6899  */
6900 static void
6901 flow_dv_translate_item_nvgre(void *matcher, void *key,
6902                              const struct rte_flow_item *item,
6903                              int inner)
6904 {
6905         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
6906         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
6907         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6908         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6909         const char *tni_flow_id_m;
6910         const char *tni_flow_id_v;
6911         char *gre_key_m;
6912         char *gre_key_v;
6913         int size;
6914         int i;
6915
6916         /* For NVGRE, GRE header fields must be set with defined values. */
6917         const struct rte_flow_item_gre gre_spec = {
6918                 .c_rsvd0_ver = RTE_BE16(0x2000),
6919                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
6920         };
6921         const struct rte_flow_item_gre gre_mask = {
6922                 .c_rsvd0_ver = RTE_BE16(0xB000),
6923                 .protocol = RTE_BE16(UINT16_MAX),
6924         };
6925         const struct rte_flow_item gre_item = {
6926                 .spec = &gre_spec,
6927                 .mask = &gre_mask,
6928                 .last = NULL,
6929         };
6930         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
6931         if (!nvgre_v)
6932                 return;
6933         if (!nvgre_m)
6934                 nvgre_m = &rte_flow_item_nvgre_mask;
6935         tni_flow_id_m = (const char *)nvgre_m->tni;
6936         tni_flow_id_v = (const char *)nvgre_v->tni;
6937         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
6938         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
6939         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
6940         memcpy(gre_key_m, tni_flow_id_m, size);
6941         for (i = 0; i < size; ++i)
6942                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
6943 }
6944
6945 /**
6946  * Add VXLAN item to matcher and to the value.
6947  *
6948  * @param[in, out] matcher
6949  *   Flow matcher.
6950  * @param[in, out] key
6951  *   Flow matcher value.
6952  * @param[in] item
6953  *   Flow pattern to translate.
6954  * @param[in] inner
6955  *   Item is inner pattern.
6956  */
6957 static void
6958 flow_dv_translate_item_vxlan(void *matcher, void *key,
6959                              const struct rte_flow_item *item,
6960                              int inner)
6961 {
6962         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
6963         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
6964         void *headers_m;
6965         void *headers_v;
6966         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6967         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6968         char *vni_m;
6969         char *vni_v;
6970         uint16_t dport;
6971         int size;
6972         int i;
6973
6974         if (inner) {
6975                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6976                                          inner_headers);
6977                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6978         } else {
6979                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6980                                          outer_headers);
6981                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6982         }
6983         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6984                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6985         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6986                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6987                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6988         }
6989         if (!vxlan_v)
6990                 return;
6991         if (!vxlan_m)
6992                 vxlan_m = &rte_flow_item_vxlan_mask;
6993         size = sizeof(vxlan_m->vni);
6994         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
6995         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
6996         memcpy(vni_m, vxlan_m->vni, size);
6997         for (i = 0; i < size; ++i)
6998                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6999 }
7000
7001 /**
7002  * Add VXLAN-GPE item to matcher and to the value.
7003  *
7004  * @param[in, out] matcher
7005  *   Flow matcher.
7006  * @param[in, out] key
7007  *   Flow matcher value.
7008  * @param[in] item
7009  *   Flow pattern to translate.
7010  * @param[in] inner
7011  *   Item is inner pattern.
7012  */
7013
7014 static void
7015 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
7016                                  const struct rte_flow_item *item, int inner)
7017 {
7018         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
7019         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
7020         void *headers_m;
7021         void *headers_v;
7022         void *misc_m =
7023                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
7024         void *misc_v =
7025                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7026         char *vni_m;
7027         char *vni_v;
7028         uint16_t dport;
7029         int size;
7030         int i;
7031         uint8_t flags_m = 0xff;
7032         uint8_t flags_v = 0xc;
7033
7034         if (inner) {
7035                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7036                                          inner_headers);
7037                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7038         } else {
7039                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7040                                          outer_headers);
7041                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7042         }
7043         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
7044                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
7045         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7046                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7047                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7048         }
7049         if (!vxlan_v)
7050                 return;
7051         if (!vxlan_m)
7052                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
7053         size = sizeof(vxlan_m->vni);
7054         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
7055         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
7056         memcpy(vni_m, vxlan_m->vni, size);
7057         for (i = 0; i < size; ++i)
7058                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
7059         if (vxlan_m->flags) {
7060                 flags_m = vxlan_m->flags;
7061                 flags_v = vxlan_v->flags;
7062         }
7063         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
7064         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
7065         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
7066                  vxlan_m->protocol);
7067         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
7068                  vxlan_v->protocol);
7069 }
7070
7071 /**
7072  * Add Geneve item to matcher and to the value.
7073  *
7074  * @param[in, out] matcher
7075  *   Flow matcher.
7076  * @param[in, out] key
7077  *   Flow matcher value.
7078  * @param[in] item
7079  *   Flow pattern to translate.
7080  * @param[in] inner
7081  *   Item is inner pattern.
7082  */
7083
7084 static void
7085 flow_dv_translate_item_geneve(void *matcher, void *key,
7086                               const struct rte_flow_item *item, int inner)
7087 {
7088         const struct rte_flow_item_geneve *geneve_m = item->mask;
7089         const struct rte_flow_item_geneve *geneve_v = item->spec;
7090         void *headers_m;
7091         void *headers_v;
7092         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7093         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7094         uint16_t dport;
7095         uint16_t gbhdr_m;
7096         uint16_t gbhdr_v;
7097         char *vni_m;
7098         char *vni_v;
7099         size_t size, i;
7100
7101         if (inner) {
7102                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7103                                          inner_headers);
7104                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7105         } else {
7106                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7107                                          outer_headers);
7108                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7109         }
7110         dport = MLX5_UDP_PORT_GENEVE;
7111         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7112                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7113                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7114         }
7115         if (!geneve_v)
7116                 return;
7117         if (!geneve_m)
7118                 geneve_m = &rte_flow_item_geneve_mask;
7119         size = sizeof(geneve_m->vni);
7120         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
7121         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
7122         memcpy(vni_m, geneve_m->vni, size);
7123         for (i = 0; i < size; ++i)
7124                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
7125         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
7126                  rte_be_to_cpu_16(geneve_m->protocol));
7127         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
7128                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
7129         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
7130         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
7131         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
7132                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7133         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
7134                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7135         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
7136                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7137         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
7138                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
7139                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7140 }
7141
7142 /**
7143  * Add MPLS item to matcher and to the value.
7144  *
7145  * @param[in, out] matcher
7146  *   Flow matcher.
7147  * @param[in, out] key
7148  *   Flow matcher value.
7149  * @param[in] item
7150  *   Flow pattern to translate.
7151  * @param[in] prev_layer
7152  *   The protocol layer indicated in previous item.
7153  * @param[in] inner
7154  *   Item is inner pattern.
7155  */
7156 static void
7157 flow_dv_translate_item_mpls(void *matcher, void *key,
7158                             const struct rte_flow_item *item,
7159                             uint64_t prev_layer,
7160                             int inner)
7161 {
7162         const uint32_t *in_mpls_m = item->mask;
7163         const uint32_t *in_mpls_v = item->spec;
7164         uint32_t *out_mpls_m = 0;
7165         uint32_t *out_mpls_v = 0;
7166         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7167         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7168         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
7169                                      misc_parameters_2);
7170         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7171         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
7172         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7173
7174         switch (prev_layer) {
7175         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7176                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
7177                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
7178                          MLX5_UDP_PORT_MPLS);
7179                 break;
7180         case MLX5_FLOW_LAYER_GRE:
7181                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
7182                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
7183                          RTE_ETHER_TYPE_MPLS);
7184                 break;
7185         default:
7186                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
7187                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
7188                          IPPROTO_MPLS);
7189                 break;
7190         }
7191         if (!in_mpls_v)
7192                 return;
7193         if (!in_mpls_m)
7194                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
7195         switch (prev_layer) {
7196         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7197                 out_mpls_m =
7198                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7199                                                  outer_first_mpls_over_udp);
7200                 out_mpls_v =
7201                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7202                                                  outer_first_mpls_over_udp);
7203                 break;
7204         case MLX5_FLOW_LAYER_GRE:
7205                 out_mpls_m =
7206                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7207                                                  outer_first_mpls_over_gre);
7208                 out_mpls_v =
7209                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7210                                                  outer_first_mpls_over_gre);
7211                 break;
7212         default:
7213                 /* Inner MPLS not over GRE is not supported. */
7214                 if (!inner) {
7215                         out_mpls_m =
7216                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7217                                                          misc2_m,
7218                                                          outer_first_mpls);
7219                         out_mpls_v =
7220                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7221                                                          misc2_v,
7222                                                          outer_first_mpls);
7223                 }
7224                 break;
7225         }
7226         if (out_mpls_m && out_mpls_v) {
7227                 *out_mpls_m = *in_mpls_m;
7228                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
7229         }
7230 }
7231
7232 /**
7233  * Add metadata register item to matcher
7234  *
7235  * @param[in, out] matcher
7236  *   Flow matcher.
7237  * @param[in, out] key
7238  *   Flow matcher value.
7239  * @param[in] reg_type
7240  *   Type of device metadata register
7241  * @param[in] value
7242  *   Register value
7243  * @param[in] mask
7244  *   Register mask
7245  */
7246 static void
7247 flow_dv_match_meta_reg(void *matcher, void *key,
7248                        enum modify_reg reg_type,
7249                        uint32_t data, uint32_t mask)
7250 {
7251         void *misc2_m =
7252                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
7253         void *misc2_v =
7254                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7255         uint32_t temp;
7256
7257         data &= mask;
7258         switch (reg_type) {
7259         case REG_A:
7260                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
7261                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
7262                 break;
7263         case REG_B:
7264                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
7265                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
7266                 break;
7267         case REG_C_0:
7268                 /*
7269                  * The metadata register C0 field might be divided into
7270                  * source vport index and META item value, we should set
7271                  * this field according to specified mask, not as whole one.
7272                  */
7273                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
7274                 temp |= mask;
7275                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
7276                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
7277                 temp &= ~mask;
7278                 temp |= data;
7279                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
7280                 break;
7281         case REG_C_1:
7282                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
7283                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
7284                 break;
7285         case REG_C_2:
7286                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
7287                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
7288                 break;
7289         case REG_C_3:
7290                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
7291                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
7292                 break;
7293         case REG_C_4:
7294                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
7295                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
7296                 break;
7297         case REG_C_5:
7298                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
7299                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
7300                 break;
7301         case REG_C_6:
7302                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
7303                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
7304                 break;
7305         case REG_C_7:
7306                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
7307                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
7308                 break;
7309         default:
7310                 MLX5_ASSERT(false);
7311                 break;
7312         }
7313 }
7314
7315 /**
7316  * Add MARK item to matcher
7317  *
7318  * @param[in] dev
7319  *   The device to configure through.
7320  * @param[in, out] matcher
7321  *   Flow matcher.
7322  * @param[in, out] key
7323  *   Flow matcher value.
7324  * @param[in] item
7325  *   Flow pattern to translate.
7326  */
7327 static void
7328 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
7329                             void *matcher, void *key,
7330                             const struct rte_flow_item *item)
7331 {
7332         struct mlx5_priv *priv = dev->data->dev_private;
7333         const struct rte_flow_item_mark *mark;
7334         uint32_t value;
7335         uint32_t mask;
7336
7337         mark = item->mask ? (const void *)item->mask :
7338                             &rte_flow_item_mark_mask;
7339         mask = mark->id & priv->sh->dv_mark_mask;
7340         mark = (const void *)item->spec;
7341         MLX5_ASSERT(mark);
7342         value = mark->id & priv->sh->dv_mark_mask & mask;
7343         if (mask) {
7344                 enum modify_reg reg;
7345
7346                 /* Get the metadata register index for the mark. */
7347                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
7348                 MLX5_ASSERT(reg > 0);
7349                 if (reg == REG_C_0) {
7350                         struct mlx5_priv *priv = dev->data->dev_private;
7351                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7352                         uint32_t shl_c0 = rte_bsf32(msk_c0);
7353
7354                         mask &= msk_c0;
7355                         mask <<= shl_c0;
7356                         value <<= shl_c0;
7357                 }
7358                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7359         }
7360 }
7361
7362 /**
7363  * Add META item to matcher
7364  *
7365  * @param[in] dev
7366  *   The devich to configure through.
7367  * @param[in, out] matcher
7368  *   Flow matcher.
7369  * @param[in, out] key
7370  *   Flow matcher value.
7371  * @param[in] attr
7372  *   Attributes of flow that includes this item.
7373  * @param[in] item
7374  *   Flow pattern to translate.
7375  */
7376 static void
7377 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
7378                             void *matcher, void *key,
7379                             const struct rte_flow_attr *attr,
7380                             const struct rte_flow_item *item)
7381 {
7382         const struct rte_flow_item_meta *meta_m;
7383         const struct rte_flow_item_meta *meta_v;
7384
7385         meta_m = (const void *)item->mask;
7386         if (!meta_m)
7387                 meta_m = &rte_flow_item_meta_mask;
7388         meta_v = (const void *)item->spec;
7389         if (meta_v) {
7390                 int reg;
7391                 uint32_t value = meta_v->data;
7392                 uint32_t mask = meta_m->data;
7393
7394                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
7395                 if (reg < 0)
7396                         return;
7397                 /*
7398                  * In datapath code there is no endianness
7399                  * coversions for perfromance reasons, all
7400                  * pattern conversions are done in rte_flow.
7401                  */
7402                 value = rte_cpu_to_be_32(value);
7403                 mask = rte_cpu_to_be_32(mask);
7404                 if (reg == REG_C_0) {
7405                         struct mlx5_priv *priv = dev->data->dev_private;
7406                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7407                         uint32_t shl_c0 = rte_bsf32(msk_c0);
7408 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
7409                         uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
7410
7411                         value >>= shr_c0;
7412                         mask >>= shr_c0;
7413 #endif
7414                         value <<= shl_c0;
7415                         mask <<= shl_c0;
7416                         MLX5_ASSERT(msk_c0);
7417                         MLX5_ASSERT(!(~msk_c0 & mask));
7418                 }
7419                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7420         }
7421 }
7422
7423 /**
7424  * Add vport metadata Reg C0 item to matcher
7425  *
7426  * @param[in, out] matcher
7427  *   Flow matcher.
7428  * @param[in, out] key
7429  *   Flow matcher value.
7430  * @param[in] reg
7431  *   Flow pattern to translate.
7432  */
7433 static void
7434 flow_dv_translate_item_meta_vport(void *matcher, void *key,
7435                                   uint32_t value, uint32_t mask)
7436 {
7437         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
7438 }
7439
7440 /**
7441  * Add tag item to matcher
7442  *
7443  * @param[in] dev
7444  *   The devich to configure through.
7445  * @param[in, out] matcher
7446  *   Flow matcher.
7447  * @param[in, out] key
7448  *   Flow matcher value.
7449  * @param[in] item
7450  *   Flow pattern to translate.
7451  */
7452 static void
7453 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
7454                                 void *matcher, void *key,
7455                                 const struct rte_flow_item *item)
7456 {
7457         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
7458         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
7459         uint32_t mask, value;
7460
7461         MLX5_ASSERT(tag_v);
7462         value = tag_v->data;
7463         mask = tag_m ? tag_m->data : UINT32_MAX;
7464         if (tag_v->id == REG_C_0) {
7465                 struct mlx5_priv *priv = dev->data->dev_private;
7466                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7467                 uint32_t shl_c0 = rte_bsf32(msk_c0);
7468
7469                 mask &= msk_c0;
7470                 mask <<= shl_c0;
7471                 value <<= shl_c0;
7472         }
7473         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
7474 }
7475
7476 /**
7477  * Add TAG item to matcher
7478  *
7479  * @param[in] dev
7480  *   The devich to configure through.
7481  * @param[in, out] matcher
7482  *   Flow matcher.
7483  * @param[in, out] key
7484  *   Flow matcher value.
7485  * @param[in] item
7486  *   Flow pattern to translate.
7487  */
7488 static void
7489 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
7490                            void *matcher, void *key,
7491                            const struct rte_flow_item *item)
7492 {
7493         const struct rte_flow_item_tag *tag_v = item->spec;
7494         const struct rte_flow_item_tag *tag_m = item->mask;
7495         enum modify_reg reg;
7496
7497         MLX5_ASSERT(tag_v);
7498         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
7499         /* Get the metadata register index for the tag. */
7500         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
7501         MLX5_ASSERT(reg > 0);
7502         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
7503 }
7504
7505 /**
7506  * Add source vport match to the specified matcher.
7507  *
7508  * @param[in, out] matcher
7509  *   Flow matcher.
7510  * @param[in, out] key
7511  *   Flow matcher value.
7512  * @param[in] port
7513  *   Source vport value to match
7514  * @param[in] mask
7515  *   Mask
7516  */
7517 static void
7518 flow_dv_translate_item_source_vport(void *matcher, void *key,
7519                                     int16_t port, uint16_t mask)
7520 {
7521         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7522         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7523
7524         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
7525         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
7526 }
7527
7528 /**
7529  * Translate port-id item to eswitch match on  port-id.
7530  *
7531  * @param[in] dev
7532  *   The devich to configure through.
7533  * @param[in, out] matcher
7534  *   Flow matcher.
7535  * @param[in, out] key
7536  *   Flow matcher value.
7537  * @param[in] item
7538  *   Flow pattern to translate.
7539  *
7540  * @return
7541  *   0 on success, a negative errno value otherwise.
7542  */
7543 static int
7544 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
7545                                void *key, const struct rte_flow_item *item)
7546 {
7547         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
7548         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
7549         struct mlx5_priv *priv;
7550         uint16_t mask, id;
7551
7552         mask = pid_m ? pid_m->id : 0xffff;
7553         id = pid_v ? pid_v->id : dev->data->port_id;
7554         priv = mlx5_port_to_eswitch_info(id, item == NULL);
7555         if (!priv)
7556                 return -rte_errno;
7557         /* Translate to vport field or to metadata, depending on mode. */
7558         if (priv->vport_meta_mask)
7559                 flow_dv_translate_item_meta_vport(matcher, key,
7560                                                   priv->vport_meta_tag,
7561                                                   priv->vport_meta_mask);
7562         else
7563                 flow_dv_translate_item_source_vport(matcher, key,
7564                                                     priv->vport_id, mask);
7565         return 0;
7566 }
7567
7568 /**
7569  * Add ICMP6 item to matcher and to the value.
7570  *
7571  * @param[in, out] matcher
7572  *   Flow matcher.
7573  * @param[in, out] key
7574  *   Flow matcher value.
7575  * @param[in] item
7576  *   Flow pattern to translate.
7577  * @param[in] inner
7578  *   Item is inner pattern.
7579  */
7580 static void
7581 flow_dv_translate_item_icmp6(void *matcher, void *key,
7582                               const struct rte_flow_item *item,
7583                               int inner)
7584 {
7585         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
7586         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
7587         void *headers_m;
7588         void *headers_v;
7589         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7590                                      misc_parameters_3);
7591         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7592         if (inner) {
7593                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7594                                          inner_headers);
7595                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7596         } else {
7597                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7598                                          outer_headers);
7599                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7600         }
7601         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7602         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
7603         if (!icmp6_v)
7604                 return;
7605         if (!icmp6_m)
7606                 icmp6_m = &rte_flow_item_icmp6_mask;
7607         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
7608         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
7609                  icmp6_v->type & icmp6_m->type);
7610         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
7611         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
7612                  icmp6_v->code & icmp6_m->code);
7613 }
7614
7615 /**
7616  * Add ICMP item to matcher and to the value.
7617  *
7618  * @param[in, out] matcher
7619  *   Flow matcher.
7620  * @param[in, out] key
7621  *   Flow matcher value.
7622  * @param[in] item
7623  *   Flow pattern to translate.
7624  * @param[in] inner
7625  *   Item is inner pattern.
7626  */
7627 static void
7628 flow_dv_translate_item_icmp(void *matcher, void *key,
7629                             const struct rte_flow_item *item,
7630                             int inner)
7631 {
7632         const struct rte_flow_item_icmp *icmp_m = item->mask;
7633         const struct rte_flow_item_icmp *icmp_v = item->spec;
7634         uint32_t icmp_header_data_m = 0;
7635         uint32_t icmp_header_data_v = 0;
7636         void *headers_m;
7637         void *headers_v;
7638         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7639                                      misc_parameters_3);
7640         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7641         if (inner) {
7642                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7643                                          inner_headers);
7644                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7645         } else {
7646                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7647                                          outer_headers);
7648                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7649         }
7650         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7651         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
7652         if (!icmp_v)
7653                 return;
7654         if (!icmp_m)
7655                 icmp_m = &rte_flow_item_icmp_mask;
7656         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
7657                  icmp_m->hdr.icmp_type);
7658         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
7659                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
7660         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
7661                  icmp_m->hdr.icmp_code);
7662         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
7663                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
7664         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
7665         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
7666         if (icmp_header_data_m) {
7667                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
7668                 icmp_header_data_v |=
7669                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
7670                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
7671                          icmp_header_data_m);
7672                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
7673                          icmp_header_data_v & icmp_header_data_m);
7674         }
7675 }
7676
7677 /**
7678  * Add GTP item to matcher and to the value.
7679  *
7680  * @param[in, out] matcher
7681  *   Flow matcher.
7682  * @param[in, out] key
7683  *   Flow matcher value.
7684  * @param[in] item
7685  *   Flow pattern to translate.
7686  * @param[in] inner
7687  *   Item is inner pattern.
7688  */
7689 static void
7690 flow_dv_translate_item_gtp(void *matcher, void *key,
7691                            const struct rte_flow_item *item, int inner)
7692 {
7693         const struct rte_flow_item_gtp *gtp_m = item->mask;
7694         const struct rte_flow_item_gtp *gtp_v = item->spec;
7695         void *headers_m;
7696         void *headers_v;
7697         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7698                                      misc_parameters_3);
7699         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7700         uint16_t dport = RTE_GTPU_UDP_PORT;
7701
7702         if (inner) {
7703                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7704                                          inner_headers);
7705                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7706         } else {
7707                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7708                                          outer_headers);
7709                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7710         }
7711         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7712                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7713                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7714         }
7715         if (!gtp_v)
7716                 return;
7717         if (!gtp_m)
7718                 gtp_m = &rte_flow_item_gtp_mask;
7719         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
7720                  gtp_m->v_pt_rsv_flags);
7721         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
7722                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
7723         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
7724         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
7725                  gtp_v->msg_type & gtp_m->msg_type);
7726         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
7727                  rte_be_to_cpu_32(gtp_m->teid));
7728         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
7729                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
7730 }
7731
7732 /**
7733  * Add eCPRI item to matcher and to the value.
7734  *
7735  * @param[in] dev
7736  *   The devich to configure through.
7737  * @param[in, out] matcher
7738  *   Flow matcher.
7739  * @param[in, out] key
7740  *   Flow matcher value.
7741  * @param[in] item
7742  *   Flow pattern to translate.
7743  * @param[in] samples
7744  *   Sample IDs to be used in the matching.
7745  */
7746 static void
7747 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
7748                              void *key, const struct rte_flow_item *item)
7749 {
7750         struct mlx5_priv *priv = dev->data->dev_private;
7751         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
7752         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
7753         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
7754                                      misc_parameters_4);
7755         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
7756         uint32_t *samples;
7757         void *dw_m;
7758         void *dw_v;
7759
7760         if (!ecpri_v)
7761                 return;
7762         if (!ecpri_m)
7763                 ecpri_m = &rte_flow_item_ecpri_mask;
7764         /*
7765          * Maximal four DW samples are supported in a single matching now.
7766          * Two are used now for a eCPRI matching:
7767          * 1. Type: one byte, mask should be 0x00ff0000 in network order
7768          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
7769          *    if any.
7770          */
7771         if (!ecpri_m->hdr.common.u32)
7772                 return;
7773         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
7774         /* Need to take the whole DW as the mask to fill the entry. */
7775         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
7776                             prog_sample_field_value_0);
7777         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
7778                             prog_sample_field_value_0);
7779         /* Already big endian (network order) in the header. */
7780         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
7781         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32;
7782         /* Sample#0, used for matching type, offset 0. */
7783         MLX5_SET(fte_match_set_misc4, misc4_m,
7784                  prog_sample_field_id_0, samples[0]);
7785         /* It makes no sense to set the sample ID in the mask field. */
7786         MLX5_SET(fte_match_set_misc4, misc4_v,
7787                  prog_sample_field_id_0, samples[0]);
7788         /*
7789          * Checking if message body part needs to be matched.
7790          * Some wildcard rules only matching type field should be supported.
7791          */
7792         if (ecpri_m->hdr.dummy[0]) {
7793                 switch (ecpri_v->hdr.common.type) {
7794                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
7795                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
7796                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
7797                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
7798                                             prog_sample_field_value_1);
7799                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
7800                                             prog_sample_field_value_1);
7801                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
7802                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0];
7803                         /* Sample#1, to match message body, offset 4. */
7804                         MLX5_SET(fte_match_set_misc4, misc4_m,
7805                                  prog_sample_field_id_1, samples[1]);
7806                         MLX5_SET(fte_match_set_misc4, misc4_v,
7807                                  prog_sample_field_id_1, samples[1]);
7808                         break;
7809                 default:
7810                         /* Others, do not match any sample ID. */
7811                         break;
7812                 }
7813         }
7814 }
7815
7816 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
7817
7818 #define HEADER_IS_ZERO(match_criteria, headers)                              \
7819         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
7820                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
7821
7822 /**
7823  * Calculate flow matcher enable bitmap.
7824  *
7825  * @param match_criteria
7826  *   Pointer to flow matcher criteria.
7827  *
7828  * @return
7829  *   Bitmap of enabled fields.
7830  */
7831 static uint8_t
7832 flow_dv_matcher_enable(uint32_t *match_criteria)
7833 {
7834         uint8_t match_criteria_enable;
7835
7836         match_criteria_enable =
7837                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
7838                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
7839         match_criteria_enable |=
7840                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
7841                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
7842         match_criteria_enable |=
7843                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
7844                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
7845         match_criteria_enable |=
7846                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
7847                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
7848         match_criteria_enable |=
7849                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
7850                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
7851         match_criteria_enable |=
7852                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
7853                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
7854         return match_criteria_enable;
7855 }
7856
7857
7858 /**
7859  * Get a flow table.
7860  *
7861  * @param[in, out] dev
7862  *   Pointer to rte_eth_dev structure.
7863  * @param[in] table_id
7864  *   Table id to use.
7865  * @param[in] egress
7866  *   Direction of the table.
7867  * @param[in] transfer
7868  *   E-Switch or NIC flow.
7869  * @param[out] error
7870  *   pointer to error structure.
7871  *
7872  * @return
7873  *   Returns tables resource based on the index, NULL in case of failed.
7874  */
7875 static struct mlx5_flow_tbl_resource *
7876 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
7877                          uint32_t table_id, uint8_t egress,
7878                          uint8_t transfer,
7879                          struct rte_flow_error *error)
7880 {
7881         struct mlx5_priv *priv = dev->data->dev_private;
7882         struct mlx5_dev_ctx_shared *sh = priv->sh;
7883         struct mlx5_flow_tbl_resource *tbl;
7884         union mlx5_flow_tbl_key table_key = {
7885                 {
7886                         .table_id = table_id,
7887                         .reserved = 0,
7888                         .domain = !!transfer,
7889                         .direction = !!egress,
7890                 }
7891         };
7892         struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
7893                                                          table_key.v64);
7894         struct mlx5_flow_tbl_data_entry *tbl_data;
7895         uint32_t idx = 0;
7896         int ret;
7897         void *domain;
7898
7899         if (pos) {
7900                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
7901                                         entry);
7902                 tbl = &tbl_data->tbl;
7903                 rte_atomic32_inc(&tbl->refcnt);
7904                 return tbl;
7905         }
7906         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
7907         if (!tbl_data) {
7908                 rte_flow_error_set(error, ENOMEM,
7909                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7910                                    NULL,
7911                                    "cannot allocate flow table data entry");
7912                 return NULL;
7913         }
7914         tbl_data->idx = idx;
7915         tbl = &tbl_data->tbl;
7916         pos = &tbl_data->entry;
7917         if (transfer)
7918                 domain = sh->fdb_domain;
7919         else if (egress)
7920                 domain = sh->tx_domain;
7921         else
7922                 domain = sh->rx_domain;
7923         ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj);
7924         if (ret) {
7925                 rte_flow_error_set(error, ENOMEM,
7926                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7927                                    NULL, "cannot create flow table object");
7928                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7929                 return NULL;
7930         }
7931         /*
7932          * No multi-threads now, but still better to initialize the reference
7933          * count before insert it into the hash list.
7934          */
7935         rte_atomic32_init(&tbl->refcnt);
7936         /* Jump action reference count is initialized here. */
7937         rte_atomic32_init(&tbl_data->jump.refcnt);
7938         pos->key = table_key.v64;
7939         ret = mlx5_hlist_insert(sh->flow_tbls, pos);
7940         if (ret < 0) {
7941                 rte_flow_error_set(error, -ret,
7942                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7943                                    "cannot insert flow table data entry");
7944                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
7945                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7946         }
7947         rte_atomic32_inc(&tbl->refcnt);
7948         return tbl;
7949 }
7950
7951 /**
7952  * Release a flow table.
7953  *
7954  * @param[in] dev
7955  *   Pointer to rte_eth_dev structure.
7956  * @param[in] tbl
7957  *   Table resource to be released.
7958  *
7959  * @return
7960  *   Returns 0 if table was released, else return 1;
7961  */
7962 static int
7963 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
7964                              struct mlx5_flow_tbl_resource *tbl)
7965 {
7966         struct mlx5_priv *priv = dev->data->dev_private;
7967         struct mlx5_dev_ctx_shared *sh = priv->sh;
7968         struct mlx5_flow_tbl_data_entry *tbl_data =
7969                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7970
7971         if (!tbl)
7972                 return 0;
7973         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
7974                 struct mlx5_hlist_entry *pos = &tbl_data->entry;
7975
7976                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
7977                 tbl->obj = NULL;
7978                 /* remove the entry from the hash list and free memory. */
7979                 mlx5_hlist_remove(sh->flow_tbls, pos);
7980                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
7981                                 tbl_data->idx);
7982                 return 0;
7983         }
7984         return 1;
7985 }
7986
7987 /**
7988  * Register the flow matcher.
7989  *
7990  * @param[in, out] dev
7991  *   Pointer to rte_eth_dev structure.
7992  * @param[in, out] matcher
7993  *   Pointer to flow matcher.
7994  * @param[in, out] key
7995  *   Pointer to flow table key.
7996  * @parm[in, out] dev_flow
7997  *   Pointer to the dev_flow.
7998  * @param[out] error
7999  *   pointer to error structure.
8000  *
8001  * @return
8002  *   0 on success otherwise -errno and errno is set.
8003  */
8004 static int
8005 flow_dv_matcher_register(struct rte_eth_dev *dev,
8006                          struct mlx5_flow_dv_matcher *matcher,
8007                          union mlx5_flow_tbl_key *key,
8008                          struct mlx5_flow *dev_flow,
8009                          struct rte_flow_error *error)
8010 {
8011         struct mlx5_priv *priv = dev->data->dev_private;
8012         struct mlx5_dev_ctx_shared *sh = priv->sh;
8013         struct mlx5_flow_dv_matcher *cache_matcher;
8014         struct mlx5dv_flow_matcher_attr dv_attr = {
8015                 .type = IBV_FLOW_ATTR_NORMAL,
8016                 .match_mask = (void *)&matcher->mask,
8017         };
8018         struct mlx5_flow_tbl_resource *tbl;
8019         struct mlx5_flow_tbl_data_entry *tbl_data;
8020         int ret;
8021
8022         tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
8023                                        key->domain, error);
8024         if (!tbl)
8025                 return -rte_errno;      /* No need to refill the error info */
8026         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
8027         /* Lookup from cache. */
8028         LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
8029                 if (matcher->crc == cache_matcher->crc &&
8030                     matcher->priority == cache_matcher->priority &&
8031                     !memcmp((const void *)matcher->mask.buf,
8032                             (const void *)cache_matcher->mask.buf,
8033                             cache_matcher->mask.size)) {
8034                         DRV_LOG(DEBUG,
8035                                 "%s group %u priority %hd use %s "
8036                                 "matcher %p: refcnt %d++",
8037                                 key->domain ? "FDB" : "NIC", key->table_id,
8038                                 cache_matcher->priority,
8039                                 key->direction ? "tx" : "rx",
8040                                 (void *)cache_matcher,
8041                                 rte_atomic32_read(&cache_matcher->refcnt));
8042                         rte_atomic32_inc(&cache_matcher->refcnt);
8043                         dev_flow->handle->dvh.matcher = cache_matcher;
8044                         /* old matcher should not make the table ref++. */
8045                         flow_dv_tbl_resource_release(dev, tbl);
8046                         return 0;
8047                 }
8048         }
8049         /* Register new matcher. */
8050         cache_matcher = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache_matcher), 0,
8051                                     SOCKET_ID_ANY);
8052         if (!cache_matcher) {
8053                 flow_dv_tbl_resource_release(dev, tbl);
8054                 return rte_flow_error_set(error, ENOMEM,
8055                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8056                                           "cannot allocate matcher memory");
8057         }
8058         *cache_matcher = *matcher;
8059         dv_attr.match_criteria_enable =
8060                 flow_dv_matcher_enable(cache_matcher->mask.buf);
8061         dv_attr.priority = matcher->priority;
8062         if (key->direction)
8063                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
8064         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
8065                                                &cache_matcher->matcher_object);
8066         if (ret) {
8067                 mlx5_free(cache_matcher);
8068 #ifdef HAVE_MLX5DV_DR
8069                 flow_dv_tbl_resource_release(dev, tbl);
8070 #endif
8071                 return rte_flow_error_set(error, ENOMEM,
8072                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8073                                           NULL, "cannot create matcher");
8074         }
8075         /* Save the table information */
8076         cache_matcher->tbl = tbl;
8077         rte_atomic32_init(&cache_matcher->refcnt);
8078         /* only matcher ref++, table ref++ already done above in get API. */
8079         rte_atomic32_inc(&cache_matcher->refcnt);
8080         LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
8081         dev_flow->handle->dvh.matcher = cache_matcher;
8082         DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
8083                 key->domain ? "FDB" : "NIC", key->table_id,
8084                 cache_matcher->priority,
8085                 key->direction ? "tx" : "rx", (void *)cache_matcher,
8086                 rte_atomic32_read(&cache_matcher->refcnt));
8087         return 0;
8088 }
8089
8090 /**
8091  * Find existing tag resource or create and register a new one.
8092  *
8093  * @param dev[in, out]
8094  *   Pointer to rte_eth_dev structure.
8095  * @param[in, out] tag_be24
8096  *   Tag value in big endian then R-shift 8.
8097  * @parm[in, out] dev_flow
8098  *   Pointer to the dev_flow.
8099  * @param[out] error
8100  *   pointer to error structure.
8101  *
8102  * @return
8103  *   0 on success otherwise -errno and errno is set.
8104  */
8105 static int
8106 flow_dv_tag_resource_register
8107                         (struct rte_eth_dev *dev,
8108                          uint32_t tag_be24,
8109                          struct mlx5_flow *dev_flow,
8110                          struct rte_flow_error *error)
8111 {
8112         struct mlx5_priv *priv = dev->data->dev_private;
8113         struct mlx5_dev_ctx_shared *sh = priv->sh;
8114         struct mlx5_flow_dv_tag_resource *cache_resource;
8115         struct mlx5_hlist_entry *entry;
8116         int ret;
8117
8118         /* Lookup a matching resource from cache. */
8119         entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
8120         if (entry) {
8121                 cache_resource = container_of
8122                         (entry, struct mlx5_flow_dv_tag_resource, entry);
8123                 rte_atomic32_inc(&cache_resource->refcnt);
8124                 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
8125                 dev_flow->dv.tag_resource = cache_resource;
8126                 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
8127                         (void *)cache_resource,
8128                         rte_atomic32_read(&cache_resource->refcnt));
8129                 return 0;
8130         }
8131         /* Register new resource. */
8132         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG],
8133                                        &dev_flow->handle->dvh.rix_tag);
8134         if (!cache_resource)
8135                 return rte_flow_error_set(error, ENOMEM,
8136                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8137                                           "cannot allocate resource memory");
8138         cache_resource->entry.key = (uint64_t)tag_be24;
8139         ret = mlx5_flow_os_create_flow_action_tag(tag_be24,
8140                                                   &cache_resource->action);
8141         if (ret) {
8142                 mlx5_free(cache_resource);
8143                 return rte_flow_error_set(error, ENOMEM,
8144                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8145                                           NULL, "cannot create action");
8146         }
8147         rte_atomic32_init(&cache_resource->refcnt);
8148         rte_atomic32_inc(&cache_resource->refcnt);
8149         if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
8150                 mlx5_flow_os_destroy_flow_action(cache_resource->action);
8151                 mlx5_free(cache_resource);
8152                 return rte_flow_error_set(error, EEXIST,
8153                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8154                                           NULL, "cannot insert tag");
8155         }
8156         dev_flow->dv.tag_resource = cache_resource;
8157         DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
8158                 (void *)cache_resource,
8159                 rte_atomic32_read(&cache_resource->refcnt));
8160         return 0;
8161 }
8162
8163 /**
8164  * Release the tag.
8165  *
8166  * @param dev
8167  *   Pointer to Ethernet device.
8168  * @param tag_idx
8169  *   Tag index.
8170  *
8171  * @return
8172  *   1 while a reference on it exists, 0 when freed.
8173  */
8174 static int
8175 flow_dv_tag_release(struct rte_eth_dev *dev,
8176                     uint32_t tag_idx)
8177 {
8178         struct mlx5_priv *priv = dev->data->dev_private;
8179         struct mlx5_dev_ctx_shared *sh = priv->sh;
8180         struct mlx5_flow_dv_tag_resource *tag;
8181
8182         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8183         if (!tag)
8184                 return 0;
8185         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
8186                 dev->data->port_id, (void *)tag,
8187                 rte_atomic32_read(&tag->refcnt));
8188         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
8189                 claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
8190                 mlx5_hlist_remove(sh->tag_table, &tag->entry);
8191                 DRV_LOG(DEBUG, "port %u tag %p: removed",
8192                         dev->data->port_id, (void *)tag);
8193                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8194                 return 0;
8195         }
8196         return 1;
8197 }
8198
8199 /**
8200  * Translate port ID action to vport.
8201  *
8202  * @param[in] dev
8203  *   Pointer to rte_eth_dev structure.
8204  * @param[in] action
8205  *   Pointer to the port ID action.
8206  * @param[out] dst_port_id
8207  *   The target port ID.
8208  * @param[out] error
8209  *   Pointer to the error structure.
8210  *
8211  * @return
8212  *   0 on success, a negative errno value otherwise and rte_errno is set.
8213  */
8214 static int
8215 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
8216                                  const struct rte_flow_action *action,
8217                                  uint32_t *dst_port_id,
8218                                  struct rte_flow_error *error)
8219 {
8220         uint32_t port;
8221         struct mlx5_priv *priv;
8222         const struct rte_flow_action_port_id *conf =
8223                         (const struct rte_flow_action_port_id *)action->conf;
8224
8225         port = conf->original ? dev->data->port_id : conf->id;
8226         priv = mlx5_port_to_eswitch_info(port, false);
8227         if (!priv)
8228                 return rte_flow_error_set(error, -rte_errno,
8229                                           RTE_FLOW_ERROR_TYPE_ACTION,
8230                                           NULL,
8231                                           "No eswitch info was found for port");
8232 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
8233         /*
8234          * This parameter is transferred to
8235          * mlx5dv_dr_action_create_dest_ib_port().
8236          */
8237         *dst_port_id = priv->dev_port;
8238 #else
8239         /*
8240          * Legacy mode, no LAG configurations is supported.
8241          * This parameter is transferred to
8242          * mlx5dv_dr_action_create_dest_vport().
8243          */
8244         *dst_port_id = priv->vport_id;
8245 #endif
8246         return 0;
8247 }
8248
8249 /**
8250  * Create a counter with aging configuration.
8251  *
8252  * @param[in] dev
8253  *   Pointer to rte_eth_dev structure.
8254  * @param[out] count
8255  *   Pointer to the counter action configuration.
8256  * @param[in] age
8257  *   Pointer to the aging action configuration.
8258  *
8259  * @return
8260  *   Index to flow counter on success, 0 otherwise.
8261  */
8262 static uint32_t
8263 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
8264                                 struct mlx5_flow *dev_flow,
8265                                 const struct rte_flow_action_count *count,
8266                                 const struct rte_flow_action_age *age)
8267 {
8268         uint32_t counter;
8269         struct mlx5_age_param *age_param;
8270
8271         if (count && count->shared)
8272                 counter = flow_dv_counter_get_shared(dev, count->id);
8273         else
8274                 counter = flow_dv_counter_alloc(dev, !!age);
8275         if (!counter || age == NULL)
8276                 return counter;
8277         age_param  = flow_dv_counter_idx_get_age(dev, counter);
8278         age_param->context = age->context ? age->context :
8279                 (void *)(uintptr_t)(dev_flow->flow_idx);
8280         age_param->timeout = age->timeout;
8281         age_param->port_id = dev->data->port_id;
8282         __atomic_store_n(&age_param->sec_since_last_hit, 0, __ATOMIC_RELAXED);
8283         __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED);
8284         return counter;
8285 }
8286 /**
8287  * Add Tx queue matcher
8288  *
8289  * @param[in] dev
8290  *   Pointer to the dev struct.
8291  * @param[in, out] matcher
8292  *   Flow matcher.
8293  * @param[in, out] key
8294  *   Flow matcher value.
8295  * @param[in] item
8296  *   Flow pattern to translate.
8297  * @param[in] inner
8298  *   Item is inner pattern.
8299  */
8300 static void
8301 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
8302                                 void *matcher, void *key,
8303                                 const struct rte_flow_item *item)
8304 {
8305         const struct mlx5_rte_flow_item_tx_queue *queue_m;
8306         const struct mlx5_rte_flow_item_tx_queue *queue_v;
8307         void *misc_m =
8308                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8309         void *misc_v =
8310                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8311         struct mlx5_txq_ctrl *txq;
8312         uint32_t queue;
8313
8314
8315         queue_m = (const void *)item->mask;
8316         if (!queue_m)
8317                 return;
8318         queue_v = (const void *)item->spec;
8319         if (!queue_v)
8320                 return;
8321         txq = mlx5_txq_get(dev, queue_v->queue);
8322         if (!txq)
8323                 return;
8324         queue = txq->obj->sq->id;
8325         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
8326         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
8327                  queue & queue_m->queue);
8328         mlx5_txq_release(dev, queue_v->queue);
8329 }
8330
8331 /**
8332  * Set the hash fields according to the @p flow information.
8333  *
8334  * @param[in] dev_flow
8335  *   Pointer to the mlx5_flow.
8336  * @param[in] rss_desc
8337  *   Pointer to the mlx5_flow_rss_desc.
8338  */
8339 static void
8340 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
8341                        struct mlx5_flow_rss_desc *rss_desc)
8342 {
8343         uint64_t items = dev_flow->handle->layers;
8344         int rss_inner = 0;
8345         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
8346
8347         dev_flow->hash_fields = 0;
8348 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
8349         if (rss_desc->level >= 2) {
8350                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
8351                 rss_inner = 1;
8352         }
8353 #endif
8354         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
8355             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
8356                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
8357                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
8358                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
8359                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
8360                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
8361                         else
8362                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
8363                 }
8364         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
8365                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
8366                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
8367                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
8368                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
8369                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
8370                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
8371                         else
8372                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
8373                 }
8374         }
8375         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
8376             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
8377                 if (rss_types & ETH_RSS_UDP) {
8378                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
8379                                 dev_flow->hash_fields |=
8380                                                 IBV_RX_HASH_SRC_PORT_UDP;
8381                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
8382                                 dev_flow->hash_fields |=
8383                                                 IBV_RX_HASH_DST_PORT_UDP;
8384                         else
8385                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
8386                 }
8387         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
8388                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
8389                 if (rss_types & ETH_RSS_TCP) {
8390                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
8391                                 dev_flow->hash_fields |=
8392                                                 IBV_RX_HASH_SRC_PORT_TCP;
8393                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
8394                                 dev_flow->hash_fields |=
8395                                                 IBV_RX_HASH_DST_PORT_TCP;
8396                         else
8397                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
8398                 }
8399         }
8400 }
8401
8402 /**
8403  * Create an Rx Hash queue.
8404  *
8405  * @param dev
8406  *   Pointer to Ethernet device.
8407  * @param[in] dev_flow
8408  *   Pointer to the mlx5_flow.
8409  * @param[in] rss_desc
8410  *   Pointer to the mlx5_flow_rss_desc.
8411  * @param[out] hrxq_idx
8412  *   Hash Rx queue index.
8413  *
8414  * @return
8415  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
8416  */
8417 static struct mlx5_hrxq *
8418 flow_dv_handle_rx_queue(struct rte_eth_dev *dev,
8419                         struct mlx5_flow *dev_flow,
8420                         struct mlx5_flow_rss_desc *rss_desc,
8421                         uint32_t *hrxq_idx)
8422 {
8423         struct mlx5_priv *priv = dev->data->dev_private;
8424         struct mlx5_flow_handle *dh = dev_flow->handle;
8425         struct mlx5_hrxq *hrxq;
8426
8427         MLX5_ASSERT(rss_desc->queue_num);
8428         *hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
8429                                   MLX5_RSS_HASH_KEY_LEN,
8430                                   dev_flow->hash_fields,
8431                                   rss_desc->queue,
8432                                   rss_desc->queue_num);
8433         if (!*hrxq_idx) {
8434                 *hrxq_idx = mlx5_hrxq_new
8435                                 (dev, rss_desc->key,
8436                                  MLX5_RSS_HASH_KEY_LEN,
8437                                  dev_flow->hash_fields,
8438                                  rss_desc->queue,
8439                                  rss_desc->queue_num,
8440                                  !!(dh->layers &
8441                                  MLX5_FLOW_LAYER_TUNNEL));
8442                 if (!*hrxq_idx)
8443                         return NULL;
8444         }
8445         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
8446                               *hrxq_idx);
8447         return hrxq;
8448 }
8449
8450 /**
8451  * Find existing sample resource or create and register a new one.
8452  *
8453  * @param[in, out] dev
8454  *   Pointer to rte_eth_dev structure.
8455  * @param[in] attr
8456  *   Attributes of flow that includes this item.
8457  * @param[in] resource
8458  *   Pointer to sample resource.
8459  * @parm[in, out] dev_flow
8460  *   Pointer to the dev_flow.
8461  * @param[in, out] sample_dv_actions
8462  *   Pointer to sample actions list.
8463  * @param[out] error
8464  *   pointer to error structure.
8465  *
8466  * @return
8467  *   0 on success otherwise -errno and errno is set.
8468  */
8469 static int
8470 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
8471                          const struct rte_flow_attr *attr,
8472                          struct mlx5_flow_dv_sample_resource *resource,
8473                          struct mlx5_flow *dev_flow,
8474                          void **sample_dv_actions,
8475                          struct rte_flow_error *error)
8476 {
8477         struct mlx5_flow_dv_sample_resource *cache_resource;
8478         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
8479         struct mlx5_priv *priv = dev->data->dev_private;
8480         struct mlx5_dev_ctx_shared *sh = priv->sh;
8481         struct mlx5_flow_tbl_resource *tbl;
8482         uint32_t idx = 0;
8483         const uint32_t next_ft_step = 1;
8484         uint32_t next_ft_id = resource->ft_id + next_ft_step;
8485
8486         /* Lookup a matching resource from cache. */
8487         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_SAMPLE], sh->sample_action_list,
8488                       idx, cache_resource, next) {
8489                 if (resource->ratio == cache_resource->ratio &&
8490                     resource->ft_type == cache_resource->ft_type &&
8491                     resource->ft_id == cache_resource->ft_id &&
8492                     resource->set_action == cache_resource->set_action &&
8493                     !memcmp((void *)&resource->sample_act,
8494                             (void *)&cache_resource->sample_act,
8495                             sizeof(struct mlx5_flow_sub_actions_list))) {
8496                         DRV_LOG(DEBUG, "sample resource %p: refcnt %d++",
8497                                 (void *)cache_resource,
8498                                 __atomic_load_n(&cache_resource->refcnt,
8499                                                 __ATOMIC_RELAXED));
8500                         __atomic_fetch_add(&cache_resource->refcnt, 1,
8501                                            __ATOMIC_RELAXED);
8502                         dev_flow->handle->dvh.rix_sample = idx;
8503                         dev_flow->dv.sample_res = cache_resource;
8504                         return 0;
8505                 }
8506         }
8507         /* Register new sample resource. */
8508         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE],
8509                                        &dev_flow->handle->dvh.rix_sample);
8510         if (!cache_resource)
8511                 return rte_flow_error_set(error, ENOMEM,
8512                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8513                                           NULL,
8514                                           "cannot allocate resource memory");
8515         *cache_resource = *resource;
8516         /* Create normal path table level */
8517         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
8518                                         attr->egress, attr->transfer, error);
8519         if (!tbl) {
8520                 rte_flow_error_set(error, ENOMEM,
8521                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8522                                           NULL,
8523                                           "fail to create normal path table "
8524                                           "for sample");
8525                 goto error;
8526         }
8527         cache_resource->normal_path_tbl = tbl;
8528         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
8529                 cache_resource->default_miss =
8530                                 mlx5_glue->dr_create_flow_action_default_miss();
8531                 if (!cache_resource->default_miss) {
8532                         rte_flow_error_set(error, ENOMEM,
8533                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8534                                                 NULL,
8535                                                 "cannot create default miss "
8536                                                 "action");
8537                         goto error;
8538                 }
8539                 sample_dv_actions[resource->sample_act.actions_num++] =
8540                                                 cache_resource->default_miss;
8541         }
8542         /* Create a DR sample action */
8543         sampler_attr.sample_ratio = cache_resource->ratio;
8544         sampler_attr.default_next_table = tbl->obj;
8545         sampler_attr.num_sample_actions = resource->sample_act.actions_num;
8546         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
8547                                                         &sample_dv_actions[0];
8548         sampler_attr.action = cache_resource->set_action;
8549         cache_resource->verbs_action =
8550                 mlx5_glue->dr_create_flow_action_sampler(&sampler_attr);
8551         if (!cache_resource->verbs_action) {
8552                 rte_flow_error_set(error, ENOMEM,
8553                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8554                                         NULL, "cannot create sample action");
8555                 goto error;
8556         }
8557         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8558         ILIST_INSERT(sh->ipool[MLX5_IPOOL_SAMPLE], &sh->sample_action_list,
8559                      dev_flow->handle->dvh.rix_sample, cache_resource,
8560                      next);
8561         dev_flow->dv.sample_res = cache_resource;
8562         DRV_LOG(DEBUG, "new sample resource %p: refcnt %d++",
8563                 (void *)cache_resource,
8564                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8565         return 0;
8566 error:
8567         if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
8568                 if (cache_resource->default_miss)
8569                         claim_zero(mlx5_glue->destroy_flow_action
8570                                 (cache_resource->default_miss));
8571         } else {
8572                 if (cache_resource->sample_idx.rix_hrxq &&
8573                     !mlx5_hrxq_release(dev,
8574                                 cache_resource->sample_idx.rix_hrxq))
8575                         cache_resource->sample_idx.rix_hrxq = 0;
8576                 if (cache_resource->sample_idx.rix_tag &&
8577                     !flow_dv_tag_release(dev,
8578                                 cache_resource->sample_idx.rix_tag))
8579                         cache_resource->sample_idx.rix_tag = 0;
8580                 if (cache_resource->sample_idx.cnt) {
8581                         flow_dv_counter_release(dev,
8582                                 cache_resource->sample_idx.cnt);
8583                         cache_resource->sample_idx.cnt = 0;
8584                 }
8585         }
8586         if (cache_resource->normal_path_tbl)
8587                 flow_dv_tbl_resource_release(dev,
8588                                 cache_resource->normal_path_tbl);
8589         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE],
8590                                 dev_flow->handle->dvh.rix_sample);
8591         dev_flow->handle->dvh.rix_sample = 0;
8592         return -rte_errno;
8593 }
8594
8595 /**
8596  * Find existing destination array resource or create and register a new one.
8597  *
8598  * @param[in, out] dev
8599  *   Pointer to rte_eth_dev structure.
8600  * @param[in] attr
8601  *   Attributes of flow that includes this item.
8602  * @param[in] resource
8603  *   Pointer to destination array resource.
8604  * @parm[in, out] dev_flow
8605  *   Pointer to the dev_flow.
8606  * @param[out] error
8607  *   pointer to error structure.
8608  *
8609  * @return
8610  *   0 on success otherwise -errno and errno is set.
8611  */
8612 static int
8613 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
8614                          const struct rte_flow_attr *attr,
8615                          struct mlx5_flow_dv_dest_array_resource *resource,
8616                          struct mlx5_flow *dev_flow,
8617                          struct rte_flow_error *error)
8618 {
8619         struct mlx5_flow_dv_dest_array_resource *cache_resource;
8620         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
8621         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
8622         struct mlx5_priv *priv = dev->data->dev_private;
8623         struct mlx5_dev_ctx_shared *sh = priv->sh;
8624         struct mlx5_flow_sub_actions_list *sample_act;
8625         struct mlx5dv_dr_domain *domain;
8626         uint32_t idx = 0;
8627
8628         /* Lookup a matching resource from cache. */
8629         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8630                       sh->dest_array_list,
8631                       idx, cache_resource, next) {
8632                 if (resource->num_of_dest == cache_resource->num_of_dest &&
8633                     resource->ft_type == cache_resource->ft_type &&
8634                     !memcmp((void *)cache_resource->sample_act,
8635                             (void *)resource->sample_act,
8636                            (resource->num_of_dest *
8637                            sizeof(struct mlx5_flow_sub_actions_list)))) {
8638                         DRV_LOG(DEBUG, "dest array resource %p: refcnt %d++",
8639                                 (void *)cache_resource,
8640                                 __atomic_load_n(&cache_resource->refcnt,
8641                                                 __ATOMIC_RELAXED));
8642                         __atomic_fetch_add(&cache_resource->refcnt, 1,
8643                                            __ATOMIC_RELAXED);
8644                         dev_flow->handle->dvh.rix_dest_array = idx;
8645                         dev_flow->dv.dest_array_res = cache_resource;
8646                         return 0;
8647                 }
8648         }
8649         /* Register new destination array resource. */
8650         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8651                                        &dev_flow->handle->dvh.rix_dest_array);
8652         if (!cache_resource)
8653                 return rte_flow_error_set(error, ENOMEM,
8654                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8655                                           NULL,
8656                                           "cannot allocate resource memory");
8657         *cache_resource = *resource;
8658         if (attr->transfer)
8659                 domain = sh->fdb_domain;
8660         else if (attr->ingress)
8661                 domain = sh->rx_domain;
8662         else
8663                 domain = sh->tx_domain;
8664         for (idx = 0; idx < resource->num_of_dest; idx++) {
8665                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
8666                                  mlx5_malloc(MLX5_MEM_ZERO,
8667                                  sizeof(struct mlx5dv_dr_action_dest_attr),
8668                                  0, SOCKET_ID_ANY);
8669                 if (!dest_attr[idx]) {
8670                         rte_flow_error_set(error, ENOMEM,
8671                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8672                                            NULL,
8673                                            "cannot allocate resource memory");
8674                         goto error;
8675                 }
8676                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
8677                 sample_act = &resource->sample_act[idx];
8678                 if (sample_act->action_flags == MLX5_FLOW_ACTION_QUEUE) {
8679                         dest_attr[idx]->dest = sample_act->dr_queue_action;
8680                 } else if (sample_act->action_flags ==
8681                           (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP)) {
8682                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
8683                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
8684                         dest_attr[idx]->dest_reformat->reformat =
8685                                         sample_act->dr_encap_action;
8686                         dest_attr[idx]->dest_reformat->dest =
8687                                         sample_act->dr_port_id_action;
8688                 } else if (sample_act->action_flags ==
8689                            MLX5_FLOW_ACTION_PORT_ID) {
8690                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
8691                 }
8692         }
8693         /* create a dest array actioin */
8694         cache_resource->action = mlx5_glue->dr_create_flow_action_dest_array
8695                                                 (domain,
8696                                                  cache_resource->num_of_dest,
8697                                                  dest_attr);
8698         if (!cache_resource->action) {
8699                 rte_flow_error_set(error, ENOMEM,
8700                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8701                                    NULL,
8702                                    "cannot create destination array action");
8703                 goto error;
8704         }
8705         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8706         ILIST_INSERT(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8707                      &sh->dest_array_list,
8708                      dev_flow->handle->dvh.rix_dest_array, cache_resource,
8709                      next);
8710         dev_flow->dv.dest_array_res = cache_resource;
8711         DRV_LOG(DEBUG, "new destination array resource %p: refcnt %d++",
8712                 (void *)cache_resource,
8713                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8714         for (idx = 0; idx < resource->num_of_dest; idx++)
8715                 mlx5_free(dest_attr[idx]);
8716         return 0;
8717 error:
8718         for (idx = 0; idx < resource->num_of_dest; idx++) {
8719                 struct mlx5_flow_sub_actions_idx *act_res =
8720                                         &cache_resource->sample_idx[idx];
8721                 if (act_res->rix_hrxq &&
8722                     !mlx5_hrxq_release(dev,
8723                                 act_res->rix_hrxq))
8724                         act_res->rix_hrxq = 0;
8725                 if (act_res->rix_encap_decap &&
8726                         !flow_dv_encap_decap_resource_release(dev,
8727                                 act_res->rix_encap_decap))
8728                         act_res->rix_encap_decap = 0;
8729                 if (act_res->rix_port_id_action &&
8730                         !flow_dv_port_id_action_resource_release(dev,
8731                                 act_res->rix_port_id_action))
8732                         act_res->rix_port_id_action = 0;
8733                 if (dest_attr[idx])
8734                         mlx5_free(dest_attr[idx]);
8735         }
8736
8737         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8738                                 dev_flow->handle->dvh.rix_dest_array);
8739         dev_flow->handle->dvh.rix_dest_array = 0;
8740         return -rte_errno;
8741 }
8742
8743 /**
8744  * Convert Sample action to DV specification.
8745  *
8746  * @param[in] dev
8747  *   Pointer to rte_eth_dev structure.
8748  * @param[in] action
8749  *   Pointer to action structure.
8750  * @param[in, out] dev_flow
8751  *   Pointer to the mlx5_flow.
8752  * @param[in] attr
8753  *   Pointer to the flow attributes.
8754  * @param[in, out] num_of_dest
8755  *   Pointer to the num of destination.
8756  * @param[in, out] sample_actions
8757  *   Pointer to sample actions list.
8758  * @param[in, out] res
8759  *   Pointer to sample resource.
8760  * @param[out] error
8761  *   Pointer to the error structure.
8762  *
8763  * @return
8764  *   0 on success, a negative errno value otherwise and rte_errno is set.
8765  */
8766 static int
8767 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
8768                                 const struct rte_flow_action *action,
8769                                 struct mlx5_flow *dev_flow,
8770                                 const struct rte_flow_attr *attr,
8771                                 uint32_t *num_of_dest,
8772                                 void **sample_actions,
8773                                 struct mlx5_flow_dv_sample_resource *res,
8774                                 struct rte_flow_error *error)
8775 {
8776         struct mlx5_priv *priv = dev->data->dev_private;
8777         const struct rte_flow_action_sample *sample_action;
8778         const struct rte_flow_action *sub_actions;
8779         const struct rte_flow_action_queue *queue;
8780         struct mlx5_flow_sub_actions_list *sample_act;
8781         struct mlx5_flow_sub_actions_idx *sample_idx;
8782         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
8783                                               priv->rss_desc)
8784                                               [!!priv->flow_nested_idx];
8785         uint64_t action_flags = 0;
8786
8787         sample_act = &res->sample_act;
8788         sample_idx = &res->sample_idx;
8789         sample_action = (const struct rte_flow_action_sample *)action->conf;
8790         res->ratio = sample_action->ratio;
8791         sub_actions = sample_action->actions;
8792         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
8793                 int type = sub_actions->type;
8794                 uint32_t pre_rix = 0;
8795                 void *pre_r;
8796                 switch (type) {
8797                 case RTE_FLOW_ACTION_TYPE_QUEUE:
8798                 {
8799                         struct mlx5_hrxq *hrxq;
8800                         uint32_t hrxq_idx;
8801
8802                         queue = sub_actions->conf;
8803                         rss_desc->queue_num = 1;
8804                         rss_desc->queue[0] = queue->index;
8805                         hrxq = flow_dv_handle_rx_queue(dev, dev_flow,
8806                                         rss_desc, &hrxq_idx);
8807                         if (!hrxq)
8808                                 return rte_flow_error_set
8809                                         (error, rte_errno,
8810                                          RTE_FLOW_ERROR_TYPE_ACTION,
8811                                          NULL,
8812                                          "cannot create fate queue");
8813                         sample_act->dr_queue_action = hrxq->action;
8814                         sample_idx->rix_hrxq = hrxq_idx;
8815                         sample_actions[sample_act->actions_num++] =
8816                                                 hrxq->action;
8817                         (*num_of_dest)++;
8818                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
8819                         if (action_flags & MLX5_FLOW_ACTION_MARK)
8820                                 dev_flow->handle->rix_hrxq = hrxq_idx;
8821                         dev_flow->handle->fate_action =
8822                                         MLX5_FLOW_FATE_QUEUE;
8823                         break;
8824                 }
8825                 case RTE_FLOW_ACTION_TYPE_MARK:
8826                 {
8827                         uint32_t tag_be = mlx5_flow_mark_set
8828                                 (((const struct rte_flow_action_mark *)
8829                                 (sub_actions->conf))->id);
8830
8831                         dev_flow->handle->mark = 1;
8832                         pre_rix = dev_flow->handle->dvh.rix_tag;
8833                         /* Save the mark resource before sample */
8834                         pre_r = dev_flow->dv.tag_resource;
8835                         if (flow_dv_tag_resource_register(dev, tag_be,
8836                                                   dev_flow, error))
8837                                 return -rte_errno;
8838                         MLX5_ASSERT(dev_flow->dv.tag_resource);
8839                         sample_act->dr_tag_action =
8840                                 dev_flow->dv.tag_resource->action;
8841                         sample_idx->rix_tag =
8842                                 dev_flow->handle->dvh.rix_tag;
8843                         sample_actions[sample_act->actions_num++] =
8844                                                 sample_act->dr_tag_action;
8845                         /* Recover the mark resource after sample */
8846                         dev_flow->dv.tag_resource = pre_r;
8847                         dev_flow->handle->dvh.rix_tag = pre_rix;
8848                         action_flags |= MLX5_FLOW_ACTION_MARK;
8849                         break;
8850                 }
8851                 case RTE_FLOW_ACTION_TYPE_COUNT:
8852                 {
8853                         uint32_t counter;
8854
8855                         counter = flow_dv_translate_create_counter(dev,
8856                                         dev_flow, sub_actions->conf, 0);
8857                         if (!counter)
8858                                 return rte_flow_error_set
8859                                                 (error, rte_errno,
8860                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8861                                                  NULL,
8862                                                  "cannot create counter"
8863                                                  " object.");
8864                         sample_idx->cnt = counter;
8865                         sample_act->dr_cnt_action =
8866                                   (flow_dv_counter_get_by_idx(dev,
8867                                   counter, NULL))->action;
8868                         sample_actions[sample_act->actions_num++] =
8869                                                 sample_act->dr_cnt_action;
8870                         action_flags |= MLX5_FLOW_ACTION_COUNT;
8871                         break;
8872                 }
8873                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
8874                 {
8875                         struct mlx5_flow_dv_port_id_action_resource
8876                                         port_id_resource;
8877                         uint32_t port_id = 0;
8878
8879                         memset(&port_id_resource, 0, sizeof(port_id_resource));
8880                         /* Save the port id resource before sample */
8881                         pre_rix = dev_flow->handle->rix_port_id_action;
8882                         pre_r = dev_flow->dv.port_id_action;
8883                         if (flow_dv_translate_action_port_id(dev, sub_actions,
8884                                                              &port_id, error))
8885                                 return -rte_errno;
8886                         port_id_resource.port_id = port_id;
8887                         if (flow_dv_port_id_action_resource_register
8888                             (dev, &port_id_resource, dev_flow, error))
8889                                 return -rte_errno;
8890                         sample_act->dr_port_id_action =
8891                                 dev_flow->dv.port_id_action->action;
8892                         sample_idx->rix_port_id_action =
8893                                 dev_flow->handle->rix_port_id_action;
8894                         sample_actions[sample_act->actions_num++] =
8895                                                 sample_act->dr_port_id_action;
8896                         /* Recover the port id resource after sample */
8897                         dev_flow->dv.port_id_action = pre_r;
8898                         dev_flow->handle->rix_port_id_action = pre_rix;
8899                         (*num_of_dest)++;
8900                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
8901                         break;
8902                 }
8903                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
8904                         /* Save the encap resource before sample */
8905                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
8906                         pre_r = dev_flow->dv.encap_decap;
8907                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
8908                                                            dev_flow,
8909                                                            attr->transfer,
8910                                                            error))
8911                                 return -rte_errno;
8912                         sample_act->dr_encap_action =
8913                                 dev_flow->dv.encap_decap->action;
8914                         sample_idx->rix_encap_decap =
8915                                 dev_flow->handle->dvh.rix_encap_decap;
8916                         sample_actions[sample_act->actions_num++] =
8917                                                 sample_act->dr_encap_action;
8918                         /* Recover the encap resource after sample */
8919                         dev_flow->dv.encap_decap = pre_r;
8920                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
8921                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
8922                         break;
8923                 default:
8924                         return rte_flow_error_set(error, EINVAL,
8925                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8926                                 NULL,
8927                                 "Not support for sampler action");
8928                 }
8929         }
8930         sample_act->action_flags = action_flags;
8931         res->ft_id = dev_flow->dv.group;
8932         if (attr->transfer) {
8933                 union {
8934                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
8935                         uint64_t set_action;
8936                 } action_ctx = { .set_action = 0 };
8937
8938                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
8939                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
8940                          MLX5_MODIFICATION_TYPE_SET);
8941                 MLX5_SET(set_action_in, action_ctx.action_in, field,
8942                          MLX5_MODI_META_REG_C_0);
8943                 MLX5_SET(set_action_in, action_ctx.action_in, data,
8944                          priv->vport_meta_tag);
8945                 res->set_action = action_ctx.set_action;
8946         } else if (attr->ingress) {
8947                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
8948         }
8949         return 0;
8950 }
8951
8952 /**
8953  * Convert Sample action to DV specification.
8954  *
8955  * @param[in] dev
8956  *   Pointer to rte_eth_dev structure.
8957  * @param[in, out] dev_flow
8958  *   Pointer to the mlx5_flow.
8959  * @param[in] attr
8960  *   Pointer to the flow attributes.
8961  * @param[in] num_of_dest
8962  *   The num of destination.
8963  * @param[in, out] res
8964  *   Pointer to sample resource.
8965  * @param[in, out] mdest_res
8966  *   Pointer to destination array resource.
8967  * @param[in] sample_actions
8968  *   Pointer to sample path actions list.
8969  * @param[in] action_flags
8970  *   Holds the actions detected until now.
8971  * @param[out] error
8972  *   Pointer to the error structure.
8973  *
8974  * @return
8975  *   0 on success, a negative errno value otherwise and rte_errno is set.
8976  */
8977 static int
8978 flow_dv_create_action_sample(struct rte_eth_dev *dev,
8979                              struct mlx5_flow *dev_flow,
8980                              const struct rte_flow_attr *attr,
8981                              uint32_t num_of_dest,
8982                              struct mlx5_flow_dv_sample_resource *res,
8983                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
8984                              void **sample_actions,
8985                              uint64_t action_flags,
8986                              struct rte_flow_error *error)
8987 {
8988         struct mlx5_priv *priv = dev->data->dev_private;
8989         /* update normal path action resource into last index of array */
8990         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
8991         struct mlx5_flow_sub_actions_list *sample_act =
8992                                         &mdest_res->sample_act[dest_index];
8993         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
8994                                               priv->rss_desc)
8995                                               [!!priv->flow_nested_idx];
8996         uint32_t normal_idx = 0;
8997         struct mlx5_hrxq *hrxq;
8998         uint32_t hrxq_idx;
8999
9000         if (num_of_dest > 1) {
9001                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
9002                         /* Handle QP action for mirroring */
9003                         hrxq = flow_dv_handle_rx_queue(dev, dev_flow,
9004                                                        rss_desc, &hrxq_idx);
9005                         if (!hrxq)
9006                                 return rte_flow_error_set
9007                                      (error, rte_errno,
9008                                       RTE_FLOW_ERROR_TYPE_ACTION,
9009                                       NULL,
9010                                       "cannot create rx queue");
9011                         normal_idx++;
9012                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
9013                         sample_act->dr_queue_action = hrxq->action;
9014                         if (action_flags & MLX5_FLOW_ACTION_MARK)
9015                                 dev_flow->handle->rix_hrxq = hrxq_idx;
9016                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9017                 }
9018                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
9019                         normal_idx++;
9020                         mdest_res->sample_idx[dest_index].rix_encap_decap =
9021                                 dev_flow->handle->dvh.rix_encap_decap;
9022                         sample_act->dr_encap_action =
9023                                 dev_flow->dv.encap_decap->action;
9024                 }
9025                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
9026                         normal_idx++;
9027                         mdest_res->sample_idx[dest_index].rix_port_id_action =
9028                                 dev_flow->handle->rix_port_id_action;
9029                         sample_act->dr_port_id_action =
9030                                 dev_flow->dv.port_id_action->action;
9031                 }
9032                 sample_act->actions_num = normal_idx;
9033                 /* update sample action resource into first index of array */
9034                 mdest_res->ft_type = res->ft_type;
9035                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
9036                                 sizeof(struct mlx5_flow_sub_actions_idx));
9037                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
9038                                 sizeof(struct mlx5_flow_sub_actions_list));
9039                 mdest_res->num_of_dest = num_of_dest;
9040                 if (flow_dv_dest_array_resource_register(dev, attr, mdest_res,
9041                                                          dev_flow, error))
9042                         return rte_flow_error_set(error, EINVAL,
9043                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9044                                                   NULL, "can't create sample "
9045                                                   "action");
9046         } else {
9047                 if (flow_dv_sample_resource_register(dev, attr, res, dev_flow,
9048                                                      sample_actions, error))
9049                         return rte_flow_error_set(error, EINVAL,
9050                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9051                                                   NULL,
9052                                                   "can't create sample action");
9053         }
9054         return 0;
9055 }
9056
9057 /**
9058  * Fill the flow with DV spec, lock free
9059  * (mutex should be acquired by caller).
9060  *
9061  * @param[in] dev
9062  *   Pointer to rte_eth_dev structure.
9063  * @param[in, out] dev_flow
9064  *   Pointer to the sub flow.
9065  * @param[in] attr
9066  *   Pointer to the flow attributes.
9067  * @param[in] items
9068  *   Pointer to the list of items.
9069  * @param[in] actions
9070  *   Pointer to the list of actions.
9071  * @param[out] error
9072  *   Pointer to the error structure.
9073  *
9074  * @return
9075  *   0 on success, a negative errno value otherwise and rte_errno is set.
9076  */
9077 static int
9078 __flow_dv_translate(struct rte_eth_dev *dev,
9079                     struct mlx5_flow *dev_flow,
9080                     const struct rte_flow_attr *attr,
9081                     const struct rte_flow_item items[],
9082                     const struct rte_flow_action actions[],
9083                     struct rte_flow_error *error)
9084 {
9085         struct mlx5_priv *priv = dev->data->dev_private;
9086         struct mlx5_dev_config *dev_conf = &priv->config;
9087         struct rte_flow *flow = dev_flow->flow;
9088         struct mlx5_flow_handle *handle = dev_flow->handle;
9089         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
9090                                               priv->rss_desc)
9091                                               [!!priv->flow_nested_idx];
9092         uint64_t item_flags = 0;
9093         uint64_t last_item = 0;
9094         uint64_t action_flags = 0;
9095         uint64_t priority = attr->priority;
9096         struct mlx5_flow_dv_matcher matcher = {
9097                 .mask = {
9098                         .size = sizeof(matcher.mask.buf) -
9099                                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
9100                 },
9101         };
9102         int actions_n = 0;
9103         bool actions_end = false;
9104         union {
9105                 struct mlx5_flow_dv_modify_hdr_resource res;
9106                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
9107                             sizeof(struct mlx5_modification_cmd) *
9108                             (MLX5_MAX_MODIFY_NUM + 1)];
9109         } mhdr_dummy;
9110         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
9111         const struct rte_flow_action_count *count = NULL;
9112         const struct rte_flow_action_age *age = NULL;
9113         union flow_dv_attr flow_attr = { .attr = 0 };
9114         uint32_t tag_be;
9115         union mlx5_flow_tbl_key tbl_key;
9116         uint32_t modify_action_position = UINT32_MAX;
9117         void *match_mask = matcher.mask.buf;
9118         void *match_value = dev_flow->dv.value.buf;
9119         uint8_t next_protocol = 0xff;
9120         struct rte_vlan_hdr vlan = { 0 };
9121         struct mlx5_flow_dv_dest_array_resource mdest_res;
9122         struct mlx5_flow_dv_sample_resource sample_res;
9123         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
9124         struct mlx5_flow_sub_actions_list *sample_act;
9125         uint32_t sample_act_pos = UINT32_MAX;
9126         uint32_t num_of_dest = 0;
9127         int tmp_actions_n = 0;
9128         uint32_t table;
9129         int ret = 0;
9130
9131         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
9132         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
9133         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
9134                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
9135         /* update normal path action resource into last index of array */
9136         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
9137         ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
9138                                        !!priv->fdb_def_rule, &table, error);
9139         if (ret)
9140                 return ret;
9141         dev_flow->dv.group = table;
9142         if (attr->transfer)
9143                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
9144         if (priority == MLX5_FLOW_PRIO_RSVD)
9145                 priority = dev_conf->flow_prio - 1;
9146         /* number of actions must be set to 0 in case of dirty stack. */
9147         mhdr_res->actions_num = 0;
9148         for (; !actions_end ; actions++) {
9149                 const struct rte_flow_action_queue *queue;
9150                 const struct rte_flow_action_rss *rss;
9151                 const struct rte_flow_action *action = actions;
9152                 const uint8_t *rss_key;
9153                 const struct rte_flow_action_meter *mtr;
9154                 struct mlx5_flow_tbl_resource *tbl;
9155                 uint32_t port_id = 0;
9156                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
9157                 int action_type = actions->type;
9158                 const struct rte_flow_action *found_action = NULL;
9159                 struct mlx5_flow_meter *fm = NULL;
9160                 uint32_t jump_group = 0;
9161
9162                 if (!mlx5_flow_os_action_supported(action_type))
9163                         return rte_flow_error_set(error, ENOTSUP,
9164                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9165                                                   actions,
9166                                                   "action not supported");
9167                 switch (action_type) {
9168                 case RTE_FLOW_ACTION_TYPE_VOID:
9169                         break;
9170                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
9171                         if (flow_dv_translate_action_port_id(dev, action,
9172                                                              &port_id, error))
9173                                 return -rte_errno;
9174                         port_id_resource.port_id = port_id;
9175                         MLX5_ASSERT(!handle->rix_port_id_action);
9176                         if (flow_dv_port_id_action_resource_register
9177                             (dev, &port_id_resource, dev_flow, error))
9178                                 return -rte_errno;
9179                         dev_flow->dv.actions[actions_n++] =
9180                                         dev_flow->dv.port_id_action->action;
9181                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9182                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
9183                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9184                         num_of_dest++;
9185                         break;
9186                 case RTE_FLOW_ACTION_TYPE_FLAG:
9187                         action_flags |= MLX5_FLOW_ACTION_FLAG;
9188                         dev_flow->handle->mark = 1;
9189                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
9190                                 struct rte_flow_action_mark mark = {
9191                                         .id = MLX5_FLOW_MARK_DEFAULT,
9192                                 };
9193
9194                                 if (flow_dv_convert_action_mark(dev, &mark,
9195                                                                 mhdr_res,
9196                                                                 error))
9197                                         return -rte_errno;
9198                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
9199                                 break;
9200                         }
9201                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
9202                         /*
9203                          * Only one FLAG or MARK is supported per device flow
9204                          * right now. So the pointer to the tag resource must be
9205                          * zero before the register process.
9206                          */
9207                         MLX5_ASSERT(!handle->dvh.rix_tag);
9208                         if (flow_dv_tag_resource_register(dev, tag_be,
9209                                                           dev_flow, error))
9210                                 return -rte_errno;
9211                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9212                         dev_flow->dv.actions[actions_n++] =
9213                                         dev_flow->dv.tag_resource->action;
9214                         break;
9215                 case RTE_FLOW_ACTION_TYPE_MARK:
9216                         action_flags |= MLX5_FLOW_ACTION_MARK;
9217                         dev_flow->handle->mark = 1;
9218                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
9219                                 const struct rte_flow_action_mark *mark =
9220                                         (const struct rte_flow_action_mark *)
9221                                                 actions->conf;
9222
9223                                 if (flow_dv_convert_action_mark(dev, mark,
9224                                                                 mhdr_res,
9225                                                                 error))
9226                                         return -rte_errno;
9227                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
9228                                 break;
9229                         }
9230                         /* Fall-through */
9231                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
9232                         /* Legacy (non-extensive) MARK action. */
9233                         tag_be = mlx5_flow_mark_set
9234                               (((const struct rte_flow_action_mark *)
9235                                (actions->conf))->id);
9236                         MLX5_ASSERT(!handle->dvh.rix_tag);
9237                         if (flow_dv_tag_resource_register(dev, tag_be,
9238                                                           dev_flow, error))
9239                                 return -rte_errno;
9240                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9241                         dev_flow->dv.actions[actions_n++] =
9242                                         dev_flow->dv.tag_resource->action;
9243                         break;
9244                 case RTE_FLOW_ACTION_TYPE_SET_META:
9245                         if (flow_dv_convert_action_set_meta
9246                                 (dev, mhdr_res, attr,
9247                                  (const struct rte_flow_action_set_meta *)
9248                                   actions->conf, error))
9249                                 return -rte_errno;
9250                         action_flags |= MLX5_FLOW_ACTION_SET_META;
9251                         break;
9252                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
9253                         if (flow_dv_convert_action_set_tag
9254                                 (dev, mhdr_res,
9255                                  (const struct rte_flow_action_set_tag *)
9256                                   actions->conf, error))
9257                                 return -rte_errno;
9258                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9259                         break;
9260                 case RTE_FLOW_ACTION_TYPE_DROP:
9261                         action_flags |= MLX5_FLOW_ACTION_DROP;
9262                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
9263                         break;
9264                 case RTE_FLOW_ACTION_TYPE_QUEUE:
9265                         queue = actions->conf;
9266                         rss_desc->queue_num = 1;
9267                         rss_desc->queue[0] = queue->index;
9268                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
9269                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9270                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
9271                         num_of_dest++;
9272                         break;
9273                 case RTE_FLOW_ACTION_TYPE_RSS:
9274                         rss = actions->conf;
9275                         memcpy(rss_desc->queue, rss->queue,
9276                                rss->queue_num * sizeof(uint16_t));
9277                         rss_desc->queue_num = rss->queue_num;
9278                         /* NULL RSS key indicates default RSS key. */
9279                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
9280                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
9281                         /*
9282                          * rss->level and rss.types should be set in advance
9283                          * when expanding items for RSS.
9284                          */
9285                         action_flags |= MLX5_FLOW_ACTION_RSS;
9286                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9287                         break;
9288                 case RTE_FLOW_ACTION_TYPE_AGE:
9289                 case RTE_FLOW_ACTION_TYPE_COUNT:
9290                         if (!dev_conf->devx) {
9291                                 return rte_flow_error_set
9292                                               (error, ENOTSUP,
9293                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9294                                                NULL,
9295                                                "count action not supported");
9296                         }
9297                         /* Save information first, will apply later. */
9298                         if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT)
9299                                 count = action->conf;
9300                         else
9301                                 age = action->conf;
9302                         action_flags |= MLX5_FLOW_ACTION_COUNT;
9303                         break;
9304                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
9305                         dev_flow->dv.actions[actions_n++] =
9306                                                 priv->sh->pop_vlan_action;
9307                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
9308                         break;
9309                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
9310                         if (!(action_flags &
9311                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
9312                                 flow_dev_get_vlan_info_from_items(items, &vlan);
9313                         vlan.eth_proto = rte_be_to_cpu_16
9314                              ((((const struct rte_flow_action_of_push_vlan *)
9315                                                    actions->conf)->ethertype));
9316                         found_action = mlx5_flow_find_action
9317                                         (actions + 1,
9318                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
9319                         if (found_action)
9320                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
9321                         found_action = mlx5_flow_find_action
9322                                         (actions + 1,
9323                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
9324                         if (found_action)
9325                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
9326                         if (flow_dv_create_action_push_vlan
9327                                             (dev, attr, &vlan, dev_flow, error))
9328                                 return -rte_errno;
9329                         dev_flow->dv.actions[actions_n++] =
9330                                         dev_flow->dv.push_vlan_res->action;
9331                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
9332                         break;
9333                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
9334                         /* of_vlan_push action handled this action */
9335                         MLX5_ASSERT(action_flags &
9336                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
9337                         break;
9338                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
9339                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
9340                                 break;
9341                         flow_dev_get_vlan_info_from_items(items, &vlan);
9342                         mlx5_update_vlan_vid_pcp(actions, &vlan);
9343                         /* If no VLAN push - this is a modify header action */
9344                         if (flow_dv_convert_action_modify_vlan_vid
9345                                                 (mhdr_res, actions, error))
9346                                 return -rte_errno;
9347                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
9348                         break;
9349                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
9350                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
9351                         if (flow_dv_create_action_l2_encap(dev, actions,
9352                                                            dev_flow,
9353                                                            attr->transfer,
9354                                                            error))
9355                                 return -rte_errno;
9356                         dev_flow->dv.actions[actions_n++] =
9357                                         dev_flow->dv.encap_decap->action;
9358                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9359                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
9360                                 sample_act->action_flags |=
9361                                                         MLX5_FLOW_ACTION_ENCAP;
9362                         break;
9363                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
9364                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
9365                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
9366                                                            attr->transfer,
9367                                                            error))
9368                                 return -rte_errno;
9369                         dev_flow->dv.actions[actions_n++] =
9370                                         dev_flow->dv.encap_decap->action;
9371                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9372                         break;
9373                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
9374                         /* Handle encap with preceding decap. */
9375                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
9376                                 if (flow_dv_create_action_raw_encap
9377                                         (dev, actions, dev_flow, attr, error))
9378                                         return -rte_errno;
9379                                 dev_flow->dv.actions[actions_n++] =
9380                                         dev_flow->dv.encap_decap->action;
9381                         } else {
9382                                 /* Handle encap without preceding decap. */
9383                                 if (flow_dv_create_action_l2_encap
9384                                     (dev, actions, dev_flow, attr->transfer,
9385                                      error))
9386                                         return -rte_errno;
9387                                 dev_flow->dv.actions[actions_n++] =
9388                                         dev_flow->dv.encap_decap->action;
9389                         }
9390                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9391                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
9392                                 sample_act->action_flags |=
9393                                                         MLX5_FLOW_ACTION_ENCAP;
9394                         break;
9395                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
9396                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
9397                                 ;
9398                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
9399                                 if (flow_dv_create_action_l2_decap
9400                                     (dev, dev_flow, attr->transfer, error))
9401                                         return -rte_errno;
9402                                 dev_flow->dv.actions[actions_n++] =
9403                                         dev_flow->dv.encap_decap->action;
9404                         }
9405                         /* If decap is followed by encap, handle it at encap. */
9406                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9407                         break;
9408                 case RTE_FLOW_ACTION_TYPE_JUMP:
9409                         jump_group = ((const struct rte_flow_action_jump *)
9410                                                         action->conf)->group;
9411                         if (dev_flow->external && jump_group <
9412                                         MLX5_MAX_TABLES_EXTERNAL)
9413                                 jump_group *= MLX5_FLOW_TABLE_FACTOR;
9414                         ret = mlx5_flow_group_to_table(attr, dev_flow->external,
9415                                                        jump_group,
9416                                                        !!priv->fdb_def_rule,
9417                                                        &table, error);
9418                         if (ret)
9419                                 return ret;
9420                         tbl = flow_dv_tbl_resource_get(dev, table,
9421                                                        attr->egress,
9422                                                        attr->transfer, error);
9423                         if (!tbl)
9424                                 return rte_flow_error_set
9425                                                 (error, errno,
9426                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9427                                                  NULL,
9428                                                  "cannot create jump action.");
9429                         if (flow_dv_jump_tbl_resource_register
9430                             (dev, tbl, dev_flow, error)) {
9431                                 flow_dv_tbl_resource_release(dev, tbl);
9432                                 return rte_flow_error_set
9433                                                 (error, errno,
9434                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9435                                                  NULL,
9436                                                  "cannot create jump action.");
9437                         }
9438                         dev_flow->dv.actions[actions_n++] =
9439                                         dev_flow->dv.jump->action;
9440                         action_flags |= MLX5_FLOW_ACTION_JUMP;
9441                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
9442                         break;
9443                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
9444                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
9445                         if (flow_dv_convert_action_modify_mac
9446                                         (mhdr_res, actions, error))
9447                                 return -rte_errno;
9448                         action_flags |= actions->type ==
9449                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
9450                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
9451                                         MLX5_FLOW_ACTION_SET_MAC_DST;
9452                         break;
9453                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
9454                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
9455                         if (flow_dv_convert_action_modify_ipv4
9456                                         (mhdr_res, actions, error))
9457                                 return -rte_errno;
9458                         action_flags |= actions->type ==
9459                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
9460                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
9461                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
9462                         break;
9463                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
9464                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
9465                         if (flow_dv_convert_action_modify_ipv6
9466                                         (mhdr_res, actions, error))
9467                                 return -rte_errno;
9468                         action_flags |= actions->type ==
9469                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
9470                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
9471                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
9472                         break;
9473                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
9474                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
9475                         if (flow_dv_convert_action_modify_tp
9476                                         (mhdr_res, actions, items,
9477                                          &flow_attr, dev_flow, !!(action_flags &
9478                                          MLX5_FLOW_ACTION_DECAP), error))
9479                                 return -rte_errno;
9480                         action_flags |= actions->type ==
9481                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
9482                                         MLX5_FLOW_ACTION_SET_TP_SRC :
9483                                         MLX5_FLOW_ACTION_SET_TP_DST;
9484                         break;
9485                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
9486                         if (flow_dv_convert_action_modify_dec_ttl
9487                                         (mhdr_res, items, &flow_attr, dev_flow,
9488                                          !!(action_flags &
9489                                          MLX5_FLOW_ACTION_DECAP), error))
9490                                 return -rte_errno;
9491                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
9492                         break;
9493                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
9494                         if (flow_dv_convert_action_modify_ttl
9495                                         (mhdr_res, actions, items, &flow_attr,
9496                                          dev_flow, !!(action_flags &
9497                                          MLX5_FLOW_ACTION_DECAP), error))
9498                                 return -rte_errno;
9499                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
9500                         break;
9501                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
9502                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
9503                         if (flow_dv_convert_action_modify_tcp_seq
9504                                         (mhdr_res, actions, error))
9505                                 return -rte_errno;
9506                         action_flags |= actions->type ==
9507                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
9508                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
9509                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
9510                         break;
9511
9512                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
9513                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
9514                         if (flow_dv_convert_action_modify_tcp_ack
9515                                         (mhdr_res, actions, error))
9516                                 return -rte_errno;
9517                         action_flags |= actions->type ==
9518                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
9519                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
9520                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
9521                         break;
9522                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
9523                         if (flow_dv_convert_action_set_reg
9524                                         (mhdr_res, actions, error))
9525                                 return -rte_errno;
9526                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9527                         break;
9528                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
9529                         if (flow_dv_convert_action_copy_mreg
9530                                         (dev, mhdr_res, actions, error))
9531                                 return -rte_errno;
9532                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9533                         break;
9534                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
9535                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
9536                         dev_flow->handle->fate_action =
9537                                         MLX5_FLOW_FATE_DEFAULT_MISS;
9538                         break;
9539                 case RTE_FLOW_ACTION_TYPE_METER:
9540                         mtr = actions->conf;
9541                         if (!flow->meter) {
9542                                 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
9543                                                             attr, error);
9544                                 if (!fm)
9545                                         return rte_flow_error_set(error,
9546                                                 rte_errno,
9547                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9548                                                 NULL,
9549                                                 "meter not found "
9550                                                 "or invalid parameters");
9551                                 flow->meter = fm->idx;
9552                         }
9553                         /* Set the meter action. */
9554                         if (!fm) {
9555                                 fm = mlx5_ipool_get(priv->sh->ipool
9556                                                 [MLX5_IPOOL_MTR], flow->meter);
9557                                 if (!fm)
9558                                         return rte_flow_error_set(error,
9559                                                 rte_errno,
9560                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9561                                                 NULL,
9562                                                 "meter not found "
9563                                                 "or invalid parameters");
9564                         }
9565                         dev_flow->dv.actions[actions_n++] =
9566                                 fm->mfts->meter_action;
9567                         action_flags |= MLX5_FLOW_ACTION_METER;
9568                         break;
9569                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
9570                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
9571                                                               actions, error))
9572                                 return -rte_errno;
9573                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
9574                         break;
9575                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
9576                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
9577                                                               actions, error))
9578                                 return -rte_errno;
9579                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
9580                         break;
9581                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
9582                         sample_act_pos = actions_n;
9583                         ret = flow_dv_translate_action_sample(dev,
9584                                                               actions,
9585                                                               dev_flow, attr,
9586                                                               &num_of_dest,
9587                                                               sample_actions,
9588                                                               &sample_res,
9589                                                               error);
9590                         if (ret < 0)
9591                                 return ret;
9592                         actions_n++;
9593                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
9594                         /* put encap action into group if work with port id */
9595                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
9596                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
9597                                 sample_act->action_flags |=
9598                                                         MLX5_FLOW_ACTION_ENCAP;
9599                         break;
9600                 case RTE_FLOW_ACTION_TYPE_END:
9601                         actions_end = true;
9602                         if (mhdr_res->actions_num) {
9603                                 /* create modify action if needed. */
9604                                 if (flow_dv_modify_hdr_resource_register
9605                                         (dev, mhdr_res, dev_flow, error))
9606                                         return -rte_errno;
9607                                 dev_flow->dv.actions[modify_action_position] =
9608                                         handle->dvh.modify_hdr->action;
9609                         }
9610                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
9611                                 flow->counter =
9612                                         flow_dv_translate_create_counter(dev,
9613                                                 dev_flow, count, age);
9614
9615                                 if (!flow->counter)
9616                                         return rte_flow_error_set
9617                                                 (error, rte_errno,
9618                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9619                                                 NULL,
9620                                                 "cannot create counter"
9621                                                 " object.");
9622                                 dev_flow->dv.actions[actions_n] =
9623                                           (flow_dv_counter_get_by_idx(dev,
9624                                           flow->counter, NULL))->action;
9625                                 actions_n++;
9626                         }
9627                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
9628                                 ret = flow_dv_create_action_sample(dev,
9629                                                           dev_flow, attr,
9630                                                           num_of_dest,
9631                                                           &sample_res,
9632                                                           &mdest_res,
9633                                                           sample_actions,
9634                                                           action_flags,
9635                                                           error);
9636                                 if (ret < 0)
9637                                         return rte_flow_error_set
9638                                                 (error, rte_errno,
9639                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9640                                                 NULL,
9641                                                 "cannot create sample action");
9642                                 if (num_of_dest > 1) {
9643                                         dev_flow->dv.actions[sample_act_pos] =
9644                                         dev_flow->dv.dest_array_res->action;
9645                                 } else {
9646                                         dev_flow->dv.actions[sample_act_pos] =
9647                                         dev_flow->dv.sample_res->verbs_action;
9648                                 }
9649                         }
9650                         break;
9651                 default:
9652                         break;
9653                 }
9654                 if (mhdr_res->actions_num &&
9655                     modify_action_position == UINT32_MAX)
9656                         modify_action_position = actions_n++;
9657         }
9658         /*
9659          * For multiple destination (sample action with ratio=1), the encap
9660          * action and port id action will be combined into group action.
9661          * So need remove the original these actions in the flow and only
9662          * use the sample action instead of.
9663          */
9664         if (num_of_dest > 1 && sample_act->dr_port_id_action) {
9665                 int i;
9666                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
9667
9668                 for (i = 0; i < actions_n; i++) {
9669                         if ((sample_act->dr_encap_action &&
9670                                 sample_act->dr_encap_action ==
9671                                 dev_flow->dv.actions[i]) ||
9672                                 (sample_act->dr_port_id_action &&
9673                                 sample_act->dr_port_id_action ==
9674                                 dev_flow->dv.actions[i]))
9675                                 continue;
9676                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
9677                 }
9678                 memcpy((void *)dev_flow->dv.actions,
9679                                 (void *)temp_actions,
9680                                 tmp_actions_n * sizeof(void *));
9681                 actions_n = tmp_actions_n;
9682         }
9683         dev_flow->dv.actions_n = actions_n;
9684         dev_flow->act_flags = action_flags;
9685         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
9686                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
9687                 int item_type = items->type;
9688
9689                 if (!mlx5_flow_os_item_supported(item_type))
9690                         return rte_flow_error_set(error, ENOTSUP,
9691                                                   RTE_FLOW_ERROR_TYPE_ITEM,
9692                                                   NULL, "item not supported");
9693                 switch (item_type) {
9694                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
9695                         flow_dv_translate_item_port_id(dev, match_mask,
9696                                                        match_value, items);
9697                         last_item = MLX5_FLOW_ITEM_PORT_ID;
9698                         break;
9699                 case RTE_FLOW_ITEM_TYPE_ETH:
9700                         flow_dv_translate_item_eth(match_mask, match_value,
9701                                                    items, tunnel,
9702                                                    dev_flow->dv.group);
9703                         matcher.priority = action_flags &
9704                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
9705                                         !dev_flow->external ?
9706                                         MLX5_PRIORITY_MAP_L3 :
9707                                         MLX5_PRIORITY_MAP_L2;
9708                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
9709                                              MLX5_FLOW_LAYER_OUTER_L2;
9710                         break;
9711                 case RTE_FLOW_ITEM_TYPE_VLAN:
9712                         flow_dv_translate_item_vlan(dev_flow,
9713                                                     match_mask, match_value,
9714                                                     items, tunnel,
9715                                                     dev_flow->dv.group);
9716                         matcher.priority = MLX5_PRIORITY_MAP_L2;
9717                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
9718                                               MLX5_FLOW_LAYER_INNER_VLAN) :
9719                                              (MLX5_FLOW_LAYER_OUTER_L2 |
9720                                               MLX5_FLOW_LAYER_OUTER_VLAN);
9721                         break;
9722                 case RTE_FLOW_ITEM_TYPE_IPV4:
9723                         mlx5_flow_tunnel_ip_check(items, next_protocol,
9724                                                   &item_flags, &tunnel);
9725                         flow_dv_translate_item_ipv4(match_mask, match_value,
9726                                                     items, item_flags, tunnel,
9727                                                     dev_flow->dv.group);
9728                         matcher.priority = MLX5_PRIORITY_MAP_L3;
9729                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
9730                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
9731                         if (items->mask != NULL &&
9732                             ((const struct rte_flow_item_ipv4 *)
9733                              items->mask)->hdr.next_proto_id) {
9734                                 next_protocol =
9735                                         ((const struct rte_flow_item_ipv4 *)
9736                                          (items->spec))->hdr.next_proto_id;
9737                                 next_protocol &=
9738                                         ((const struct rte_flow_item_ipv4 *)
9739                                          (items->mask))->hdr.next_proto_id;
9740                         } else {
9741                                 /* Reset for inner layer. */
9742                                 next_protocol = 0xff;
9743                         }
9744                         break;
9745                 case RTE_FLOW_ITEM_TYPE_IPV6:
9746                         mlx5_flow_tunnel_ip_check(items, next_protocol,
9747                                                   &item_flags, &tunnel);
9748                         flow_dv_translate_item_ipv6(match_mask, match_value,
9749                                                     items, item_flags, tunnel,
9750                                                     dev_flow->dv.group);
9751                         matcher.priority = MLX5_PRIORITY_MAP_L3;
9752                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
9753                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
9754                         if (items->mask != NULL &&
9755                             ((const struct rte_flow_item_ipv6 *)
9756                              items->mask)->hdr.proto) {
9757                                 next_protocol =
9758                                         ((const struct rte_flow_item_ipv6 *)
9759                                          items->spec)->hdr.proto;
9760                                 next_protocol &=
9761                                         ((const struct rte_flow_item_ipv6 *)
9762                                          items->mask)->hdr.proto;
9763                         } else {
9764                                 /* Reset for inner layer. */
9765                                 next_protocol = 0xff;
9766                         }
9767                         break;
9768                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
9769                         flow_dv_translate_item_ipv6_frag_ext(match_mask,
9770                                                              match_value,
9771                                                              items, tunnel);
9772                         last_item = tunnel ?
9773                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
9774                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
9775                         if (items->mask != NULL &&
9776                             ((const struct rte_flow_item_ipv6_frag_ext *)
9777                              items->mask)->hdr.next_header) {
9778                                 next_protocol =
9779                                 ((const struct rte_flow_item_ipv6_frag_ext *)
9780                                  items->spec)->hdr.next_header;
9781                                 next_protocol &=
9782                                 ((const struct rte_flow_item_ipv6_frag_ext *)
9783                                  items->mask)->hdr.next_header;
9784                         } else {
9785                                 /* Reset for inner layer. */
9786                                 next_protocol = 0xff;
9787                         }
9788                         break;
9789                 case RTE_FLOW_ITEM_TYPE_TCP:
9790                         flow_dv_translate_item_tcp(match_mask, match_value,
9791                                                    items, tunnel);
9792                         matcher.priority = MLX5_PRIORITY_MAP_L4;
9793                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
9794                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
9795                         break;
9796                 case RTE_FLOW_ITEM_TYPE_UDP:
9797                         flow_dv_translate_item_udp(match_mask, match_value,
9798                                                    items, tunnel);
9799                         matcher.priority = MLX5_PRIORITY_MAP_L4;
9800                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
9801                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
9802                         break;
9803                 case RTE_FLOW_ITEM_TYPE_GRE:
9804                         flow_dv_translate_item_gre(match_mask, match_value,
9805                                                    items, tunnel);
9806                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9807                         last_item = MLX5_FLOW_LAYER_GRE;
9808                         break;
9809                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
9810                         flow_dv_translate_item_gre_key(match_mask,
9811                                                        match_value, items);
9812                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
9813                         break;
9814                 case RTE_FLOW_ITEM_TYPE_NVGRE:
9815                         flow_dv_translate_item_nvgre(match_mask, match_value,
9816                                                      items, tunnel);
9817                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9818                         last_item = MLX5_FLOW_LAYER_GRE;
9819                         break;
9820                 case RTE_FLOW_ITEM_TYPE_VXLAN:
9821                         flow_dv_translate_item_vxlan(match_mask, match_value,
9822                                                      items, tunnel);
9823                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9824                         last_item = MLX5_FLOW_LAYER_VXLAN;
9825                         break;
9826                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
9827                         flow_dv_translate_item_vxlan_gpe(match_mask,
9828                                                          match_value, items,
9829                                                          tunnel);
9830                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9831                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
9832                         break;
9833                 case RTE_FLOW_ITEM_TYPE_GENEVE:
9834                         flow_dv_translate_item_geneve(match_mask, match_value,
9835                                                       items, tunnel);
9836                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9837                         last_item = MLX5_FLOW_LAYER_GENEVE;
9838                         break;
9839                 case RTE_FLOW_ITEM_TYPE_MPLS:
9840                         flow_dv_translate_item_mpls(match_mask, match_value,
9841                                                     items, last_item, tunnel);
9842                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9843                         last_item = MLX5_FLOW_LAYER_MPLS;
9844                         break;
9845                 case RTE_FLOW_ITEM_TYPE_MARK:
9846                         flow_dv_translate_item_mark(dev, match_mask,
9847                                                     match_value, items);
9848                         last_item = MLX5_FLOW_ITEM_MARK;
9849                         break;
9850                 case RTE_FLOW_ITEM_TYPE_META:
9851                         flow_dv_translate_item_meta(dev, match_mask,
9852                                                     match_value, attr, items);
9853                         last_item = MLX5_FLOW_ITEM_METADATA;
9854                         break;
9855                 case RTE_FLOW_ITEM_TYPE_ICMP:
9856                         flow_dv_translate_item_icmp(match_mask, match_value,
9857                                                     items, tunnel);
9858                         last_item = MLX5_FLOW_LAYER_ICMP;
9859                         break;
9860                 case RTE_FLOW_ITEM_TYPE_ICMP6:
9861                         flow_dv_translate_item_icmp6(match_mask, match_value,
9862                                                       items, tunnel);
9863                         last_item = MLX5_FLOW_LAYER_ICMP6;
9864                         break;
9865                 case RTE_FLOW_ITEM_TYPE_TAG:
9866                         flow_dv_translate_item_tag(dev, match_mask,
9867                                                    match_value, items);
9868                         last_item = MLX5_FLOW_ITEM_TAG;
9869                         break;
9870                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
9871                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
9872                                                         match_value, items);
9873                         last_item = MLX5_FLOW_ITEM_TAG;
9874                         break;
9875                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
9876                         flow_dv_translate_item_tx_queue(dev, match_mask,
9877                                                         match_value,
9878                                                         items);
9879                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
9880                         break;
9881                 case RTE_FLOW_ITEM_TYPE_GTP:
9882                         flow_dv_translate_item_gtp(match_mask, match_value,
9883                                                    items, tunnel);
9884                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9885                         last_item = MLX5_FLOW_LAYER_GTP;
9886                         break;
9887                 case RTE_FLOW_ITEM_TYPE_ECPRI:
9888                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
9889                                 /* Create it only the first time to be used. */
9890                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
9891                                 if (ret)
9892                                         return rte_flow_error_set
9893                                                 (error, -ret,
9894                                                 RTE_FLOW_ERROR_TYPE_ITEM,
9895                                                 NULL,
9896                                                 "cannot create eCPRI parser");
9897                         }
9898                         /* Adjust the length matcher and device flow value. */
9899                         matcher.mask.size = MLX5_ST_SZ_BYTES(fte_match_param);
9900                         dev_flow->dv.value.size =
9901                                         MLX5_ST_SZ_BYTES(fte_match_param);
9902                         flow_dv_translate_item_ecpri(dev, match_mask,
9903                                                      match_value, items);
9904                         /* No other protocol should follow eCPRI layer. */
9905                         last_item = MLX5_FLOW_LAYER_ECPRI;
9906                         break;
9907                 default:
9908                         break;
9909                 }
9910                 item_flags |= last_item;
9911         }
9912         /*
9913          * When E-Switch mode is enabled, we have two cases where we need to
9914          * set the source port manually.
9915          * The first one, is in case of Nic steering rule, and the second is
9916          * E-Switch rule where no port_id item was found. In both cases
9917          * the source port is set according the current port in use.
9918          */
9919         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
9920             (priv->representor || priv->master)) {
9921                 if (flow_dv_translate_item_port_id(dev, match_mask,
9922                                                    match_value, NULL))
9923                         return -rte_errno;
9924         }
9925 #ifdef RTE_LIBRTE_MLX5_DEBUG
9926         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
9927                                               dev_flow->dv.value.buf));
9928 #endif
9929         /*
9930          * Layers may be already initialized from prefix flow if this dev_flow
9931          * is the suffix flow.
9932          */
9933         handle->layers |= item_flags;
9934         if (action_flags & MLX5_FLOW_ACTION_RSS)
9935                 flow_dv_hashfields_set(dev_flow, rss_desc);
9936         /* Register matcher. */
9937         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
9938                                     matcher.mask.size);
9939         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
9940                                                      matcher.priority);
9941         /* reserved field no needs to be set to 0 here. */
9942         tbl_key.domain = attr->transfer;
9943         tbl_key.direction = attr->egress;
9944         tbl_key.table_id = dev_flow->dv.group;
9945         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
9946                 return -rte_errno;
9947         return 0;
9948 }
9949
9950 /**
9951  * Apply the flow to the NIC, lock free,
9952  * (mutex should be acquired by caller).
9953  *
9954  * @param[in] dev
9955  *   Pointer to the Ethernet device structure.
9956  * @param[in, out] flow
9957  *   Pointer to flow structure.
9958  * @param[out] error
9959  *   Pointer to error structure.
9960  *
9961  * @return
9962  *   0 on success, a negative errno value otherwise and rte_errno is set.
9963  */
9964 static int
9965 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
9966                 struct rte_flow_error *error)
9967 {
9968         struct mlx5_flow_dv_workspace *dv;
9969         struct mlx5_flow_handle *dh;
9970         struct mlx5_flow_handle_dv *dv_h;
9971         struct mlx5_flow *dev_flow;
9972         struct mlx5_priv *priv = dev->data->dev_private;
9973         uint32_t handle_idx;
9974         int n;
9975         int err;
9976         int idx;
9977
9978         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
9979                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
9980                 dv = &dev_flow->dv;
9981                 dh = dev_flow->handle;
9982                 dv_h = &dh->dvh;
9983                 n = dv->actions_n;
9984                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
9985                         if (dv->transfer) {
9986                                 dv->actions[n++] = priv->sh->esw_drop_action;
9987                         } else {
9988                                 struct mlx5_hrxq *drop_hrxq;
9989                                 drop_hrxq = mlx5_drop_action_create(dev);
9990                                 if (!drop_hrxq) {
9991                                         rte_flow_error_set
9992                                                 (error, errno,
9993                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9994                                                  NULL,
9995                                                  "cannot get drop hash queue");
9996                                         goto error;
9997                                 }
9998                                 /*
9999                                  * Drop queues will be released by the specify
10000                                  * mlx5_drop_action_destroy() function. Assign
10001                                  * the special index to hrxq to mark the queue
10002                                  * has been allocated.
10003                                  */
10004                                 dh->rix_hrxq = UINT32_MAX;
10005                                 dv->actions[n++] = drop_hrxq->action;
10006                         }
10007                 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
10008                            !dv_h->rix_sample && !dv_h->rix_dest_array) {
10009                         struct mlx5_hrxq *hrxq;
10010                         uint32_t hrxq_idx;
10011                         struct mlx5_flow_rss_desc *rss_desc =
10012                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
10013                                 [!!priv->flow_nested_idx];
10014
10015                         MLX5_ASSERT(rss_desc->queue_num);
10016                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
10017                                                  MLX5_RSS_HASH_KEY_LEN,
10018                                                  dev_flow->hash_fields,
10019                                                  rss_desc->queue,
10020                                                  rss_desc->queue_num);
10021                         if (!hrxq_idx) {
10022                                 hrxq_idx = mlx5_hrxq_new
10023                                                 (dev, rss_desc->key,
10024                                                  MLX5_RSS_HASH_KEY_LEN,
10025                                                  dev_flow->hash_fields,
10026                                                  rss_desc->queue,
10027                                                  rss_desc->queue_num,
10028                                                  !!(dh->layers &
10029                                                  MLX5_FLOW_LAYER_TUNNEL));
10030                         }
10031                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
10032                                               hrxq_idx);
10033                         if (!hrxq) {
10034                                 rte_flow_error_set
10035                                         (error, rte_errno,
10036                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10037                                          "cannot get hash queue");
10038                                 goto error;
10039                         }
10040                         dh->rix_hrxq = hrxq_idx;
10041                         dv->actions[n++] = hrxq->action;
10042                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
10043                         if (flow_dv_default_miss_resource_register
10044                                         (dev, error)) {
10045                                 rte_flow_error_set
10046                                         (error, rte_errno,
10047                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10048                                          "cannot create default miss resource");
10049                                 goto error_default_miss;
10050                         }
10051                         dh->rix_default_fate =  MLX5_FLOW_FATE_DEFAULT_MISS;
10052                         dv->actions[n++] = priv->sh->default_miss.action;
10053                 }
10054                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
10055                                                (void *)&dv->value, n,
10056                                                dv->actions, &dh->drv_flow);
10057                 if (err) {
10058                         rte_flow_error_set(error, errno,
10059                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10060                                            NULL,
10061                                            "hardware refuses to create flow");
10062                         goto error;
10063                 }
10064                 if (priv->vmwa_context &&
10065                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
10066                         /*
10067                          * The rule contains the VLAN pattern.
10068                          * For VF we are going to create VLAN
10069                          * interface to make hypervisor set correct
10070                          * e-Switch vport context.
10071                          */
10072                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
10073                 }
10074         }
10075         return 0;
10076 error:
10077         if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
10078                 flow_dv_default_miss_resource_release(dev);
10079 error_default_miss:
10080         err = rte_errno; /* Save rte_errno before cleanup. */
10081         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
10082                        handle_idx, dh, next) {
10083                 /* hrxq is union, don't clear it if the flag is not set. */
10084                 if (dh->rix_hrxq) {
10085                         if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
10086                                 mlx5_drop_action_destroy(dev);
10087                                 dh->rix_hrxq = 0;
10088                         } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
10089                                 mlx5_hrxq_release(dev, dh->rix_hrxq);
10090                                 dh->rix_hrxq = 0;
10091                         }
10092                 }
10093                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
10094                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
10095         }
10096         rte_errno = err; /* Restore rte_errno. */
10097         return -rte_errno;
10098 }
10099
10100 /**
10101  * Release the flow matcher.
10102  *
10103  * @param dev
10104  *   Pointer to Ethernet device.
10105  * @param handle
10106  *   Pointer to mlx5_flow_handle.
10107  *
10108  * @return
10109  *   1 while a reference on it exists, 0 when freed.
10110  */
10111 static int
10112 flow_dv_matcher_release(struct rte_eth_dev *dev,
10113                         struct mlx5_flow_handle *handle)
10114 {
10115         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
10116
10117         MLX5_ASSERT(matcher->matcher_object);
10118         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
10119                 dev->data->port_id, (void *)matcher,
10120                 rte_atomic32_read(&matcher->refcnt));
10121         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
10122                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10123                            (matcher->matcher_object));
10124                 LIST_REMOVE(matcher, next);
10125                 /* table ref-- in release interface. */
10126                 flow_dv_tbl_resource_release(dev, matcher->tbl);
10127                 mlx5_free(matcher);
10128                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
10129                         dev->data->port_id, (void *)matcher);
10130                 return 0;
10131         }
10132         return 1;
10133 }
10134
10135 /**
10136  * Release an encap/decap resource.
10137  *
10138  * @param dev
10139  *   Pointer to Ethernet device.
10140  * @param encap_decap_idx
10141  *   Index of encap decap resource.
10142  *
10143  * @return
10144  *   1 while a reference on it exists, 0 when freed.
10145  */
10146 static int
10147 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
10148                                      uint32_t encap_decap_idx)
10149 {
10150         struct mlx5_priv *priv = dev->data->dev_private;
10151         uint32_t idx = encap_decap_idx;
10152         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
10153
10154         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
10155                          idx);
10156         if (!cache_resource)
10157                 return 0;
10158         MLX5_ASSERT(cache_resource->action);
10159         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
10160                 (void *)cache_resource,
10161                 rte_atomic32_read(&cache_resource->refcnt));
10162         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10163                 claim_zero(mlx5_flow_os_destroy_flow_action
10164                                                 (cache_resource->action));
10165                 mlx5_hlist_remove(priv->sh->encaps_decaps,
10166                                   &cache_resource->entry);
10167                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
10168                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
10169                         (void *)cache_resource);
10170                 return 0;
10171         }
10172         return 1;
10173 }
10174
10175 /**
10176  * Release an jump to table action resource.
10177  *
10178  * @param dev
10179  *   Pointer to Ethernet device.
10180  * @param handle
10181  *   Pointer to mlx5_flow_handle.
10182  *
10183  * @return
10184  *   1 while a reference on it exists, 0 when freed.
10185  */
10186 static int
10187 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
10188                                   struct mlx5_flow_handle *handle)
10189 {
10190         struct mlx5_priv *priv = dev->data->dev_private;
10191         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
10192         struct mlx5_flow_tbl_data_entry *tbl_data;
10193
10194         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
10195                              handle->rix_jump);
10196         if (!tbl_data)
10197                 return 0;
10198         cache_resource = &tbl_data->jump;
10199         MLX5_ASSERT(cache_resource->action);
10200         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
10201                 (void *)cache_resource,
10202                 rte_atomic32_read(&cache_resource->refcnt));
10203         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10204                 claim_zero(mlx5_flow_os_destroy_flow_action
10205                                                 (cache_resource->action));
10206                 /* jump action memory free is inside the table release. */
10207                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
10208                 DRV_LOG(DEBUG, "jump table resource %p: removed",
10209                         (void *)cache_resource);
10210                 return 0;
10211         }
10212         return 1;
10213 }
10214
10215 /**
10216  * Release a default miss resource.
10217  *
10218  * @param dev
10219  *   Pointer to Ethernet device.
10220  * @return
10221  *   1 while a reference on it exists, 0 when freed.
10222  */
10223 static int
10224 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
10225 {
10226         struct mlx5_priv *priv = dev->data->dev_private;
10227         struct mlx5_dev_ctx_shared *sh = priv->sh;
10228         struct mlx5_flow_default_miss_resource *cache_resource =
10229                         &sh->default_miss;
10230
10231         MLX5_ASSERT(cache_resource->action);
10232         DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--",
10233                         (void *)cache_resource->action,
10234                         rte_atomic32_read(&cache_resource->refcnt));
10235         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10236                 claim_zero(mlx5_glue->destroy_flow_action
10237                                 (cache_resource->action));
10238                 DRV_LOG(DEBUG, "default miss resource %p: removed",
10239                                 (void *)cache_resource->action);
10240                 return 0;
10241         }
10242         return 1;
10243 }
10244
10245 /**
10246  * Release a modify-header resource.
10247  *
10248  * @param dev
10249  *   Pointer to Ethernet device.
10250  * @param handle
10251  *   Pointer to mlx5_flow_handle.
10252  *
10253  * @return
10254  *   1 while a reference on it exists, 0 when freed.
10255  */
10256 static int
10257 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
10258                                     struct mlx5_flow_handle *handle)
10259 {
10260         struct mlx5_priv *priv = dev->data->dev_private;
10261         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
10262                                                         handle->dvh.modify_hdr;
10263
10264         MLX5_ASSERT(cache_resource->action);
10265         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
10266                 (void *)cache_resource,
10267                 rte_atomic32_read(&cache_resource->refcnt));
10268         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10269                 claim_zero(mlx5_flow_os_destroy_flow_action
10270                                                 (cache_resource->action));
10271                 mlx5_hlist_remove(priv->sh->modify_cmds,
10272                                   &cache_resource->entry);
10273                 mlx5_free(cache_resource);
10274                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
10275                         (void *)cache_resource);
10276                 return 0;
10277         }
10278         return 1;
10279 }
10280
10281 /**
10282  * Release port ID action resource.
10283  *
10284  * @param dev
10285  *   Pointer to Ethernet device.
10286  * @param handle
10287  *   Pointer to mlx5_flow_handle.
10288  *
10289  * @return
10290  *   1 while a reference on it exists, 0 when freed.
10291  */
10292 static int
10293 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
10294                                         uint32_t port_id)
10295 {
10296         struct mlx5_priv *priv = dev->data->dev_private;
10297         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
10298         uint32_t idx = port_id;
10299
10300         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
10301                                         idx);
10302         if (!cache_resource)
10303                 return 0;
10304         MLX5_ASSERT(cache_resource->action);
10305         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
10306                 (void *)cache_resource,
10307                 rte_atomic32_read(&cache_resource->refcnt));
10308         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10309                 claim_zero(mlx5_flow_os_destroy_flow_action
10310                                                 (cache_resource->action));
10311                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
10312                              &priv->sh->port_id_action_list, idx,
10313                              cache_resource, next);
10314                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx);
10315                 DRV_LOG(DEBUG, "port id action resource %p: removed",
10316                         (void *)cache_resource);
10317                 return 0;
10318         }
10319         return 1;
10320 }
10321
10322 /**
10323  * Release push vlan action resource.
10324  *
10325  * @param dev
10326  *   Pointer to Ethernet device.
10327  * @param handle
10328  *   Pointer to mlx5_flow_handle.
10329  *
10330  * @return
10331  *   1 while a reference on it exists, 0 when freed.
10332  */
10333 static int
10334 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
10335                                           struct mlx5_flow_handle *handle)
10336 {
10337         struct mlx5_priv *priv = dev->data->dev_private;
10338         uint32_t idx = handle->dvh.rix_push_vlan;
10339         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
10340
10341         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
10342                                         idx);
10343         if (!cache_resource)
10344                 return 0;
10345         MLX5_ASSERT(cache_resource->action);
10346         DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
10347                 (void *)cache_resource,
10348                 rte_atomic32_read(&cache_resource->refcnt));
10349         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10350                 claim_zero(mlx5_flow_os_destroy_flow_action
10351                                                 (cache_resource->action));
10352                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
10353                              &priv->sh->push_vlan_action_list, idx,
10354                              cache_resource, next);
10355                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
10356                 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
10357                         (void *)cache_resource);
10358                 return 0;
10359         }
10360         return 1;
10361 }
10362
10363 /**
10364  * Release the fate resource.
10365  *
10366  * @param dev
10367  *   Pointer to Ethernet device.
10368  * @param handle
10369  *   Pointer to mlx5_flow_handle.
10370  */
10371 static void
10372 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
10373                                struct mlx5_flow_handle *handle)
10374 {
10375         if (!handle->rix_fate)
10376                 return;
10377         switch (handle->fate_action) {
10378         case MLX5_FLOW_FATE_DROP:
10379                 mlx5_drop_action_destroy(dev);
10380                 break;
10381         case MLX5_FLOW_FATE_QUEUE:
10382                 mlx5_hrxq_release(dev, handle->rix_hrxq);
10383                 break;
10384         case MLX5_FLOW_FATE_JUMP:
10385                 flow_dv_jump_tbl_resource_release(dev, handle);
10386                 break;
10387         case MLX5_FLOW_FATE_PORT_ID:
10388                 flow_dv_port_id_action_resource_release(dev,
10389                                 handle->rix_port_id_action);
10390                 break;
10391         case MLX5_FLOW_FATE_DEFAULT_MISS:
10392                 flow_dv_default_miss_resource_release(dev);
10393                 break;
10394         default:
10395                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
10396                 break;
10397         }
10398         handle->rix_fate = 0;
10399 }
10400
10401 /**
10402  * Release an sample resource.
10403  *
10404  * @param dev
10405  *   Pointer to Ethernet device.
10406  * @param handle
10407  *   Pointer to mlx5_flow_handle.
10408  *
10409  * @return
10410  *   1 while a reference on it exists, 0 when freed.
10411  */
10412 static int
10413 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
10414                                      struct mlx5_flow_handle *handle)
10415 {
10416         struct mlx5_priv *priv = dev->data->dev_private;
10417         uint32_t idx = handle->dvh.rix_sample;
10418         struct mlx5_flow_dv_sample_resource *cache_resource;
10419
10420         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
10421                          idx);
10422         if (!cache_resource)
10423                 return 0;
10424         MLX5_ASSERT(cache_resource->verbs_action);
10425         DRV_LOG(DEBUG, "sample resource %p: refcnt %d--",
10426                 (void *)cache_resource,
10427                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10428         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10429                                __ATOMIC_RELAXED) == 0) {
10430                 if (cache_resource->verbs_action)
10431                         claim_zero(mlx5_glue->destroy_flow_action
10432                                         (cache_resource->verbs_action));
10433                 if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
10434                         if (cache_resource->default_miss)
10435                                 claim_zero(mlx5_glue->destroy_flow_action
10436                                   (cache_resource->default_miss));
10437                 }
10438                 if (cache_resource->normal_path_tbl)
10439                         flow_dv_tbl_resource_release(dev,
10440                                 cache_resource->normal_path_tbl);
10441         }
10442         if (cache_resource->sample_idx.rix_hrxq &&
10443                 !mlx5_hrxq_release(dev,
10444                         cache_resource->sample_idx.rix_hrxq))
10445                 cache_resource->sample_idx.rix_hrxq = 0;
10446         if (cache_resource->sample_idx.rix_tag &&
10447                 !flow_dv_tag_release(dev,
10448                         cache_resource->sample_idx.rix_tag))
10449                 cache_resource->sample_idx.rix_tag = 0;
10450         if (cache_resource->sample_idx.cnt) {
10451                 flow_dv_counter_release(dev,
10452                         cache_resource->sample_idx.cnt);
10453                 cache_resource->sample_idx.cnt = 0;
10454         }
10455         if (!__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)) {
10456                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
10457                              &priv->sh->sample_action_list, idx,
10458                              cache_resource, next);
10459                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], idx);
10460                 DRV_LOG(DEBUG, "sample resource %p: removed",
10461                         (void *)cache_resource);
10462                 return 0;
10463         }
10464         return 1;
10465 }
10466
10467 /**
10468  * Release an destination array resource.
10469  *
10470  * @param dev
10471  *   Pointer to Ethernet device.
10472  * @param handle
10473  *   Pointer to mlx5_flow_handle.
10474  *
10475  * @return
10476  *   1 while a reference on it exists, 0 when freed.
10477  */
10478 static int
10479 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
10480                                      struct mlx5_flow_handle *handle)
10481 {
10482         struct mlx5_priv *priv = dev->data->dev_private;
10483         struct mlx5_flow_dv_dest_array_resource *cache_resource;
10484         struct mlx5_flow_sub_actions_idx *mdest_act_res;
10485         uint32_t idx = handle->dvh.rix_dest_array;
10486         uint32_t i = 0;
10487
10488         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10489                          idx);
10490         if (!cache_resource)
10491                 return 0;
10492         MLX5_ASSERT(cache_resource->action);
10493         DRV_LOG(DEBUG, "destination array resource %p: refcnt %d--",
10494                 (void *)cache_resource,
10495                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10496         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10497                                __ATOMIC_RELAXED) == 0) {
10498                 if (cache_resource->action)
10499                         claim_zero(mlx5_glue->destroy_flow_action
10500                                                 (cache_resource->action));
10501                 for (; i < cache_resource->num_of_dest; i++) {
10502                         mdest_act_res = &cache_resource->sample_idx[i];
10503                         if (mdest_act_res->rix_hrxq) {
10504                                 mlx5_hrxq_release(dev,
10505                                         mdest_act_res->rix_hrxq);
10506                                 mdest_act_res->rix_hrxq = 0;
10507                         }
10508                         if (mdest_act_res->rix_encap_decap) {
10509                                 flow_dv_encap_decap_resource_release(dev,
10510                                         mdest_act_res->rix_encap_decap);
10511                                 mdest_act_res->rix_encap_decap = 0;
10512                         }
10513                         if (mdest_act_res->rix_port_id_action) {
10514                                 flow_dv_port_id_action_resource_release(dev,
10515                                         mdest_act_res->rix_port_id_action);
10516                                 mdest_act_res->rix_port_id_action = 0;
10517                         }
10518                         if (mdest_act_res->rix_tag) {
10519                                 flow_dv_tag_release(dev,
10520                                         mdest_act_res->rix_tag);
10521                                 mdest_act_res->rix_tag = 0;
10522                         }
10523                 }
10524                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10525                              &priv->sh->dest_array_list, idx,
10526                              cache_resource, next);
10527                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], idx);
10528                 DRV_LOG(DEBUG, "destination array resource %p: removed",
10529                         (void *)cache_resource);
10530                 return 0;
10531         }
10532         return 1;
10533 }
10534
10535 /**
10536  * Remove the flow from the NIC but keeps it in memory.
10537  * Lock free, (mutex should be acquired by caller).
10538  *
10539  * @param[in] dev
10540  *   Pointer to Ethernet device.
10541  * @param[in, out] flow
10542  *   Pointer to flow structure.
10543  */
10544 static void
10545 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
10546 {
10547         struct mlx5_flow_handle *dh;
10548         uint32_t handle_idx;
10549         struct mlx5_priv *priv = dev->data->dev_private;
10550
10551         if (!flow)
10552                 return;
10553         handle_idx = flow->dev_handles;
10554         while (handle_idx) {
10555                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10556                                     handle_idx);
10557                 if (!dh)
10558                         return;
10559                 if (dh->drv_flow) {
10560                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
10561                         dh->drv_flow = NULL;
10562                 }
10563                 if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
10564                     dh->fate_action == MLX5_FLOW_FATE_QUEUE ||
10565                     dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
10566                         flow_dv_fate_resource_release(dev, dh);
10567                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
10568                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
10569                 handle_idx = dh->next.next;
10570         }
10571 }
10572
10573 /**
10574  * Remove the flow from the NIC and the memory.
10575  * Lock free, (mutex should be acquired by caller).
10576  *
10577  * @param[in] dev
10578  *   Pointer to the Ethernet device structure.
10579  * @param[in, out] flow
10580  *   Pointer to flow structure.
10581  */
10582 static void
10583 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
10584 {
10585         struct mlx5_flow_handle *dev_handle;
10586         struct mlx5_priv *priv = dev->data->dev_private;
10587
10588         if (!flow)
10589                 return;
10590         __flow_dv_remove(dev, flow);
10591         if (flow->counter) {
10592                 flow_dv_counter_release(dev, flow->counter);
10593                 flow->counter = 0;
10594         }
10595         if (flow->meter) {
10596                 struct mlx5_flow_meter *fm;
10597
10598                 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
10599                                     flow->meter);
10600                 if (fm)
10601                         mlx5_flow_meter_detach(fm);
10602                 flow->meter = 0;
10603         }
10604         while (flow->dev_handles) {
10605                 uint32_t tmp_idx = flow->dev_handles;
10606
10607                 dev_handle = mlx5_ipool_get(priv->sh->ipool
10608                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
10609                 if (!dev_handle)
10610                         return;
10611                 flow->dev_handles = dev_handle->next.next;
10612                 if (dev_handle->dvh.matcher)
10613                         flow_dv_matcher_release(dev, dev_handle);
10614                 if (dev_handle->dvh.rix_sample)
10615                         flow_dv_sample_resource_release(dev, dev_handle);
10616                 if (dev_handle->dvh.rix_dest_array)
10617                         flow_dv_dest_array_resource_release(dev, dev_handle);
10618                 if (dev_handle->dvh.rix_encap_decap)
10619                         flow_dv_encap_decap_resource_release(dev,
10620                                 dev_handle->dvh.rix_encap_decap);
10621                 if (dev_handle->dvh.modify_hdr)
10622                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
10623                 if (dev_handle->dvh.rix_push_vlan)
10624                         flow_dv_push_vlan_action_resource_release(dev,
10625                                                                   dev_handle);
10626                 if (dev_handle->dvh.rix_tag)
10627                         flow_dv_tag_release(dev,
10628                                             dev_handle->dvh.rix_tag);
10629                 flow_dv_fate_resource_release(dev, dev_handle);
10630                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10631                            tmp_idx);
10632         }
10633 }
10634
10635 /**
10636  * Query a dv flow  rule for its statistics via devx.
10637  *
10638  * @param[in] dev
10639  *   Pointer to Ethernet device.
10640  * @param[in] flow
10641  *   Pointer to the sub flow.
10642  * @param[out] data
10643  *   data retrieved by the query.
10644  * @param[out] error
10645  *   Perform verbose error reporting if not NULL.
10646  *
10647  * @return
10648  *   0 on success, a negative errno value otherwise and rte_errno is set.
10649  */
10650 static int
10651 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
10652                     void *data, struct rte_flow_error *error)
10653 {
10654         struct mlx5_priv *priv = dev->data->dev_private;
10655         struct rte_flow_query_count *qc = data;
10656
10657         if (!priv->config.devx)
10658                 return rte_flow_error_set(error, ENOTSUP,
10659                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10660                                           NULL,
10661                                           "counters are not supported");
10662         if (flow->counter) {
10663                 uint64_t pkts, bytes;
10664                 struct mlx5_flow_counter *cnt;
10665
10666                 cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
10667                                                  NULL);
10668                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
10669                                                &bytes);
10670
10671                 if (err)
10672                         return rte_flow_error_set(error, -err,
10673                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10674                                         NULL, "cannot read counters");
10675                 qc->hits_set = 1;
10676                 qc->bytes_set = 1;
10677                 qc->hits = pkts - cnt->hits;
10678                 qc->bytes = bytes - cnt->bytes;
10679                 if (qc->reset) {
10680                         cnt->hits = pkts;
10681                         cnt->bytes = bytes;
10682                 }
10683                 return 0;
10684         }
10685         return rte_flow_error_set(error, EINVAL,
10686                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10687                                   NULL,
10688                                   "counters are not available");
10689 }
10690
10691 /**
10692  * Query a flow rule AGE action for aging information.
10693  *
10694  * @param[in] dev
10695  *   Pointer to Ethernet device.
10696  * @param[in] flow
10697  *   Pointer to the sub flow.
10698  * @param[out] data
10699  *   data retrieved by the query.
10700  * @param[out] error
10701  *   Perform verbose error reporting if not NULL.
10702  *
10703  * @return
10704  *   0 on success, a negative errno value otherwise and rte_errno is set.
10705  */
10706 static int
10707 flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow,
10708                   void *data, struct rte_flow_error *error)
10709 {
10710         struct rte_flow_query_age *resp = data;
10711
10712         if (flow->counter) {
10713                 struct mlx5_age_param *age_param =
10714                                 flow_dv_counter_idx_get_age(dev, flow->counter);
10715
10716                 if (!age_param || !age_param->timeout)
10717                         return rte_flow_error_set
10718                                         (error, EINVAL,
10719                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10720                                          NULL, "cannot read age data");
10721                 resp->aged = __atomic_load_n(&age_param->state,
10722                                              __ATOMIC_RELAXED) ==
10723                                                         AGE_TMOUT ? 1 : 0;
10724                 resp->sec_since_last_hit_valid = !resp->aged;
10725                 if (resp->sec_since_last_hit_valid)
10726                         resp->sec_since_last_hit =
10727                                 __atomic_load_n(&age_param->sec_since_last_hit,
10728                                                 __ATOMIC_RELAXED);
10729                 return 0;
10730         }
10731         return rte_flow_error_set(error, EINVAL,
10732                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10733                                   NULL,
10734                                   "age data not available");
10735 }
10736
10737 /**
10738  * Query a flow.
10739  *
10740  * @see rte_flow_query()
10741  * @see rte_flow_ops
10742  */
10743 static int
10744 flow_dv_query(struct rte_eth_dev *dev,
10745               struct rte_flow *flow __rte_unused,
10746               const struct rte_flow_action *actions __rte_unused,
10747               void *data __rte_unused,
10748               struct rte_flow_error *error __rte_unused)
10749 {
10750         int ret = -EINVAL;
10751
10752         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
10753                 switch (actions->type) {
10754                 case RTE_FLOW_ACTION_TYPE_VOID:
10755                         break;
10756                 case RTE_FLOW_ACTION_TYPE_COUNT:
10757                         ret = flow_dv_query_count(dev, flow, data, error);
10758                         break;
10759                 case RTE_FLOW_ACTION_TYPE_AGE:
10760                         ret = flow_dv_query_age(dev, flow, data, error);
10761                         break;
10762                 default:
10763                         return rte_flow_error_set(error, ENOTSUP,
10764                                                   RTE_FLOW_ERROR_TYPE_ACTION,
10765                                                   actions,
10766                                                   "action not supported");
10767                 }
10768         }
10769         return ret;
10770 }
10771
10772 /**
10773  * Destroy the meter table set.
10774  * Lock free, (mutex should be acquired by caller).
10775  *
10776  * @param[in] dev
10777  *   Pointer to Ethernet device.
10778  * @param[in] tbl
10779  *   Pointer to the meter table set.
10780  *
10781  * @return
10782  *   Always 0.
10783  */
10784 static int
10785 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
10786                         struct mlx5_meter_domains_infos *tbl)
10787 {
10788         struct mlx5_priv *priv = dev->data->dev_private;
10789         struct mlx5_meter_domains_infos *mtd =
10790                                 (struct mlx5_meter_domains_infos *)tbl;
10791
10792         if (!mtd || !priv->config.dv_flow_en)
10793                 return 0;
10794         if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
10795                 claim_zero(mlx5_flow_os_destroy_flow
10796                            (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
10797         if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
10798                 claim_zero(mlx5_flow_os_destroy_flow
10799                            (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
10800         if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
10801                 claim_zero(mlx5_flow_os_destroy_flow
10802                            (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
10803         if (mtd->egress.color_matcher)
10804                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10805                            (mtd->egress.color_matcher));
10806         if (mtd->egress.any_matcher)
10807                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10808                            (mtd->egress.any_matcher));
10809         if (mtd->egress.tbl)
10810                 flow_dv_tbl_resource_release(dev, mtd->egress.tbl);
10811         if (mtd->egress.sfx_tbl)
10812                 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl);
10813         if (mtd->ingress.color_matcher)
10814                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10815                            (mtd->ingress.color_matcher));
10816         if (mtd->ingress.any_matcher)
10817                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10818                            (mtd->ingress.any_matcher));
10819         if (mtd->ingress.tbl)
10820                 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl);
10821         if (mtd->ingress.sfx_tbl)
10822                 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl);
10823         if (mtd->transfer.color_matcher)
10824                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10825                            (mtd->transfer.color_matcher));
10826         if (mtd->transfer.any_matcher)
10827                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10828                            (mtd->transfer.any_matcher));
10829         if (mtd->transfer.tbl)
10830                 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl);
10831         if (mtd->transfer.sfx_tbl)
10832                 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl);
10833         if (mtd->drop_actn)
10834                 claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn));
10835         mlx5_free(mtd);
10836         return 0;
10837 }
10838
10839 /* Number of meter flow actions, count and jump or count and drop. */
10840 #define METER_ACTIONS 2
10841
10842 /**
10843  * Create specify domain meter table and suffix table.
10844  *
10845  * @param[in] dev
10846  *   Pointer to Ethernet device.
10847  * @param[in,out] mtb
10848  *   Pointer to DV meter table set.
10849  * @param[in] egress
10850  *   Table attribute.
10851  * @param[in] transfer
10852  *   Table attribute.
10853  * @param[in] color_reg_c_idx
10854  *   Reg C index for color match.
10855  *
10856  * @return
10857  *   0 on success, -1 otherwise and rte_errno is set.
10858  */
10859 static int
10860 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
10861                            struct mlx5_meter_domains_infos *mtb,
10862                            uint8_t egress, uint8_t transfer,
10863                            uint32_t color_reg_c_idx)
10864 {
10865         struct mlx5_priv *priv = dev->data->dev_private;
10866         struct mlx5_dev_ctx_shared *sh = priv->sh;
10867         struct mlx5_flow_dv_match_params mask = {
10868                 .size = sizeof(mask.buf),
10869         };
10870         struct mlx5_flow_dv_match_params value = {
10871                 .size = sizeof(value.buf),
10872         };
10873         struct mlx5dv_flow_matcher_attr dv_attr = {
10874                 .type = IBV_FLOW_ATTR_NORMAL,
10875                 .priority = 0,
10876                 .match_criteria_enable = 0,
10877                 .match_mask = (void *)&mask,
10878         };
10879         void *actions[METER_ACTIONS];
10880         struct mlx5_meter_domain_info *dtb;
10881         struct rte_flow_error error;
10882         int i = 0;
10883         int ret;
10884
10885         if (transfer)
10886                 dtb = &mtb->transfer;
10887         else if (egress)
10888                 dtb = &mtb->egress;
10889         else
10890                 dtb = &mtb->ingress;
10891         /* Create the meter table with METER level. */
10892         dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
10893                                             egress, transfer, &error);
10894         if (!dtb->tbl) {
10895                 DRV_LOG(ERR, "Failed to create meter policer table.");
10896                 return -1;
10897         }
10898         /* Create the meter suffix table with SUFFIX level. */
10899         dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
10900                                             MLX5_FLOW_TABLE_LEVEL_SUFFIX,
10901                                             egress, transfer, &error);
10902         if (!dtb->sfx_tbl) {
10903                 DRV_LOG(ERR, "Failed to create meter suffix table.");
10904                 return -1;
10905         }
10906         /* Create matchers, Any and Color. */
10907         dv_attr.priority = 3;
10908         dv_attr.match_criteria_enable = 0;
10909         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
10910                                                &dtb->any_matcher);
10911         if (ret) {
10912                 DRV_LOG(ERR, "Failed to create meter"
10913                              " policer default matcher.");
10914                 goto error_exit;
10915         }
10916         dv_attr.priority = 0;
10917         dv_attr.match_criteria_enable =
10918                                 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
10919         flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
10920                                rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
10921         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
10922                                                &dtb->color_matcher);
10923         if (ret) {
10924                 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
10925                 goto error_exit;
10926         }
10927         if (mtb->count_actns[RTE_MTR_DROPPED])
10928                 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
10929         actions[i++] = mtb->drop_actn;
10930         /* Default rule: lowest priority, match any, actions: drop. */
10931         ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i,
10932                                        actions,
10933                                        &dtb->policer_rules[RTE_MTR_DROPPED]);
10934         if (ret) {
10935                 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
10936                 goto error_exit;
10937         }
10938         return 0;
10939 error_exit:
10940         return -1;
10941 }
10942
10943 /**
10944  * Create the needed meter and suffix tables.
10945  * Lock free, (mutex should be acquired by caller).
10946  *
10947  * @param[in] dev
10948  *   Pointer to Ethernet device.
10949  * @param[in] fm
10950  *   Pointer to the flow meter.
10951  *
10952  * @return
10953  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
10954  */
10955 static struct mlx5_meter_domains_infos *
10956 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
10957                        const struct mlx5_flow_meter *fm)
10958 {
10959         struct mlx5_priv *priv = dev->data->dev_private;
10960         struct mlx5_meter_domains_infos *mtb;
10961         int ret;
10962         int i;
10963
10964         if (!priv->mtr_en) {
10965                 rte_errno = ENOTSUP;
10966                 return NULL;
10967         }
10968         mtb = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*mtb), 0, SOCKET_ID_ANY);
10969         if (!mtb) {
10970                 DRV_LOG(ERR, "Failed to allocate memory for meter.");
10971                 return NULL;
10972         }
10973         /* Create meter count actions */
10974         for (i = 0; i <= RTE_MTR_DROPPED; i++) {
10975                 struct mlx5_flow_counter *cnt;
10976                 if (!fm->policer_stats.cnt[i])
10977                         continue;
10978                 cnt = flow_dv_counter_get_by_idx(dev,
10979                       fm->policer_stats.cnt[i], NULL);
10980                 mtb->count_actns[i] = cnt->action;
10981         }
10982         /* Create drop action. */
10983         ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn);
10984         if (ret) {
10985                 DRV_LOG(ERR, "Failed to create drop action.");
10986                 goto error_exit;
10987         }
10988         /* Egress meter table. */
10989         ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
10990         if (ret) {
10991                 DRV_LOG(ERR, "Failed to prepare egress meter table.");
10992                 goto error_exit;
10993         }
10994         /* Ingress meter table. */
10995         ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
10996         if (ret) {
10997                 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
10998                 goto error_exit;
10999         }
11000         /* FDB meter table. */
11001         if (priv->config.dv_esw_en) {
11002                 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
11003                                                  priv->mtr_color_reg);
11004                 if (ret) {
11005                         DRV_LOG(ERR, "Failed to prepare fdb meter table.");
11006                         goto error_exit;
11007                 }
11008         }
11009         return mtb;
11010 error_exit:
11011         flow_dv_destroy_mtr_tbl(dev, mtb);
11012         return NULL;
11013 }
11014
11015 /**
11016  * Destroy domain policer rule.
11017  *
11018  * @param[in] dt
11019  *   Pointer to domain table.
11020  */
11021 static void
11022 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
11023 {
11024         int i;
11025
11026         for (i = 0; i < RTE_MTR_DROPPED; i++) {
11027                 if (dt->policer_rules[i]) {
11028                         claim_zero(mlx5_flow_os_destroy_flow
11029                                    (dt->policer_rules[i]));
11030                         dt->policer_rules[i] = NULL;
11031                 }
11032         }
11033         if (dt->jump_actn) {
11034                 claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn));
11035                 dt->jump_actn = NULL;
11036         }
11037 }
11038
11039 /**
11040  * Destroy policer rules.
11041  *
11042  * @param[in] dev
11043  *   Pointer to Ethernet device.
11044  * @param[in] fm
11045  *   Pointer to flow meter structure.
11046  * @param[in] attr
11047  *   Pointer to flow attributes.
11048  *
11049  * @return
11050  *   Always 0.
11051  */
11052 static int
11053 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
11054                               const struct mlx5_flow_meter *fm,
11055                               const struct rte_flow_attr *attr)
11056 {
11057         struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
11058
11059         if (!mtb)
11060                 return 0;
11061         if (attr->egress)
11062                 flow_dv_destroy_domain_policer_rule(&mtb->egress);
11063         if (attr->ingress)
11064                 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
11065         if (attr->transfer)
11066                 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
11067         return 0;
11068 }
11069
11070 /**
11071  * Create specify domain meter policer rule.
11072  *
11073  * @param[in] fm
11074  *   Pointer to flow meter structure.
11075  * @param[in] mtb
11076  *   Pointer to DV meter table set.
11077  * @param[in] mtr_reg_c
11078  *   Color match REG_C.
11079  *
11080  * @return
11081  *   0 on success, -1 otherwise.
11082  */
11083 static int
11084 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
11085                                     struct mlx5_meter_domain_info *dtb,
11086                                     uint8_t mtr_reg_c)
11087 {
11088         struct mlx5_flow_dv_match_params matcher = {
11089                 .size = sizeof(matcher.buf),
11090         };
11091         struct mlx5_flow_dv_match_params value = {
11092                 .size = sizeof(value.buf),
11093         };
11094         struct mlx5_meter_domains_infos *mtb = fm->mfts;
11095         void *actions[METER_ACTIONS];
11096         int i;
11097         int ret = 0;
11098
11099         /* Create jump action. */
11100         if (!dtb->jump_actn)
11101                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
11102                                 (dtb->sfx_tbl->obj, &dtb->jump_actn);
11103         if (ret) {
11104                 DRV_LOG(ERR, "Failed to create policer jump action.");
11105                 goto error;
11106         }
11107         for (i = 0; i < RTE_MTR_DROPPED; i++) {
11108                 int j = 0;
11109
11110                 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
11111                                        rte_col_2_mlx5_col(i), UINT8_MAX);
11112                 if (mtb->count_actns[i])
11113                         actions[j++] = mtb->count_actns[i];
11114                 if (fm->action[i] == MTR_POLICER_ACTION_DROP)
11115                         actions[j++] = mtb->drop_actn;
11116                 else
11117                         actions[j++] = dtb->jump_actn;
11118                 ret = mlx5_flow_os_create_flow(dtb->color_matcher,
11119                                                (void *)&value, j, actions,
11120                                                &dtb->policer_rules[i]);
11121                 if (ret) {
11122                         DRV_LOG(ERR, "Failed to create policer rule.");
11123                         goto error;
11124                 }
11125         }
11126         return 0;
11127 error:
11128         rte_errno = errno;
11129         return -1;
11130 }
11131
11132 /**
11133  * Create policer rules.
11134  *
11135  * @param[in] dev
11136  *   Pointer to Ethernet device.
11137  * @param[in] fm
11138  *   Pointer to flow meter structure.
11139  * @param[in] attr
11140  *   Pointer to flow attributes.
11141  *
11142  * @return
11143  *   0 on success, -1 otherwise.
11144  */
11145 static int
11146 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
11147                              struct mlx5_flow_meter *fm,
11148                              const struct rte_flow_attr *attr)
11149 {
11150         struct mlx5_priv *priv = dev->data->dev_private;
11151         struct mlx5_meter_domains_infos *mtb = fm->mfts;
11152         int ret;
11153
11154         if (attr->egress) {
11155                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
11156                                                 priv->mtr_color_reg);
11157                 if (ret) {
11158                         DRV_LOG(ERR, "Failed to create egress policer.");
11159                         goto error;
11160                 }
11161         }
11162         if (attr->ingress) {
11163                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
11164                                                 priv->mtr_color_reg);
11165                 if (ret) {
11166                         DRV_LOG(ERR, "Failed to create ingress policer.");
11167                         goto error;
11168                 }
11169         }
11170         if (attr->transfer) {
11171                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
11172                                                 priv->mtr_color_reg);
11173                 if (ret) {
11174                         DRV_LOG(ERR, "Failed to create transfer policer.");
11175                         goto error;
11176                 }
11177         }
11178         return 0;
11179 error:
11180         flow_dv_destroy_policer_rules(dev, fm, attr);
11181         return -1;
11182 }
11183
11184 /**
11185  * Validate the batch counter support in root table.
11186  *
11187  * Create a simple flow with invalid counter and drop action on root table to
11188  * validate if batch counter with offset on root table is supported or not.
11189  *
11190  * @param[in] dev
11191  *   Pointer to rte_eth_dev structure.
11192  *
11193  * @return
11194  *   0 on success, a negative errno value otherwise and rte_errno is set.
11195  */
11196 int
11197 mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev)
11198 {
11199         struct mlx5_priv *priv = dev->data->dev_private;
11200         struct mlx5_dev_ctx_shared *sh = priv->sh;
11201         struct mlx5_flow_dv_match_params mask = {
11202                 .size = sizeof(mask.buf),
11203         };
11204         struct mlx5_flow_dv_match_params value = {
11205                 .size = sizeof(value.buf),
11206         };
11207         struct mlx5dv_flow_matcher_attr dv_attr = {
11208                 .type = IBV_FLOW_ATTR_NORMAL,
11209                 .priority = 0,
11210                 .match_criteria_enable = 0,
11211                 .match_mask = (void *)&mask,
11212         };
11213         void *actions[2] = { 0 };
11214         struct mlx5_flow_tbl_resource *tbl = NULL, *dest_tbl = NULL;
11215         struct mlx5_devx_obj *dcs = NULL;
11216         void *matcher = NULL;
11217         void *flow = NULL;
11218         int i, ret = -1;
11219
11220         tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, NULL);
11221         if (!tbl)
11222                 goto err;
11223         dest_tbl = flow_dv_tbl_resource_get(dev, 1, 0, 0, NULL);
11224         if (!dest_tbl)
11225                 goto err;
11226         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
11227         if (!dcs)
11228                 goto err;
11229         ret = mlx5_flow_os_create_flow_action_count(dcs->obj, UINT16_MAX,
11230                                                     &actions[0]);
11231         if (ret)
11232                 goto err;
11233         ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
11234                                 (dest_tbl->obj, &actions[1]);
11235         if (ret)
11236                 goto err;
11237         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
11238         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
11239                                                &matcher);
11240         if (ret)
11241                 goto err;
11242         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 2,
11243                                        actions, &flow);
11244 err:
11245         /*
11246          * If batch counter with offset is not supported, the driver will not
11247          * validate the invalid offset value, flow create should success.
11248          * In this case, it means batch counter is not supported in root table.
11249          *
11250          * Otherwise, if flow create is failed, counter offset is supported.
11251          */
11252         if (flow) {
11253                 DRV_LOG(INFO, "Batch counter is not supported in root "
11254                               "table. Switch to fallback mode.");
11255                 rte_errno = ENOTSUP;
11256                 ret = -rte_errno;
11257                 claim_zero(mlx5_flow_os_destroy_flow(flow));
11258         } else {
11259                 /* Check matcher to make sure validate fail at flow create. */
11260                 if (!matcher || (matcher && errno != EINVAL))
11261                         DRV_LOG(ERR, "Unexpected error in counter offset "
11262                                      "support detection");
11263                 ret = 0;
11264         }
11265         for (i = 0; i < 2; i++) {
11266                 if (actions[i])
11267                         claim_zero(mlx5_flow_os_destroy_flow_action
11268                                    (actions[i]));
11269         }
11270         if (matcher)
11271                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
11272         if (tbl)
11273                 flow_dv_tbl_resource_release(dev, tbl);
11274         if (dest_tbl)
11275                 flow_dv_tbl_resource_release(dev, dest_tbl);
11276         if (dcs)
11277                 claim_zero(mlx5_devx_cmd_destroy(dcs));
11278         return ret;
11279 }
11280
11281 /**
11282  * Query a devx counter.
11283  *
11284  * @param[in] dev
11285  *   Pointer to the Ethernet device structure.
11286  * @param[in] cnt
11287  *   Index to the flow counter.
11288  * @param[in] clear
11289  *   Set to clear the counter statistics.
11290  * @param[out] pkts
11291  *   The statistics value of packets.
11292  * @param[out] bytes
11293  *   The statistics value of bytes.
11294  *
11295  * @return
11296  *   0 on success, otherwise return -1.
11297  */
11298 static int
11299 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
11300                       uint64_t *pkts, uint64_t *bytes)
11301 {
11302         struct mlx5_priv *priv = dev->data->dev_private;
11303         struct mlx5_flow_counter *cnt;
11304         uint64_t inn_pkts, inn_bytes;
11305         int ret;
11306
11307         if (!priv->config.devx)
11308                 return -1;
11309
11310         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
11311         if (ret)
11312                 return -1;
11313         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
11314         *pkts = inn_pkts - cnt->hits;
11315         *bytes = inn_bytes - cnt->bytes;
11316         if (clear) {
11317                 cnt->hits = inn_pkts;
11318                 cnt->bytes = inn_bytes;
11319         }
11320         return 0;
11321 }
11322
11323 /**
11324  * Get aged-out flows.
11325  *
11326  * @param[in] dev
11327  *   Pointer to the Ethernet device structure.
11328  * @param[in] context
11329  *   The address of an array of pointers to the aged-out flows contexts.
11330  * @param[in] nb_contexts
11331  *   The length of context array pointers.
11332  * @param[out] error
11333  *   Perform verbose error reporting if not NULL. Initialized in case of
11334  *   error only.
11335  *
11336  * @return
11337  *   how many contexts get in success, otherwise negative errno value.
11338  *   if nb_contexts is 0, return the amount of all aged contexts.
11339  *   if nb_contexts is not 0 , return the amount of aged flows reported
11340  *   in the context array.
11341  * @note: only stub for now
11342  */
11343 static int
11344 flow_get_aged_flows(struct rte_eth_dev *dev,
11345                     void **context,
11346                     uint32_t nb_contexts,
11347                     struct rte_flow_error *error)
11348 {
11349         struct mlx5_priv *priv = dev->data->dev_private;
11350         struct mlx5_age_info *age_info;
11351         struct mlx5_age_param *age_param;
11352         struct mlx5_flow_counter *counter;
11353         int nb_flows = 0;
11354
11355         if (nb_contexts && !context)
11356                 return rte_flow_error_set(error, EINVAL,
11357                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11358                                           NULL,
11359                                           "Should assign at least one flow or"
11360                                           " context to get if nb_contexts != 0");
11361         age_info = GET_PORT_AGE_INFO(priv);
11362         rte_spinlock_lock(&age_info->aged_sl);
11363         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
11364                 nb_flows++;
11365                 if (nb_contexts) {
11366                         age_param = MLX5_CNT_TO_AGE(counter);
11367                         context[nb_flows - 1] = age_param->context;
11368                         if (!(--nb_contexts))
11369                                 break;
11370                 }
11371         }
11372         rte_spinlock_unlock(&age_info->aged_sl);
11373         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
11374         return nb_flows;
11375 }
11376
11377 /*
11378  * Mutex-protected thunk to lock-free  __flow_dv_translate().
11379  */
11380 static int
11381 flow_dv_translate(struct rte_eth_dev *dev,
11382                   struct mlx5_flow *dev_flow,
11383                   const struct rte_flow_attr *attr,
11384                   const struct rte_flow_item items[],
11385                   const struct rte_flow_action actions[],
11386                   struct rte_flow_error *error)
11387 {
11388         int ret;
11389
11390         flow_dv_shared_lock(dev);
11391         ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
11392         flow_dv_shared_unlock(dev);
11393         return ret;
11394 }
11395
11396 /*
11397  * Mutex-protected thunk to lock-free  __flow_dv_apply().
11398  */
11399 static int
11400 flow_dv_apply(struct rte_eth_dev *dev,
11401               struct rte_flow *flow,
11402               struct rte_flow_error *error)
11403 {
11404         int ret;
11405
11406         flow_dv_shared_lock(dev);
11407         ret = __flow_dv_apply(dev, flow, error);
11408         flow_dv_shared_unlock(dev);
11409         return ret;
11410 }
11411
11412 /*
11413  * Mutex-protected thunk to lock-free __flow_dv_remove().
11414  */
11415 static void
11416 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
11417 {
11418         flow_dv_shared_lock(dev);
11419         __flow_dv_remove(dev, flow);
11420         flow_dv_shared_unlock(dev);
11421 }
11422
11423 /*
11424  * Mutex-protected thunk to lock-free __flow_dv_destroy().
11425  */
11426 static void
11427 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
11428 {
11429         flow_dv_shared_lock(dev);
11430         __flow_dv_destroy(dev, flow);
11431         flow_dv_shared_unlock(dev);
11432 }
11433
11434 /*
11435  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
11436  */
11437 static uint32_t
11438 flow_dv_counter_allocate(struct rte_eth_dev *dev)
11439 {
11440         uint32_t cnt;
11441
11442         flow_dv_shared_lock(dev);
11443         cnt = flow_dv_counter_alloc(dev, 0);
11444         flow_dv_shared_unlock(dev);
11445         return cnt;
11446 }
11447
11448 /*
11449  * Mutex-protected thunk to lock-free flow_dv_counter_release().
11450  */
11451 static void
11452 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
11453 {
11454         flow_dv_shared_lock(dev);
11455         flow_dv_counter_release(dev, cnt);
11456         flow_dv_shared_unlock(dev);
11457 }
11458
11459 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
11460         .validate = flow_dv_validate,
11461         .prepare = flow_dv_prepare,
11462         .translate = flow_dv_translate,
11463         .apply = flow_dv_apply,
11464         .remove = flow_dv_remove,
11465         .destroy = flow_dv_destroy,
11466         .query = flow_dv_query,
11467         .create_mtr_tbls = flow_dv_create_mtr_tbl,
11468         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
11469         .create_policer_rules = flow_dv_create_policer_rules,
11470         .destroy_policer_rules = flow_dv_destroy_policer_rules,
11471         .counter_alloc = flow_dv_counter_allocate,
11472         .counter_free = flow_dv_counter_free,
11473         .counter_query = flow_dv_counter_query,
11474         .get_aged_flows = flow_get_aged_flows,
11475 };
11476
11477 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
11478