drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <rte_ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_ip.h>
  19 #include <rte_gre.h>
  20 #include <rte_vxlan.h>
  21 #include <rte_gtp.h>
  22 #include <rte_eal_paging.h>
  23 #include <rte_mpls.h>
  24
  25 #include <mlx5_glue.h>
  26 #include <mlx5_devx_cmds.h>
  27 #include <mlx5_prm.h>
  28 #include <mlx5_malloc.h>
  29
  30 #include "mlx5_defs.h"
  31 #include "mlx5.h"
  32 #include "mlx5_common_os.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_flow_os.h"
  35 #include "mlx5_rxtx.h"
  36
  37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  38
  39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  41 #endif
  42
  43 #ifndef HAVE_MLX5DV_DR_ESWITCH
  44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  46 #endif
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR
  50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  51 #endif
  52
  53 /* VLAN header definitions */
  54 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  55 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  56 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  57 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  58 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  59
  60 union flow_dv_attr {
  61         struct {
  62                 uint32_t valid:1;
  63                 uint32_t ipv4:1;
  64                 uint32_t ipv6:1;
  65                 uint32_t tcp:1;
  66                 uint32_t udp:1;
  67                 uint32_t reserved:27;
  68         };
  69         uint32_t attr;
  70 };
  71
  72 static int
  73 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
  74                              struct mlx5_flow_tbl_resource *tbl);
  75
  76 static int
  77 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev);
  78
  79 static int
  80 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  81                                       uint32_t encap_decap_idx);
  82
  83 static int
  84 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  85                                         uint32_t port_id);
  86
  87 /**
  88  * Initialize flow attributes structure according to flow items' types.
  89  *
  90  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  91  * mode. For tunnel mode, the items to be modified are the outermost ones.
  92  *
  93  * @param[in] item
  94  *   Pointer to item specification.
  95  * @param[out] attr
  96  *   Pointer to flow attributes structure.
  97  * @param[in] dev_flow
  98  *   Pointer to the sub flow.
  99  * @param[in] tunnel_decap
 100  *   Whether action is after tunnel decapsulation.
 101  */
 102 static void
 103 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 104                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 105 {
 106         uint64_t layers = dev_flow->handle->layers;
 107
 108         /*
 109          * If layers is already initialized, it means this dev_flow is the
 110          * suffix flow, the layers flags is set by the prefix flow. Need to
 111          * use the layer flags from prefix flow as the suffix flow may not
 112          * have the user defined items as the flow is split.
 113          */
 114         if (layers) {
 115                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 116                         attr->ipv4 = 1;
 117                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 118                         attr->ipv6 = 1;
 119                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 120                         attr->tcp = 1;
 121                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 122                         attr->udp = 1;
 123                 attr->valid = 1;
 124                 return;
 125         }
 126         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 127                 uint8_t next_protocol = 0xff;
 128                 switch (item->type) {
 129                 case RTE_FLOW_ITEM_TYPE_GRE:
 130                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 131                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 132                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 133                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 134                 case RTE_FLOW_ITEM_TYPE_MPLS:
 135                         if (tunnel_decap)
 136                                 attr->attr = 0;
 137                         break;
 138                 case RTE_FLOW_ITEM_TYPE_IPV4:
 139                         if (!attr->ipv6)
 140                                 attr->ipv4 = 1;
 141                         if (item->mask != NULL &&
 142                             ((const struct rte_flow_item_ipv4 *)
 143                             item->mask)->hdr.next_proto_id)
 144                                 next_protocol =
 145                                     ((const struct rte_flow_item_ipv4 *)
 146                                       (item->spec))->hdr.next_proto_id &
 147                                     ((const struct rte_flow_item_ipv4 *)
 148                                       (item->mask))->hdr.next_proto_id;
 149                         if ((next_protocol == IPPROTO_IPIP ||
 150                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 151                                 attr->attr = 0;
 152                         break;
 153                 case RTE_FLOW_ITEM_TYPE_IPV6:
 154                         if (!attr->ipv4)
 155                                 attr->ipv6 = 1;
 156                         if (item->mask != NULL &&
 157                             ((const struct rte_flow_item_ipv6 *)
 158                             item->mask)->hdr.proto)
 159                                 next_protocol =
 160                                     ((const struct rte_flow_item_ipv6 *)
 161                                       (item->spec))->hdr.proto &
 162                                     ((const struct rte_flow_item_ipv6 *)
 163                                       (item->mask))->hdr.proto;
 164                         if ((next_protocol == IPPROTO_IPIP ||
 165                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 166                                 attr->attr = 0;
 167                         break;
 168                 case RTE_FLOW_ITEM_TYPE_UDP:
 169                         if (!attr->tcp)
 170                                 attr->udp = 1;
 171                         break;
 172                 case RTE_FLOW_ITEM_TYPE_TCP:
 173                         if (!attr->udp)
 174                                 attr->tcp = 1;
 175                         break;
 176                 default:
 177                         break;
 178                 }
 179         }
 180         attr->valid = 1;
 181 }
 182
 183 /**
 184  * Convert rte_mtr_color to mlx5 color.
 185  *
 186  * @param[in] rcol
 187  *   rte_mtr_color.
 188  *
 189  * @return
 190  *   mlx5 color.
 191  */
 192 static int
 193 rte_col_2_mlx5_col(enum rte_color rcol)
 194 {
 195         switch (rcol) {
 196         case RTE_COLOR_GREEN:
 197                 return MLX5_FLOW_COLOR_GREEN;
 198         case RTE_COLOR_YELLOW:
 199                 return MLX5_FLOW_COLOR_YELLOW;
 200         case RTE_COLOR_RED:
 201                 return MLX5_FLOW_COLOR_RED;
 202         default:
 203                 break;
 204         }
 205         return MLX5_FLOW_COLOR_UNDEFINED;
 206 }
 207
 208 struct field_modify_info {
 209         uint32_t size; /* Size of field in protocol header, in bytes. */
 210         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 211         enum mlx5_modification_field id;
 212 };
 213
 214 struct field_modify_info modify_eth[] = {
 215         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 216         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 217         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 218         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 219         {0, 0, 0},
 220 };
 221
 222 struct field_modify_info modify_vlan_out_first_vid[] = {
 223         /* Size in bits !!! */
 224         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 225         {0, 0, 0},
 226 };
 227
 228 struct field_modify_info modify_ipv4[] = {
 229         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 230         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 231         {4, 12, MLX5_MODI_OUT_SIPV4},
 232         {4, 16, MLX5_MODI_OUT_DIPV4},
 233         {0, 0, 0},
 234 };
 235
 236 struct field_modify_info modify_ipv6[] = {
 237         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 238         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 239         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 240         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 241         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 242         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 243         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 244         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 245         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 246         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 247         {0, 0, 0},
 248 };
 249
 250 struct field_modify_info modify_udp[] = {
 251         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 252         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 253         {0, 0, 0},
 254 };
 255
 256 struct field_modify_info modify_tcp[] = {
 257         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 258         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 259         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 260         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 261         {0, 0, 0},
 262 };
 263
 264 static void
 265 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 266                           uint8_t next_protocol, uint64_t *item_flags,
 267                           int *tunnel)
 268 {
 269         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 270                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 271         if (next_protocol == IPPROTO_IPIP) {
 272                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 273                 *tunnel = 1;
 274         }
 275         if (next_protocol == IPPROTO_IPV6) {
 276                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 277                 *tunnel = 1;
 278         }
 279 }
 280
 281 /**
 282  * Acquire the synchronizing object to protect multithreaded access
 283  * to shared dv context. Lock occurs only if context is actually
 284  * shared, i.e. we have multiport IB device and representors are
 285  * created.
 286  *
 287  * @param[in] dev
 288  *   Pointer to the rte_eth_dev structure.
 289  */
 290 static void
 291 flow_dv_shared_lock(struct rte_eth_dev *dev)
 292 {
 293         struct mlx5_priv *priv = dev->data->dev_private;
 294         struct mlx5_dev_ctx_shared *sh = priv->sh;
 295
 296         if (sh->dv_refcnt > 1) {
 297                 int ret;
 298
 299                 ret = pthread_mutex_lock(&sh->dv_mutex);
 300                 MLX5_ASSERT(!ret);
 301                 (void)ret;
 302         }
 303 }
 304
 305 static void
 306 flow_dv_shared_unlock(struct rte_eth_dev *dev)
 307 {
 308         struct mlx5_priv *priv = dev->data->dev_private;
 309         struct mlx5_dev_ctx_shared *sh = priv->sh;
 310
 311         if (sh->dv_refcnt > 1) {
 312                 int ret;
 313
 314                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 315                 MLX5_ASSERT(!ret);
 316                 (void)ret;
 317         }
 318 }
 319
 320 /* Update VLAN's VID/PCP based on input rte_flow_action.
 321  *
 322  * @param[in] action
 323  *   Pointer to struct rte_flow_action.
 324  * @param[out] vlan
 325  *   Pointer to struct rte_vlan_hdr.
 326  */
 327 static void
 328 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 329                          struct rte_vlan_hdr *vlan)
 330 {
 331         uint16_t vlan_tci;
 332         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 333                 vlan_tci =
 334                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 335                                                action->conf)->vlan_pcp;
 336                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 337                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 338                 vlan->vlan_tci |= vlan_tci;
 339         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 340                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 341                 vlan->vlan_tci |= rte_be_to_cpu_16
 342                     (((const struct rte_flow_action_of_set_vlan_vid *)
 343                                              action->conf)->vlan_vid);
 344         }
 345 }
 346
 347 /**
 348  * Fetch 1, 2, 3 or 4 byte field from the byte array
 349  * and return as unsigned integer in host-endian format.
 350  *
 351  * @param[in] data
 352  *   Pointer to data array.
 353  * @param[in] size
 354  *   Size of field to extract.
 355  *
 356  * @return
 357  *   converted field in host endian format.
 358  */
 359 static inline uint32_t
 360 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 361 {
 362         uint32_t ret;
 363
 364         switch (size) {
 365         case 1:
 366                 ret = *data;
 367                 break;
 368         case 2:
 369                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 370                 break;
 371         case 3:
 372                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 373                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 374                 break;
 375         case 4:
 376                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 377                 break;
 378         default:
 379                 MLX5_ASSERT(false);
 380                 ret = 0;
 381                 break;
 382         }
 383         return ret;
 384 }
 385
 386 /**
 387  * Convert modify-header action to DV specification.
 388  *
 389  * Data length of each action is determined by provided field description
 390  * and the item mask. Data bit offset and width of each action is determined
 391  * by provided item mask.
 392  *
 393  * @param[in] item
 394  *   Pointer to item specification.
 395  * @param[in] field
 396  *   Pointer to field modification information.
 397  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 398  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 399  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 400  * @param[in] dcopy
 401  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 402  *   Negative offset value sets the same offset as source offset.
 403  *   size field is ignored, value is taken from source field.
 404  * @param[in,out] resource
 405  *   Pointer to the modify-header resource.
 406  * @param[in] type
 407  *   Type of modification.
 408  * @param[out] error
 409  *   Pointer to the error structure.
 410  *
 411  * @return
 412  *   0 on success, a negative errno value otherwise and rte_errno is set.
 413  */
 414 static int
 415 flow_dv_convert_modify_action(struct rte_flow_item *item,
 416                               struct field_modify_info *field,
 417                               struct field_modify_info *dcopy,
 418                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 419                               uint32_t type, struct rte_flow_error *error)
 420 {
 421         uint32_t i = resource->actions_num;
 422         struct mlx5_modification_cmd *actions = resource->actions;
 423
 424         /*
 425          * The item and mask are provided in big-endian format.
 426          * The fields should be presented as in big-endian format either.
 427          * Mask must be always present, it defines the actual field width.
 428          */
 429         MLX5_ASSERT(item->mask);
 430         MLX5_ASSERT(field->size);
 431         do {
 432                 unsigned int size_b;
 433                 unsigned int off_b;
 434                 uint32_t mask;
 435                 uint32_t data;
 436
 437                 if (i >= MLX5_MAX_MODIFY_NUM)
 438                         return rte_flow_error_set(error, EINVAL,
 439                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 440                                  "too many items to modify");
 441                 /* Fetch variable byte size mask from the array. */
 442                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 443                                            field->offset, field->size);
 444                 if (!mask) {
 445                         ++field;
 446                         continue;
 447                 }
 448                 /* Deduce actual data width in bits from mask value. */
 449                 off_b = rte_bsf32(mask);
 450                 size_b = sizeof(uint32_t) * CHAR_BIT -
 451                          off_b - __builtin_clz(mask);
 452                 MLX5_ASSERT(size_b);
 453                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 454                 actions[i] = (struct mlx5_modification_cmd) {
 455                         .action_type = type,
 456                         .field = field->id,
 457                         .offset = off_b,
 458                         .length = size_b,
 459                 };
 460                 /* Convert entire record to expected big-endian format. */
 461                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 462                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 463                         MLX5_ASSERT(dcopy);
 464                         actions[i].dst_field = dcopy->id;
 465                         actions[i].dst_offset =
 466                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 467                         /* Convert entire record to big-endian format. */
 468                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 469                 } else {
 470                         MLX5_ASSERT(item->spec);
 471                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 472                                                    field->offset, field->size);
 473                         /* Shift out the trailing masked bits from data. */
 474                         data = (data & mask) >> off_b;
 475                         actions[i].data1 = rte_cpu_to_be_32(data);
 476                 }
 477                 ++i;
 478                 ++field;
 479         } while (field->size);
 480         if (resource->actions_num == i)
 481                 return rte_flow_error_set(error, EINVAL,
 482                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 483                                           "invalid modification flow item");
 484         resource->actions_num = i;
 485         return 0;
 486 }
 487
 488 /**
 489  * Convert modify-header set IPv4 address action to DV specification.
 490  *
 491  * @param[in,out] resource
 492  *   Pointer to the modify-header resource.
 493  * @param[in] action
 494  *   Pointer to action specification.
 495  * @param[out] error
 496  *   Pointer to the error structure.
 497  *
 498  * @return
 499  *   0 on success, a negative errno value otherwise and rte_errno is set.
 500  */
 501 static int
 502 flow_dv_convert_action_modify_ipv4
 503                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 504                          const struct rte_flow_action *action,
 505                          struct rte_flow_error *error)
 506 {
 507         const struct rte_flow_action_set_ipv4 *conf =
 508                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 509         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 510         struct rte_flow_item_ipv4 ipv4;
 511         struct rte_flow_item_ipv4 ipv4_mask;
 512
 513         memset(&ipv4, 0, sizeof(ipv4));
 514         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 515         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 516                 ipv4.hdr.src_addr = conf->ipv4_addr;
 517                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 518         } else {
 519                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 520                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 521         }
 522         item.spec = &ipv4;
 523         item.mask = &ipv4_mask;
 524         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 525                                              MLX5_MODIFICATION_TYPE_SET, error);
 526 }
 527
 528 /**
 529  * Convert modify-header set IPv6 address action to DV specification.
 530  *
 531  * @param[in,out] resource
 532  *   Pointer to the modify-header resource.
 533  * @param[in] action
 534  *   Pointer to action specification.
 535  * @param[out] error
 536  *   Pointer to the error structure.
 537  *
 538  * @return
 539  *   0 on success, a negative errno value otherwise and rte_errno is set.
 540  */
 541 static int
 542 flow_dv_convert_action_modify_ipv6
 543                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 544                          const struct rte_flow_action *action,
 545                          struct rte_flow_error *error)
 546 {
 547         const struct rte_flow_action_set_ipv6 *conf =
 548                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 549         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 550         struct rte_flow_item_ipv6 ipv6;
 551         struct rte_flow_item_ipv6 ipv6_mask;
 552
 553         memset(&ipv6, 0, sizeof(ipv6));
 554         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 555         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 556                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 557                        sizeof(ipv6.hdr.src_addr));
 558                 memcpy(&ipv6_mask.hdr.src_addr,
 559                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 560                        sizeof(ipv6.hdr.src_addr));
 561         } else {
 562                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 563                        sizeof(ipv6.hdr.dst_addr));
 564                 memcpy(&ipv6_mask.hdr.dst_addr,
 565                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 566                        sizeof(ipv6.hdr.dst_addr));
 567         }
 568         item.spec = &ipv6;
 569         item.mask = &ipv6_mask;
 570         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 571                                              MLX5_MODIFICATION_TYPE_SET, error);
 572 }
 573
 574 /**
 575  * Convert modify-header set MAC address action to DV specification.
 576  *
 577  * @param[in,out] resource
 578  *   Pointer to the modify-header resource.
 579  * @param[in] action
 580  *   Pointer to action specification.
 581  * @param[out] error
 582  *   Pointer to the error structure.
 583  *
 584  * @return
 585  *   0 on success, a negative errno value otherwise and rte_errno is set.
 586  */
 587 static int
 588 flow_dv_convert_action_modify_mac
 589                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 590                          const struct rte_flow_action *action,
 591                          struct rte_flow_error *error)
 592 {
 593         const struct rte_flow_action_set_mac *conf =
 594                 (const struct rte_flow_action_set_mac *)(action->conf);
 595         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 596         struct rte_flow_item_eth eth;
 597         struct rte_flow_item_eth eth_mask;
 598
 599         memset(&eth, 0, sizeof(eth));
 600         memset(&eth_mask, 0, sizeof(eth_mask));
 601         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 602                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 603                        sizeof(eth.src.addr_bytes));
 604                 memcpy(&eth_mask.src.addr_bytes,
 605                        &rte_flow_item_eth_mask.src.addr_bytes,
 606                        sizeof(eth_mask.src.addr_bytes));
 607         } else {
 608                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 609                        sizeof(eth.dst.addr_bytes));
 610                 memcpy(&eth_mask.dst.addr_bytes,
 611                        &rte_flow_item_eth_mask.dst.addr_bytes,
 612                        sizeof(eth_mask.dst.addr_bytes));
 613         }
 614         item.spec = &eth;
 615         item.mask = &eth_mask;
 616         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 617                                              MLX5_MODIFICATION_TYPE_SET, error);
 618 }
 619
 620 /**
 621  * Convert modify-header set VLAN VID action to DV specification.
 622  *
 623  * @param[in,out] resource
 624  *   Pointer to the modify-header resource.
 625  * @param[in] action
 626  *   Pointer to action specification.
 627  * @param[out] error
 628  *   Pointer to the error structure.
 629  *
 630  * @return
 631  *   0 on success, a negative errno value otherwise and rte_errno is set.
 632  */
 633 static int
 634 flow_dv_convert_action_modify_vlan_vid
 635                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 636                          const struct rte_flow_action *action,
 637                          struct rte_flow_error *error)
 638 {
 639         const struct rte_flow_action_of_set_vlan_vid *conf =
 640                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 641         int i = resource->actions_num;
 642         struct mlx5_modification_cmd *actions = resource->actions;
 643         struct field_modify_info *field = modify_vlan_out_first_vid;
 644
 645         if (i >= MLX5_MAX_MODIFY_NUM)
 646                 return rte_flow_error_set(error, EINVAL,
 647                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 648                          "too many items to modify");
 649         actions[i] = (struct mlx5_modification_cmd) {
 650                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 651                 .field = field->id,
 652                 .length = field->size,
 653                 .offset = field->offset,
 654         };
 655         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 656         actions[i].data1 = conf->vlan_vid;
 657         actions[i].data1 = actions[i].data1 << 16;
 658         resource->actions_num = ++i;
 659         return 0;
 660 }
 661
 662 /**
 663  * Convert modify-header set TP action to DV specification.
 664  *
 665  * @param[in,out] resource
 666  *   Pointer to the modify-header resource.
 667  * @param[in] action
 668  *   Pointer to action specification.
 669  * @param[in] items
 670  *   Pointer to rte_flow_item objects list.
 671  * @param[in] attr
 672  *   Pointer to flow attributes structure.
 673  * @param[in] dev_flow
 674  *   Pointer to the sub flow.
 675  * @param[in] tunnel_decap
 676  *   Whether action is after tunnel decapsulation.
 677  * @param[out] error
 678  *   Pointer to the error structure.
 679  *
 680  * @return
 681  *   0 on success, a negative errno value otherwise and rte_errno is set.
 682  */
 683 static int
 684 flow_dv_convert_action_modify_tp
 685                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 686                          const struct rte_flow_action *action,
 687                          const struct rte_flow_item *items,
 688                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 689                          bool tunnel_decap, struct rte_flow_error *error)
 690 {
 691         const struct rte_flow_action_set_tp *conf =
 692                 (const struct rte_flow_action_set_tp *)(action->conf);
 693         struct rte_flow_item item;
 694         struct rte_flow_item_udp udp;
 695         struct rte_flow_item_udp udp_mask;
 696         struct rte_flow_item_tcp tcp;
 697         struct rte_flow_item_tcp tcp_mask;
 698         struct field_modify_info *field;
 699
 700         if (!attr->valid)
 701                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 702         if (attr->udp) {
 703                 memset(&udp, 0, sizeof(udp));
 704                 memset(&udp_mask, 0, sizeof(udp_mask));
 705                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 706                         udp.hdr.src_port = conf->port;
 707                         udp_mask.hdr.src_port =
 708                                         rte_flow_item_udp_mask.hdr.src_port;
 709                 } else {
 710                         udp.hdr.dst_port = conf->port;
 711                         udp_mask.hdr.dst_port =
 712                                         rte_flow_item_udp_mask.hdr.dst_port;
 713                 }
 714                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 715                 item.spec = &udp;
 716                 item.mask = &udp_mask;
 717                 field = modify_udp;
 718         } else {
 719                 MLX5_ASSERT(attr->tcp);
 720                 memset(&tcp, 0, sizeof(tcp));
 721                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 722                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 723                         tcp.hdr.src_port = conf->port;
 724                         tcp_mask.hdr.src_port =
 725                                         rte_flow_item_tcp_mask.hdr.src_port;
 726                 } else {
 727                         tcp.hdr.dst_port = conf->port;
 728                         tcp_mask.hdr.dst_port =
 729                                         rte_flow_item_tcp_mask.hdr.dst_port;
 730                 }
 731                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 732                 item.spec = &tcp;
 733                 item.mask = &tcp_mask;
 734                 field = modify_tcp;
 735         }
 736         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 737                                              MLX5_MODIFICATION_TYPE_SET, error);
 738 }
 739
 740 /**
 741  * Convert modify-header set TTL action to DV specification.
 742  *
 743  * @param[in,out] resource
 744  *   Pointer to the modify-header resource.
 745  * @param[in] action
 746  *   Pointer to action specification.
 747  * @param[in] items
 748  *   Pointer to rte_flow_item objects list.
 749  * @param[in] attr
 750  *   Pointer to flow attributes structure.
 751  * @param[in] dev_flow
 752  *   Pointer to the sub flow.
 753  * @param[in] tunnel_decap
 754  *   Whether action is after tunnel decapsulation.
 755  * @param[out] error
 756  *   Pointer to the error structure.
 757  *
 758  * @return
 759  *   0 on success, a negative errno value otherwise and rte_errno is set.
 760  */
 761 static int
 762 flow_dv_convert_action_modify_ttl
 763                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 764                          const struct rte_flow_action *action,
 765                          const struct rte_flow_item *items,
 766                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 767                          bool tunnel_decap, struct rte_flow_error *error)
 768 {
 769         const struct rte_flow_action_set_ttl *conf =
 770                 (const struct rte_flow_action_set_ttl *)(action->conf);
 771         struct rte_flow_item item;
 772         struct rte_flow_item_ipv4 ipv4;
 773         struct rte_flow_item_ipv4 ipv4_mask;
 774         struct rte_flow_item_ipv6 ipv6;
 775         struct rte_flow_item_ipv6 ipv6_mask;
 776         struct field_modify_info *field;
 777
 778         if (!attr->valid)
 779                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 780         if (attr->ipv4) {
 781                 memset(&ipv4, 0, sizeof(ipv4));
 782                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 783                 ipv4.hdr.time_to_live = conf->ttl_value;
 784                 ipv4_mask.hdr.time_to_live = 0xFF;
 785                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 786                 item.spec = &ipv4;
 787                 item.mask = &ipv4_mask;
 788                 field = modify_ipv4;
 789         } else {
 790                 MLX5_ASSERT(attr->ipv6);
 791                 memset(&ipv6, 0, sizeof(ipv6));
 792                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 793                 ipv6.hdr.hop_limits = conf->ttl_value;
 794                 ipv6_mask.hdr.hop_limits = 0xFF;
 795                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 796                 item.spec = &ipv6;
 797                 item.mask = &ipv6_mask;
 798                 field = modify_ipv6;
 799         }
 800         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 801                                              MLX5_MODIFICATION_TYPE_SET, error);
 802 }
 803
 804 /**
 805  * Convert modify-header decrement TTL action to DV specification.
 806  *
 807  * @param[in,out] resource
 808  *   Pointer to the modify-header resource.
 809  * @param[in] action
 810  *   Pointer to action specification.
 811  * @param[in] items
 812  *   Pointer to rte_flow_item objects list.
 813  * @param[in] attr
 814  *   Pointer to flow attributes structure.
 815  * @param[in] dev_flow
 816  *   Pointer to the sub flow.
 817  * @param[in] tunnel_decap
 818  *   Whether action is after tunnel decapsulation.
 819  * @param[out] error
 820  *   Pointer to the error structure.
 821  *
 822  * @return
 823  *   0 on success, a negative errno value otherwise and rte_errno is set.
 824  */
 825 static int
 826 flow_dv_convert_action_modify_dec_ttl
 827                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 828                          const struct rte_flow_item *items,
 829                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 830                          bool tunnel_decap, struct rte_flow_error *error)
 831 {
 832         struct rte_flow_item item;
 833         struct rte_flow_item_ipv4 ipv4;
 834         struct rte_flow_item_ipv4 ipv4_mask;
 835         struct rte_flow_item_ipv6 ipv6;
 836         struct rte_flow_item_ipv6 ipv6_mask;
 837         struct field_modify_info *field;
 838
 839         if (!attr->valid)
 840                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 841         if (attr->ipv4) {
 842                 memset(&ipv4, 0, sizeof(ipv4));
 843                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 844                 ipv4.hdr.time_to_live = 0xFF;
 845                 ipv4_mask.hdr.time_to_live = 0xFF;
 846                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 847                 item.spec = &ipv4;
 848                 item.mask = &ipv4_mask;
 849                 field = modify_ipv4;
 850         } else {
 851                 MLX5_ASSERT(attr->ipv6);
 852                 memset(&ipv6, 0, sizeof(ipv6));
 853                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 854                 ipv6.hdr.hop_limits = 0xFF;
 855                 ipv6_mask.hdr.hop_limits = 0xFF;
 856                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 857                 item.spec = &ipv6;
 858                 item.mask = &ipv6_mask;
 859                 field = modify_ipv6;
 860         }
 861         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 862                                              MLX5_MODIFICATION_TYPE_ADD, error);
 863 }
 864
 865 /**
 866  * Convert modify-header increment/decrement TCP Sequence number
 867  * to DV specification.
 868  *
 869  * @param[in,out] resource
 870  *   Pointer to the modify-header resource.
 871  * @param[in] action
 872  *   Pointer to action specification.
 873  * @param[out] error
 874  *   Pointer to the error structure.
 875  *
 876  * @return
 877  *   0 on success, a negative errno value otherwise and rte_errno is set.
 878  */
 879 static int
 880 flow_dv_convert_action_modify_tcp_seq
 881                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 882                          const struct rte_flow_action *action,
 883                          struct rte_flow_error *error)
 884 {
 885         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 886         uint64_t value = rte_be_to_cpu_32(*conf);
 887         struct rte_flow_item item;
 888         struct rte_flow_item_tcp tcp;
 889         struct rte_flow_item_tcp tcp_mask;
 890
 891         memset(&tcp, 0, sizeof(tcp));
 892         memset(&tcp_mask, 0, sizeof(tcp_mask));
 893         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 894                 /*
 895                  * The HW has no decrement operation, only increment operation.
 896                  * To simulate decrement X from Y using increment operation
 897                  * we need to add UINT32_MAX X times to Y.
 898                  * Each adding of UINT32_MAX decrements Y by 1.
 899                  */
 900                 value *= UINT32_MAX;
 901         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 902         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 903         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 904         item.spec = &tcp;
 905         item.mask = &tcp_mask;
 906         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 907                                              MLX5_MODIFICATION_TYPE_ADD, error);
 908 }
 909
 910 /**
 911  * Convert modify-header increment/decrement TCP Acknowledgment number
 912  * to DV specification.
 913  *
 914  * @param[in,out] resource
 915  *   Pointer to the modify-header resource.
 916  * @param[in] action
 917  *   Pointer to action specification.
 918  * @param[out] error
 919  *   Pointer to the error structure.
 920  *
 921  * @return
 922  *   0 on success, a negative errno value otherwise and rte_errno is set.
 923  */
 924 static int
 925 flow_dv_convert_action_modify_tcp_ack
 926                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 927                          const struct rte_flow_action *action,
 928                          struct rte_flow_error *error)
 929 {
 930         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 931         uint64_t value = rte_be_to_cpu_32(*conf);
 932         struct rte_flow_item item;
 933         struct rte_flow_item_tcp tcp;
 934         struct rte_flow_item_tcp tcp_mask;
 935
 936         memset(&tcp, 0, sizeof(tcp));
 937         memset(&tcp_mask, 0, sizeof(tcp_mask));
 938         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 939                 /*
 940                  * The HW has no decrement operation, only increment operation.
 941                  * To simulate decrement X from Y using increment operation
 942                  * we need to add UINT32_MAX X times to Y.
 943                  * Each adding of UINT32_MAX decrements Y by 1.
 944                  */
 945                 value *= UINT32_MAX;
 946         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 947         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 948         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 949         item.spec = &tcp;
 950         item.mask = &tcp_mask;
 951         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 952                                              MLX5_MODIFICATION_TYPE_ADD, error);
 953 }
 954
 955 static enum mlx5_modification_field reg_to_field[] = {
 956         [REG_NON] = MLX5_MODI_OUT_NONE,
 957         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 958         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 959         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 960         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 961         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 962         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 963         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 964         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 965         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 966         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 967 };
 968
 969 /**
 970  * Convert register set to DV specification.
 971  *
 972  * @param[in,out] resource
 973  *   Pointer to the modify-header resource.
 974  * @param[in] action
 975  *   Pointer to action specification.
 976  * @param[out] error
 977  *   Pointer to the error structure.
 978  *
 979  * @return
 980  *   0 on success, a negative errno value otherwise and rte_errno is set.
 981  */
 982 static int
 983 flow_dv_convert_action_set_reg
 984                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 985                          const struct rte_flow_action *action,
 986                          struct rte_flow_error *error)
 987 {
 988         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 989         struct mlx5_modification_cmd *actions = resource->actions;
 990         uint32_t i = resource->actions_num;
 991
 992         if (i >= MLX5_MAX_MODIFY_NUM)
 993                 return rte_flow_error_set(error, EINVAL,
 994                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 995                                           "too many items to modify");
 996         MLX5_ASSERT(conf->id != REG_NON);
 997         MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field));
 998         actions[i] = (struct mlx5_modification_cmd) {
 999                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1000                 .field = reg_to_field[conf->id],
1001         };
1002         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1003         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1004         ++i;
1005         resource->actions_num = i;
1006         return 0;
1007 }
1008
1009 /**
1010  * Convert SET_TAG action to DV specification.
1011  *
1012  * @param[in] dev
1013  *   Pointer to the rte_eth_dev structure.
1014  * @param[in,out] resource
1015  *   Pointer to the modify-header resource.
1016  * @param[in] conf
1017  *   Pointer to action specification.
1018  * @param[out] error
1019  *   Pointer to the error structure.
1020  *
1021  * @return
1022  *   0 on success, a negative errno value otherwise and rte_errno is set.
1023  */
1024 static int
1025 flow_dv_convert_action_set_tag
1026                         (struct rte_eth_dev *dev,
1027                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1028                          const struct rte_flow_action_set_tag *conf,
1029                          struct rte_flow_error *error)
1030 {
1031         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1032         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1033         struct rte_flow_item item = {
1034                 .spec = &data,
1035                 .mask = &mask,
1036         };
1037         struct field_modify_info reg_c_x[] = {
1038                 [1] = {0, 0, 0},
1039         };
1040         enum mlx5_modification_field reg_type;
1041         int ret;
1042
1043         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1044         if (ret < 0)
1045                 return ret;
1046         MLX5_ASSERT(ret != REG_NON);
1047         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1048         reg_type = reg_to_field[ret];
1049         MLX5_ASSERT(reg_type > 0);
1050         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1051         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1052                                              MLX5_MODIFICATION_TYPE_SET, error);
1053 }
1054
1055 /**
1056  * Convert internal COPY_REG action to DV specification.
1057  *
1058  * @param[in] dev
1059  *   Pointer to the rte_eth_dev structure.
1060  * @param[in,out] res
1061  *   Pointer to the modify-header resource.
1062  * @param[in] action
1063  *   Pointer to action specification.
1064  * @param[out] error
1065  *   Pointer to the error structure.
1066  *
1067  * @return
1068  *   0 on success, a negative errno value otherwise and rte_errno is set.
1069  */
1070 static int
1071 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1072                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1073                                  const struct rte_flow_action *action,
1074                                  struct rte_flow_error *error)
1075 {
1076         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1077         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1078         struct rte_flow_item item = {
1079                 .spec = NULL,
1080                 .mask = &mask,
1081         };
1082         struct field_modify_info reg_src[] = {
1083                 {4, 0, reg_to_field[conf->src]},
1084                 {0, 0, 0},
1085         };
1086         struct field_modify_info reg_dst = {
1087                 .offset = 0,
1088                 .id = reg_to_field[conf->dst],
1089         };
1090         /* Adjust reg_c[0] usage according to reported mask. */
1091         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1092                 struct mlx5_priv *priv = dev->data->dev_private;
1093                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1094
1095                 MLX5_ASSERT(reg_c0);
1096                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1097                 if (conf->dst == REG_C_0) {
1098                         /* Copy to reg_c[0], within mask only. */
1099                         reg_dst.offset = rte_bsf32(reg_c0);
1100                         /*
1101                          * Mask is ignoring the enianness, because
1102                          * there is no conversion in datapath.
1103                          */
1104 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1105                         /* Copy from destination lower bits to reg_c[0]. */
1106                         mask = reg_c0 >> reg_dst.offset;
1107 #else
1108                         /* Copy from destination upper bits to reg_c[0]. */
1109                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1110                                           rte_fls_u32(reg_c0));
1111 #endif
1112                 } else {
1113                         mask = rte_cpu_to_be_32(reg_c0);
1114 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1115                         /* Copy from reg_c[0] to destination lower bits. */
1116                         reg_dst.offset = 0;
1117 #else
1118                         /* Copy from reg_c[0] to destination upper bits. */
1119                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1120                                          (rte_fls_u32(reg_c0) -
1121                                           rte_bsf32(reg_c0));
1122 #endif
1123                 }
1124         }
1125         return flow_dv_convert_modify_action(&item,
1126                                              reg_src, &reg_dst, res,
1127                                              MLX5_MODIFICATION_TYPE_COPY,
1128                                              error);
1129 }
1130
1131 /**
1132  * Convert MARK action to DV specification. This routine is used
1133  * in extensive metadata only and requires metadata register to be
1134  * handled. In legacy mode hardware tag resource is engaged.
1135  *
1136  * @param[in] dev
1137  *   Pointer to the rte_eth_dev structure.
1138  * @param[in] conf
1139  *   Pointer to MARK action specification.
1140  * @param[in,out] resource
1141  *   Pointer to the modify-header resource.
1142  * @param[out] error
1143  *   Pointer to the error structure.
1144  *
1145  * @return
1146  *   0 on success, a negative errno value otherwise and rte_errno is set.
1147  */
1148 static int
1149 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1150                             const struct rte_flow_action_mark *conf,
1151                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1152                             struct rte_flow_error *error)
1153 {
1154         struct mlx5_priv *priv = dev->data->dev_private;
1155         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1156                                            priv->sh->dv_mark_mask);
1157         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1158         struct rte_flow_item item = {
1159                 .spec = &data,
1160                 .mask = &mask,
1161         };
1162         struct field_modify_info reg_c_x[] = {
1163                 [1] = {0, 0, 0},
1164         };
1165         int reg;
1166
1167         if (!mask)
1168                 return rte_flow_error_set(error, EINVAL,
1169                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1170                                           NULL, "zero mark action mask");
1171         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1172         if (reg < 0)
1173                 return reg;
1174         MLX5_ASSERT(reg > 0);
1175         if (reg == REG_C_0) {
1176                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1177                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1178
1179                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1180                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1181                 mask = rte_cpu_to_be_32(mask << shl_c0);
1182         }
1183         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1184         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1185                                              MLX5_MODIFICATION_TYPE_SET, error);
1186 }
1187
1188 /**
1189  * Get metadata register index for specified steering domain.
1190  *
1191  * @param[in] dev
1192  *   Pointer to the rte_eth_dev structure.
1193  * @param[in] attr
1194  *   Attributes of flow to determine steering domain.
1195  * @param[out] error
1196  *   Pointer to the error structure.
1197  *
1198  * @return
1199  *   positive index on success, a negative errno value otherwise
1200  *   and rte_errno is set.
1201  */
1202 static enum modify_reg
1203 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1204                          const struct rte_flow_attr *attr,
1205                          struct rte_flow_error *error)
1206 {
1207         int reg =
1208                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1209                                           MLX5_METADATA_FDB :
1210                                             attr->egress ?
1211                                             MLX5_METADATA_TX :
1212                                             MLX5_METADATA_RX, 0, error);
1213         if (reg < 0)
1214                 return rte_flow_error_set(error,
1215                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1216                                           NULL, "unavailable "
1217                                           "metadata register");
1218         return reg;
1219 }
1220
1221 /**
1222  * Convert SET_META action to DV specification.
1223  *
1224  * @param[in] dev
1225  *   Pointer to the rte_eth_dev structure.
1226  * @param[in,out] resource
1227  *   Pointer to the modify-header resource.
1228  * @param[in] attr
1229  *   Attributes of flow that includes this item.
1230  * @param[in] conf
1231  *   Pointer to action specification.
1232  * @param[out] error
1233  *   Pointer to the error structure.
1234  *
1235  * @return
1236  *   0 on success, a negative errno value otherwise and rte_errno is set.
1237  */
1238 static int
1239 flow_dv_convert_action_set_meta
1240                         (struct rte_eth_dev *dev,
1241                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1242                          const struct rte_flow_attr *attr,
1243                          const struct rte_flow_action_set_meta *conf,
1244                          struct rte_flow_error *error)
1245 {
1246         uint32_t data = conf->data;
1247         uint32_t mask = conf->mask;
1248         struct rte_flow_item item = {
1249                 .spec = &data,
1250                 .mask = &mask,
1251         };
1252         struct field_modify_info reg_c_x[] = {
1253                 [1] = {0, 0, 0},
1254         };
1255         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1256
1257         if (reg < 0)
1258                 return reg;
1259         /*
1260          * In datapath code there is no endianness
1261          * coversions for perfromance reasons, all
1262          * pattern conversions are done in rte_flow.
1263          */
1264         if (reg == REG_C_0) {
1265                 struct mlx5_priv *priv = dev->data->dev_private;
1266                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1267                 uint32_t shl_c0;
1268
1269                 MLX5_ASSERT(msk_c0);
1270 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1271                 shl_c0 = rte_bsf32(msk_c0);
1272 #else
1273                 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1274 #endif
1275                 mask <<= shl_c0;
1276                 data <<= shl_c0;
1277                 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1278         }
1279         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1280         /* The routine expects parameters in memory as big-endian ones. */
1281         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1282                                              MLX5_MODIFICATION_TYPE_SET, error);
1283 }
1284
1285 /**
1286  * Convert modify-header set IPv4 DSCP action to DV specification.
1287  *
1288  * @param[in,out] resource
1289  *   Pointer to the modify-header resource.
1290  * @param[in] action
1291  *   Pointer to action specification.
1292  * @param[out] error
1293  *   Pointer to the error structure.
1294  *
1295  * @return
1296  *   0 on success, a negative errno value otherwise and rte_errno is set.
1297  */
1298 static int
1299 flow_dv_convert_action_modify_ipv4_dscp
1300                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1301                          const struct rte_flow_action *action,
1302                          struct rte_flow_error *error)
1303 {
1304         const struct rte_flow_action_set_dscp *conf =
1305                 (const struct rte_flow_action_set_dscp *)(action->conf);
1306         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1307         struct rte_flow_item_ipv4 ipv4;
1308         struct rte_flow_item_ipv4 ipv4_mask;
1309
1310         memset(&ipv4, 0, sizeof(ipv4));
1311         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1312         ipv4.hdr.type_of_service = conf->dscp;
1313         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1314         item.spec = &ipv4;
1315         item.mask = &ipv4_mask;
1316         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1317                                              MLX5_MODIFICATION_TYPE_SET, error);
1318 }
1319
1320 /**
1321  * Convert modify-header set IPv6 DSCP action to DV specification.
1322  *
1323  * @param[in,out] resource
1324  *   Pointer to the modify-header resource.
1325  * @param[in] action
1326  *   Pointer to action specification.
1327  * @param[out] error
1328  *   Pointer to the error structure.
1329  *
1330  * @return
1331  *   0 on success, a negative errno value otherwise and rte_errno is set.
1332  */
1333 static int
1334 flow_dv_convert_action_modify_ipv6_dscp
1335                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1336                          const struct rte_flow_action *action,
1337                          struct rte_flow_error *error)
1338 {
1339         const struct rte_flow_action_set_dscp *conf =
1340                 (const struct rte_flow_action_set_dscp *)(action->conf);
1341         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1342         struct rte_flow_item_ipv6 ipv6;
1343         struct rte_flow_item_ipv6 ipv6_mask;
1344
1345         memset(&ipv6, 0, sizeof(ipv6));
1346         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1347         /*
1348          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1349          * rdma-core only accept the DSCP bits byte aligned start from
1350          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1351          * bits in IPv6 case as rdma-core requires byte aligned value.
1352          */
1353         ipv6.hdr.vtc_flow = conf->dscp;
1354         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1355         item.spec = &ipv6;
1356         item.mask = &ipv6_mask;
1357         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1358                                              MLX5_MODIFICATION_TYPE_SET, error);
1359 }
1360
1361 /**
1362  * Validate MARK item.
1363  *
1364  * @param[in] dev
1365  *   Pointer to the rte_eth_dev structure.
1366  * @param[in] item
1367  *   Item specification.
1368  * @param[in] attr
1369  *   Attributes of flow that includes this item.
1370  * @param[out] error
1371  *   Pointer to error structure.
1372  *
1373  * @return
1374  *   0 on success, a negative errno value otherwise and rte_errno is set.
1375  */
1376 static int
1377 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1378                            const struct rte_flow_item *item,
1379                            const struct rte_flow_attr *attr __rte_unused,
1380                            struct rte_flow_error *error)
1381 {
1382         struct mlx5_priv *priv = dev->data->dev_private;
1383         struct mlx5_dev_config *config = &priv->config;
1384         const struct rte_flow_item_mark *spec = item->spec;
1385         const struct rte_flow_item_mark *mask = item->mask;
1386         const struct rte_flow_item_mark nic_mask = {
1387                 .id = priv->sh->dv_mark_mask,
1388         };
1389         int ret;
1390
1391         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1392                 return rte_flow_error_set(error, ENOTSUP,
1393                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1394                                           "extended metadata feature"
1395                                           " isn't enabled");
1396         if (!mlx5_flow_ext_mreg_supported(dev))
1397                 return rte_flow_error_set(error, ENOTSUP,
1398                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1399                                           "extended metadata register"
1400                                           " isn't supported");
1401         if (!nic_mask.id)
1402                 return rte_flow_error_set(error, ENOTSUP,
1403                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1404                                           "extended metadata register"
1405                                           " isn't available");
1406         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1407         if (ret < 0)
1408                 return ret;
1409         if (!spec)
1410                 return rte_flow_error_set(error, EINVAL,
1411                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1412                                           item->spec,
1413                                           "data cannot be empty");
1414         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1415                 return rte_flow_error_set(error, EINVAL,
1416                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1417                                           &spec->id,
1418                                           "mark id exceeds the limit");
1419         if (!mask)
1420                 mask = &nic_mask;
1421         if (!mask->id)
1422                 return rte_flow_error_set(error, EINVAL,
1423                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1424                                         "mask cannot be zero");
1425
1426         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1427                                         (const uint8_t *)&nic_mask,
1428                                         sizeof(struct rte_flow_item_mark),
1429                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1430         if (ret < 0)
1431                 return ret;
1432         return 0;
1433 }
1434
1435 /**
1436  * Validate META item.
1437  *
1438  * @param[in] dev
1439  *   Pointer to the rte_eth_dev structure.
1440  * @param[in] item
1441  *   Item specification.
1442  * @param[in] attr
1443  *   Attributes of flow that includes this item.
1444  * @param[out] error
1445  *   Pointer to error structure.
1446  *
1447  * @return
1448  *   0 on success, a negative errno value otherwise and rte_errno is set.
1449  */
1450 static int
1451 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1452                            const struct rte_flow_item *item,
1453                            const struct rte_flow_attr *attr,
1454                            struct rte_flow_error *error)
1455 {
1456         struct mlx5_priv *priv = dev->data->dev_private;
1457         struct mlx5_dev_config *config = &priv->config;
1458         const struct rte_flow_item_meta *spec = item->spec;
1459         const struct rte_flow_item_meta *mask = item->mask;
1460         struct rte_flow_item_meta nic_mask = {
1461                 .data = UINT32_MAX
1462         };
1463         int reg;
1464         int ret;
1465
1466         if (!spec)
1467                 return rte_flow_error_set(error, EINVAL,
1468                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1469                                           item->spec,
1470                                           "data cannot be empty");
1471         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1472                 if (!mlx5_flow_ext_mreg_supported(dev))
1473                         return rte_flow_error_set(error, ENOTSUP,
1474                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1475                                           "extended metadata register"
1476                                           " isn't supported");
1477                 reg = flow_dv_get_metadata_reg(dev, attr, error);
1478                 if (reg < 0)
1479                         return reg;
1480                 if (reg == REG_B)
1481                         return rte_flow_error_set(error, ENOTSUP,
1482                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1483                                           "match on reg_b "
1484                                           "isn't supported");
1485                 if (reg != REG_A)
1486                         nic_mask.data = priv->sh->dv_meta_mask;
1487         } else if (attr->transfer) {
1488                 return rte_flow_error_set(error, ENOTSUP,
1489                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1490                                         "extended metadata feature "
1491                                         "should be enabled when "
1492                                         "meta item is requested "
1493                                         "with e-switch mode ");
1494         }
1495         if (!mask)
1496                 mask = &rte_flow_item_meta_mask;
1497         if (!mask->data)
1498                 return rte_flow_error_set(error, EINVAL,
1499                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1500                                         "mask cannot be zero");
1501
1502         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1503                                         (const uint8_t *)&nic_mask,
1504                                         sizeof(struct rte_flow_item_meta),
1505                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1506         return ret;
1507 }
1508
1509 /**
1510  * Validate TAG item.
1511  *
1512  * @param[in] dev
1513  *   Pointer to the rte_eth_dev structure.
1514  * @param[in] item
1515  *   Item specification.
1516  * @param[in] attr
1517  *   Attributes of flow that includes this item.
1518  * @param[out] error
1519  *   Pointer to error structure.
1520  *
1521  * @return
1522  *   0 on success, a negative errno value otherwise and rte_errno is set.
1523  */
1524 static int
1525 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1526                           const struct rte_flow_item *item,
1527                           const struct rte_flow_attr *attr __rte_unused,
1528                           struct rte_flow_error *error)
1529 {
1530         const struct rte_flow_item_tag *spec = item->spec;
1531         const struct rte_flow_item_tag *mask = item->mask;
1532         const struct rte_flow_item_tag nic_mask = {
1533                 .data = RTE_BE32(UINT32_MAX),
1534                 .index = 0xff,
1535         };
1536         int ret;
1537
1538         if (!mlx5_flow_ext_mreg_supported(dev))
1539                 return rte_flow_error_set(error, ENOTSUP,
1540                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1541                                           "extensive metadata register"
1542                                           " isn't supported");
1543         if (!spec)
1544                 return rte_flow_error_set(error, EINVAL,
1545                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1546                                           item->spec,
1547                                           "data cannot be empty");
1548         if (!mask)
1549                 mask = &rte_flow_item_tag_mask;
1550         if (!mask->data)
1551                 return rte_flow_error_set(error, EINVAL,
1552                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1553                                         "mask cannot be zero");
1554
1555         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1556                                         (const uint8_t *)&nic_mask,
1557                                         sizeof(struct rte_flow_item_tag),
1558                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1559         if (ret < 0)
1560                 return ret;
1561         if (mask->index != 0xff)
1562                 return rte_flow_error_set(error, EINVAL,
1563                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1564                                           "partial mask for tag index"
1565                                           " is not supported");
1566         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1567         if (ret < 0)
1568                 return ret;
1569         MLX5_ASSERT(ret != REG_NON);
1570         return 0;
1571 }
1572
1573 /**
1574  * Validate vport item.
1575  *
1576  * @param[in] dev
1577  *   Pointer to the rte_eth_dev structure.
1578  * @param[in] item
1579  *   Item specification.
1580  * @param[in] attr
1581  *   Attributes of flow that includes this item.
1582  * @param[in] item_flags
1583  *   Bit-fields that holds the items detected until now.
1584  * @param[out] error
1585  *   Pointer to error structure.
1586  *
1587  * @return
1588  *   0 on success, a negative errno value otherwise and rte_errno is set.
1589  */
1590 static int
1591 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1592                               const struct rte_flow_item *item,
1593                               const struct rte_flow_attr *attr,
1594                               uint64_t item_flags,
1595                               struct rte_flow_error *error)
1596 {
1597         const struct rte_flow_item_port_id *spec = item->spec;
1598         const struct rte_flow_item_port_id *mask = item->mask;
1599         const struct rte_flow_item_port_id switch_mask = {
1600                         .id = 0xffffffff,
1601         };
1602         struct mlx5_priv *esw_priv;
1603         struct mlx5_priv *dev_priv;
1604         int ret;
1605
1606         if (!attr->transfer)
1607                 return rte_flow_error_set(error, EINVAL,
1608                                           RTE_FLOW_ERROR_TYPE_ITEM,
1609                                           NULL,
1610                                           "match on port id is valid only"
1611                                           " when transfer flag is enabled");
1612         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1613                 return rte_flow_error_set(error, ENOTSUP,
1614                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1615                                           "multiple source ports are not"
1616                                           " supported");
1617         if (!mask)
1618                 mask = &switch_mask;
1619         if (mask->id != 0xffffffff)
1620                 return rte_flow_error_set(error, ENOTSUP,
1621                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1622                                            mask,
1623                                            "no support for partial mask on"
1624                                            " \"id\" field");
1625         ret = mlx5_flow_item_acceptable
1626                                 (item, (const uint8_t *)mask,
1627                                  (const uint8_t *)&rte_flow_item_port_id_mask,
1628                                  sizeof(struct rte_flow_item_port_id),
1629                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1630         if (ret)
1631                 return ret;
1632         if (!spec)
1633                 return 0;
1634         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1635         if (!esw_priv)
1636                 return rte_flow_error_set(error, rte_errno,
1637                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1638                                           "failed to obtain E-Switch info for"
1639                                           " port");
1640         dev_priv = mlx5_dev_to_eswitch_info(dev);
1641         if (!dev_priv)
1642                 return rte_flow_error_set(error, rte_errno,
1643                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1644                                           NULL,
1645                                           "failed to obtain E-Switch info");
1646         if (esw_priv->domain_id != dev_priv->domain_id)
1647                 return rte_flow_error_set(error, EINVAL,
1648                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1649                                           "cannot match on a port from a"
1650                                           " different E-Switch");
1651         return 0;
1652 }
1653
1654 /**
1655  * Validate VLAN item.
1656  *
1657  * @param[in] item
1658  *   Item specification.
1659  * @param[in] item_flags
1660  *   Bit-fields that holds the items detected until now.
1661  * @param[in] dev
1662  *   Ethernet device flow is being created on.
1663  * @param[out] error
1664  *   Pointer to error structure.
1665  *
1666  * @return
1667  *   0 on success, a negative errno value otherwise and rte_errno is set.
1668  */
1669 static int
1670 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
1671                            uint64_t item_flags,
1672                            struct rte_eth_dev *dev,
1673                            struct rte_flow_error *error)
1674 {
1675         const struct rte_flow_item_vlan *mask = item->mask;
1676         const struct rte_flow_item_vlan nic_mask = {
1677                 .tci = RTE_BE16(UINT16_MAX),
1678                 .inner_type = RTE_BE16(UINT16_MAX),
1679         };
1680         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1681         int ret;
1682         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1683                                         MLX5_FLOW_LAYER_INNER_L4) :
1684                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1685                                         MLX5_FLOW_LAYER_OUTER_L4);
1686         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1687                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1688
1689         if (item_flags & vlanm)
1690                 return rte_flow_error_set(error, EINVAL,
1691                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1692                                           "multiple VLAN layers not supported");
1693         else if ((item_flags & l34m) != 0)
1694                 return rte_flow_error_set(error, EINVAL,
1695                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1696                                           "VLAN cannot follow L3/L4 layer");
1697         if (!mask)
1698                 mask = &rte_flow_item_vlan_mask;
1699         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1700                                         (const uint8_t *)&nic_mask,
1701                                         sizeof(struct rte_flow_item_vlan),
1702                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1703         if (ret)
1704                 return ret;
1705         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1706                 struct mlx5_priv *priv = dev->data->dev_private;
1707
1708                 if (priv->vmwa_context) {
1709                         /*
1710                          * Non-NULL context means we have a virtual machine
1711                          * and SR-IOV enabled, we have to create VLAN interface
1712                          * to make hypervisor to setup E-Switch vport
1713                          * context correctly. We avoid creating the multiple
1714                          * VLAN interfaces, so we cannot support VLAN tag mask.
1715                          */
1716                         return rte_flow_error_set(error, EINVAL,
1717                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1718                                                   item,
1719                                                   "VLAN tag mask is not"
1720                                                   " supported in virtual"
1721                                                   " environment");
1722                 }
1723         }
1724         return 0;
1725 }
1726
1727 /*
1728  * GTP flags are contained in 1 byte of the format:
1729  * -------------------------------------------
1730  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
1731  * |-----------------------------------------|
1732  * | value | Version | PT | Res | E | S | PN |
1733  * -------------------------------------------
1734  *
1735  * Matching is supported only for GTP flags E, S, PN.
1736  */
1737 #define MLX5_GTP_FLAGS_MASK     0x07
1738
1739 /**
1740  * Validate GTP item.
1741  *
1742  * @param[in] dev
1743  *   Pointer to the rte_eth_dev structure.
1744  * @param[in] item
1745  *   Item specification.
1746  * @param[in] item_flags
1747  *   Bit-fields that holds the items detected until now.
1748  * @param[out] error
1749  *   Pointer to error structure.
1750  *
1751  * @return
1752  *   0 on success, a negative errno value otherwise and rte_errno is set.
1753  */
1754 static int
1755 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
1756                           const struct rte_flow_item *item,
1757                           uint64_t item_flags,
1758                           struct rte_flow_error *error)
1759 {
1760         struct mlx5_priv *priv = dev->data->dev_private;
1761         const struct rte_flow_item_gtp *spec = item->spec;
1762         const struct rte_flow_item_gtp *mask = item->mask;
1763         const struct rte_flow_item_gtp nic_mask = {
1764                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
1765                 .msg_type = 0xff,
1766                 .teid = RTE_BE32(0xffffffff),
1767         };
1768
1769         if (!priv->config.hca_attr.tunnel_stateless_gtp)
1770                 return rte_flow_error_set(error, ENOTSUP,
1771                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1772                                           "GTP support is not enabled");
1773         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1774                 return rte_flow_error_set(error, ENOTSUP,
1775                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1776                                           "multiple tunnel layers not"
1777                                           " supported");
1778         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1779                 return rte_flow_error_set(error, EINVAL,
1780                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1781                                           "no outer UDP layer found");
1782         if (!mask)
1783                 mask = &rte_flow_item_gtp_mask;
1784         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
1785                 return rte_flow_error_set(error, ENOTSUP,
1786                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1787                                           "Match is supported for GTP"
1788                                           " flags only");
1789         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1790                                          (const uint8_t *)&nic_mask,
1791                                          sizeof(struct rte_flow_item_gtp),
1792                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1793 }
1794
1795 /**
1796  * Validate IPV4 item.
1797  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
1798  * add specific validation of fragment_offset field,
1799  *
1800  * @param[in] item
1801  *   Item specification.
1802  * @param[in] item_flags
1803  *   Bit-fields that holds the items detected until now.
1804  * @param[out] error
1805  *   Pointer to error structure.
1806  *
1807  * @return
1808  *   0 on success, a negative errno value otherwise and rte_errno is set.
1809  */
1810 static int
1811 flow_dv_validate_item_ipv4(const struct rte_flow_item *item,
1812                            uint64_t item_flags,
1813                            uint64_t last_item,
1814                            uint16_t ether_type,
1815                            struct rte_flow_error *error)
1816 {
1817         int ret;
1818         const struct rte_flow_item_ipv4 *spec = item->spec;
1819         const struct rte_flow_item_ipv4 *last = item->last;
1820         const struct rte_flow_item_ipv4 *mask = item->mask;
1821         rte_be16_t fragment_offset_spec = 0;
1822         rte_be16_t fragment_offset_last = 0;
1823         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
1824                 .hdr = {
1825                         .src_addr = RTE_BE32(0xffffffff),
1826                         .dst_addr = RTE_BE32(0xffffffff),
1827                         .type_of_service = 0xff,
1828                         .fragment_offset = RTE_BE16(0xffff),
1829                         .next_proto_id = 0xff,
1830                         .time_to_live = 0xff,
1831                 },
1832         };
1833
1834         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
1835                                            ether_type, &nic_ipv4_mask,
1836                                            MLX5_ITEM_RANGE_ACCEPTED, error);
1837         if (ret < 0)
1838                 return ret;
1839         if (spec && mask)
1840                 fragment_offset_spec = spec->hdr.fragment_offset &
1841                                        mask->hdr.fragment_offset;
1842         if (!fragment_offset_spec)
1843                 return 0;
1844         /*
1845          * spec and mask are valid, enforce using full mask to make sure the
1846          * complete value is used correctly.
1847          */
1848         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1849                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1850                 return rte_flow_error_set(error, EINVAL,
1851                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1852                                           item, "must use full mask for"
1853                                           " fragment_offset");
1854         /*
1855          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
1856          * indicating this is 1st fragment of fragmented packet.
1857          * This is not yet supported in MLX5, return appropriate error message.
1858          */
1859         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
1860                 return rte_flow_error_set(error, ENOTSUP,
1861                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1862                                           "match on first fragment not "
1863                                           "supported");
1864         if (fragment_offset_spec && !last)
1865                 return rte_flow_error_set(error, ENOTSUP,
1866                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1867                                           "specified value not supported");
1868         /* spec and last are valid, validate the specified range. */
1869         fragment_offset_last = last->hdr.fragment_offset &
1870                                mask->hdr.fragment_offset;
1871         /*
1872          * Match on fragment_offset spec 0x2001 and last 0x3fff
1873          * means MF is 1 and frag-offset is > 0.
1874          * This packet is fragment 2nd and onward, excluding last.
1875          * This is not yet supported in MLX5, return appropriate
1876          * error message.
1877          */
1878         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
1879             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1880                 return rte_flow_error_set(error, ENOTSUP,
1881                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1882                                           last, "match on following "
1883                                           "fragments not supported");
1884         /*
1885          * Match on fragment_offset spec 0x0001 and last 0x1fff
1886          * means MF is 0 and frag-offset is > 0.
1887          * This packet is last fragment of fragmented packet.
1888          * This is not yet supported in MLX5, return appropriate
1889          * error message.
1890          */
1891         if (fragment_offset_spec == RTE_BE16(1) &&
1892             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
1893                 return rte_flow_error_set(error, ENOTSUP,
1894                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1895                                           last, "match on last "
1896                                           "fragment not supported");
1897         /*
1898          * Match on fragment_offset spec 0x0001 and last 0x3fff
1899          * means MF and/or frag-offset is not 0.
1900          * This is a fragmented packet.
1901          * Other range values are invalid and rejected.
1902          */
1903         if (!(fragment_offset_spec == RTE_BE16(1) &&
1904               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
1905                 return rte_flow_error_set(error, ENOTSUP,
1906                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
1907                                           "specified range not supported");
1908         return 0;
1909 }
1910
1911 /**
1912  * Validate IPV6 fragment extension item.
1913  *
1914  * @param[in] item
1915  *   Item specification.
1916  * @param[in] item_flags
1917  *   Bit-fields that holds the items detected until now.
1918  * @param[out] error
1919  *   Pointer to error structure.
1920  *
1921  * @return
1922  *   0 on success, a negative errno value otherwise and rte_errno is set.
1923  */
1924 static int
1925 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
1926                                     uint64_t item_flags,
1927                                     struct rte_flow_error *error)
1928 {
1929         const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
1930         const struct rte_flow_item_ipv6_frag_ext *last = item->last;
1931         const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
1932         rte_be16_t frag_data_spec = 0;
1933         rte_be16_t frag_data_last = 0;
1934         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1935         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1936                                       MLX5_FLOW_LAYER_OUTER_L4;
1937         int ret = 0;
1938         struct rte_flow_item_ipv6_frag_ext nic_mask = {
1939                 .hdr = {
1940                         .next_header = 0xff,
1941                         .frag_data = RTE_BE16(0xffff),
1942                 },
1943         };
1944
1945         if (item_flags & l4m)
1946                 return rte_flow_error_set(error, EINVAL,
1947                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1948                                           "ipv6 fragment extension item cannot "
1949                                           "follow L4 item.");
1950         if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
1951             (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
1952                 return rte_flow_error_set(error, EINVAL,
1953                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1954                                           "ipv6 fragment extension item must "
1955                                           "follow ipv6 item");
1956         if (spec && mask)
1957                 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
1958         if (!frag_data_spec)
1959                 return 0;
1960         /*
1961          * spec and mask are valid, enforce using full mask to make sure the
1962          * complete value is used correctly.
1963          */
1964         if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
1965                                 RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
1966                 return rte_flow_error_set(error, EINVAL,
1967                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1968                                           item, "must use full mask for"
1969                                           " frag_data");
1970         /*
1971          * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
1972          * This is 1st fragment of fragmented packet.
1973          */
1974         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
1975                 return rte_flow_error_set(error, ENOTSUP,
1976                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1977                                           "match on first fragment not "
1978                                           "supported");
1979         if (frag_data_spec && !last)
1980                 return rte_flow_error_set(error, EINVAL,
1981                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1982                                           "specified value not supported");
1983         ret = mlx5_flow_item_acceptable
1984                                 (item, (const uint8_t *)mask,
1985                                  (const uint8_t *)&nic_mask,
1986                                  sizeof(struct rte_flow_item_ipv6_frag_ext),
1987                                  MLX5_ITEM_RANGE_ACCEPTED, error);
1988         if (ret)
1989                 return ret;
1990         /* spec and last are valid, validate the specified range. */
1991         frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
1992         /*
1993          * Match on frag_data spec 0x0009 and last 0xfff9
1994          * means M is 1 and frag-offset is > 0.
1995          * This packet is fragment 2nd and onward, excluding last.
1996          * This is not yet supported in MLX5, return appropriate
1997          * error message.
1998          */
1999         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
2000                                        RTE_IPV6_EHDR_MF_MASK) &&
2001             frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2002                 return rte_flow_error_set(error, ENOTSUP,
2003                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2004                                           last, "match on following "
2005                                           "fragments not supported");
2006         /*
2007          * Match on frag_data spec 0x0008 and last 0xfff8
2008          * means M is 0 and frag-offset is > 0.
2009          * This packet is last fragment of fragmented packet.
2010          * This is not yet supported in MLX5, return appropriate
2011          * error message.
2012          */
2013         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
2014             frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
2015                 return rte_flow_error_set(error, ENOTSUP,
2016                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2017                                           last, "match on last "
2018                                           "fragment not supported");
2019         /* Other range values are invalid and rejected. */
2020         return rte_flow_error_set(error, EINVAL,
2021                                   RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2022                                   "specified range not supported");
2023 }
2024
2025 /**
2026  * Validate the pop VLAN action.
2027  *
2028  * @param[in] dev
2029  *   Pointer to the rte_eth_dev structure.
2030  * @param[in] action_flags
2031  *   Holds the actions detected until now.
2032  * @param[in] action
2033  *   Pointer to the pop vlan action.
2034  * @param[in] item_flags
2035  *   The items found in this flow rule.
2036  * @param[in] attr
2037  *   Pointer to flow attributes.
2038  * @param[out] error
2039  *   Pointer to error structure.
2040  *
2041  * @return
2042  *   0 on success, a negative errno value otherwise and rte_errno is set.
2043  */
2044 static int
2045 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
2046                                  uint64_t action_flags,
2047                                  const struct rte_flow_action *action,
2048                                  uint64_t item_flags,
2049                                  const struct rte_flow_attr *attr,
2050                                  struct rte_flow_error *error)
2051 {
2052         const struct mlx5_priv *priv = dev->data->dev_private;
2053
2054         (void)action;
2055         (void)attr;
2056         if (!priv->sh->pop_vlan_action)
2057                 return rte_flow_error_set(error, ENOTSUP,
2058                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2059                                           NULL,
2060                                           "pop vlan action is not supported");
2061         if (attr->egress)
2062                 return rte_flow_error_set(error, ENOTSUP,
2063                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2064                                           NULL,
2065                                           "pop vlan action not supported for "
2066                                           "egress");
2067         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
2068                 return rte_flow_error_set(error, ENOTSUP,
2069                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2070                                           "no support for multiple VLAN "
2071                                           "actions");
2072         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
2073         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
2074             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
2075                 return rte_flow_error_set(error, ENOTSUP,
2076                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2077                                           NULL,
2078                                           "cannot pop vlan after decap without "
2079                                           "match on inner vlan in the flow");
2080         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
2081         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
2082             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2083                 return rte_flow_error_set(error, ENOTSUP,
2084                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2085                                           NULL,
2086                                           "cannot pop vlan without a "
2087                                           "match on (outer) vlan in the flow");
2088         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2089                 return rte_flow_error_set(error, EINVAL,
2090                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2091                                           "wrong action order, port_id should "
2092                                           "be after pop VLAN action");
2093         if (!attr->transfer && priv->representor)
2094                 return rte_flow_error_set(error, ENOTSUP,
2095                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2096                                           "pop vlan action for VF representor "
2097                                           "not supported on NIC table");
2098         return 0;
2099 }
2100
2101 /**
2102  * Get VLAN default info from vlan match info.
2103  *
2104  * @param[in] items
2105  *   the list of item specifications.
2106  * @param[out] vlan
2107  *   pointer VLAN info to fill to.
2108  *
2109  * @return
2110  *   0 on success, a negative errno value otherwise and rte_errno is set.
2111  */
2112 static void
2113 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2114                                   struct rte_vlan_hdr *vlan)
2115 {
2116         const struct rte_flow_item_vlan nic_mask = {
2117                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2118                                 MLX5DV_FLOW_VLAN_VID_MASK),
2119                 .inner_type = RTE_BE16(0xffff),
2120         };
2121
2122         if (items == NULL)
2123                 return;
2124         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2125                 int type = items->type;
2126
2127                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2128                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2129                         break;
2130         }
2131         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2132                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2133                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2134
2135                 /* If VLAN item in pattern doesn't contain data, return here. */
2136                 if (!vlan_v)
2137                         return;
2138                 if (!vlan_m)
2139                         vlan_m = &nic_mask;
2140                 /* Only full match values are accepted */
2141                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2142                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2143                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2144                         vlan->vlan_tci |=
2145                                 rte_be_to_cpu_16(vlan_v->tci &
2146                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2147                 }
2148                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2149                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2150                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2151                         vlan->vlan_tci |=
2152                                 rte_be_to_cpu_16(vlan_v->tci &
2153                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2154                 }
2155                 if (vlan_m->inner_type == nic_mask.inner_type)
2156                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2157                                                            vlan_m->inner_type);
2158         }
2159 }
2160
2161 /**
2162  * Validate the push VLAN action.
2163  *
2164  * @param[in] dev
2165  *   Pointer to the rte_eth_dev structure.
2166  * @param[in] action_flags
2167  *   Holds the actions detected until now.
2168  * @param[in] item_flags
2169  *   The items found in this flow rule.
2170  * @param[in] action
2171  *   Pointer to the action structure.
2172  * @param[in] attr
2173  *   Pointer to flow attributes
2174  * @param[out] error
2175  *   Pointer to error structure.
2176  *
2177  * @return
2178  *   0 on success, a negative errno value otherwise and rte_errno is set.
2179  */
2180 static int
2181 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2182                                   uint64_t action_flags,
2183                                   const struct rte_flow_item_vlan *vlan_m,
2184                                   const struct rte_flow_action *action,
2185                                   const struct rte_flow_attr *attr,
2186                                   struct rte_flow_error *error)
2187 {
2188         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2189         const struct mlx5_priv *priv = dev->data->dev_private;
2190
2191         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2192             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2193                 return rte_flow_error_set(error, EINVAL,
2194                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2195                                           "invalid vlan ethertype");
2196         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2197                 return rte_flow_error_set(error, EINVAL,
2198                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2199                                           "wrong action order, port_id should "
2200                                           "be after push VLAN");
2201         if (!attr->transfer && priv->representor)
2202                 return rte_flow_error_set(error, ENOTSUP,
2203                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2204                                           "push vlan action for VF representor "
2205                                           "not supported on NIC table");
2206         if (vlan_m &&
2207             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2208             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2209                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2210             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2211             !(mlx5_flow_find_action
2212                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2213                 return rte_flow_error_set(error, EINVAL,
2214                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2215                                           "not full match mask on VLAN PCP and "
2216                                           "there is no of_set_vlan_pcp action, "
2217                                           "push VLAN action cannot figure out "
2218                                           "PCP value");
2219         if (vlan_m &&
2220             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2221             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2222                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2223             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2224             !(mlx5_flow_find_action
2225                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2226                 return rte_flow_error_set(error, EINVAL,
2227                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2228                                           "not full match mask on VLAN VID and "
2229                                           "there is no of_set_vlan_vid action, "
2230                                           "push VLAN action cannot figure out "
2231                                           "VID value");
2232         (void)attr;
2233         return 0;
2234 }
2235
2236 /**
2237  * Validate the set VLAN PCP.
2238  *
2239  * @param[in] action_flags
2240  *   Holds the actions detected until now.
2241  * @param[in] actions
2242  *   Pointer to the list of actions remaining in the flow rule.
2243  * @param[out] error
2244  *   Pointer to error structure.
2245  *
2246  * @return
2247  *   0 on success, a negative errno value otherwise and rte_errno is set.
2248  */
2249 static int
2250 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2251                                      const struct rte_flow_action actions[],
2252                                      struct rte_flow_error *error)
2253 {
2254         const struct rte_flow_action *action = actions;
2255         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2256
2257         if (conf->vlan_pcp > 7)
2258                 return rte_flow_error_set(error, EINVAL,
2259                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2260                                           "VLAN PCP value is too big");
2261         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2262                 return rte_flow_error_set(error, ENOTSUP,
2263                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2264                                           "set VLAN PCP action must follow "
2265                                           "the push VLAN action");
2266         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2267                 return rte_flow_error_set(error, ENOTSUP,
2268                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2269                                           "Multiple VLAN PCP modification are "
2270                                           "not supported");
2271         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2272                 return rte_flow_error_set(error, EINVAL,
2273                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2274                                           "wrong action order, port_id should "
2275                                           "be after set VLAN PCP");
2276         return 0;
2277 }
2278
2279 /**
2280  * Validate the set VLAN VID.
2281  *
2282  * @param[in] item_flags
2283  *   Holds the items detected in this rule.
2284  * @param[in] action_flags
2285  *   Holds the actions detected until now.
2286  * @param[in] actions
2287  *   Pointer to the list of actions remaining in the flow rule.
2288  * @param[out] error
2289  *   Pointer to error structure.
2290  *
2291  * @return
2292  *   0 on success, a negative errno value otherwise and rte_errno is set.
2293  */
2294 static int
2295 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2296                                      uint64_t action_flags,
2297                                      const struct rte_flow_action actions[],
2298                                      struct rte_flow_error *error)
2299 {
2300         const struct rte_flow_action *action = actions;
2301         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2302
2303         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2304                 return rte_flow_error_set(error, EINVAL,
2305                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2306                                           "VLAN VID value is too big");
2307         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2308             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2309                 return rte_flow_error_set(error, ENOTSUP,
2310                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2311                                           "set VLAN VID action must follow push"
2312                                           " VLAN action or match on VLAN item");
2313         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2314                 return rte_flow_error_set(error, ENOTSUP,
2315                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2316                                           "Multiple VLAN VID modifications are "
2317                                           "not supported");
2318         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2319                 return rte_flow_error_set(error, EINVAL,
2320                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2321                                           "wrong action order, port_id should "
2322                                           "be after set VLAN VID");
2323         return 0;
2324 }
2325
2326 /*
2327  * Validate the FLAG action.
2328  *
2329  * @param[in] dev
2330  *   Pointer to the rte_eth_dev structure.
2331  * @param[in] action_flags
2332  *   Holds the actions detected until now.
2333  * @param[in] attr
2334  *   Pointer to flow attributes
2335  * @param[out] error
2336  *   Pointer to error structure.
2337  *
2338  * @return
2339  *   0 on success, a negative errno value otherwise and rte_errno is set.
2340  */
2341 static int
2342 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
2343                              uint64_t action_flags,
2344                              const struct rte_flow_attr *attr,
2345                              struct rte_flow_error *error)
2346 {
2347         struct mlx5_priv *priv = dev->data->dev_private;
2348         struct mlx5_dev_config *config = &priv->config;
2349         int ret;
2350
2351         /* Fall back if no extended metadata register support. */
2352         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2353                 return mlx5_flow_validate_action_flag(action_flags, attr,
2354                                                       error);
2355         /* Extensive metadata mode requires registers. */
2356         if (!mlx5_flow_ext_mreg_supported(dev))
2357                 return rte_flow_error_set(error, ENOTSUP,
2358                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2359                                           "no metadata registers "
2360                                           "to support flag action");
2361         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
2362                 return rte_flow_error_set(error, ENOTSUP,
2363                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2364                                           "extended metadata register"
2365                                           " isn't available");
2366         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2367         if (ret < 0)
2368                 return ret;
2369         MLX5_ASSERT(ret > 0);
2370         if (action_flags & MLX5_FLOW_ACTION_MARK)
2371                 return rte_flow_error_set(error, EINVAL,
2372                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2373                                           "can't mark and flag in same flow");
2374         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2375                 return rte_flow_error_set(error, EINVAL,
2376                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2377                                           "can't have 2 flag"
2378                                           " actions in same flow");
2379         return 0;
2380 }
2381
2382 /**
2383  * Validate MARK action.
2384  *
2385  * @param[in] dev
2386  *   Pointer to the rte_eth_dev structure.
2387  * @param[in] action
2388  *   Pointer to action.
2389  * @param[in] action_flags
2390  *   Holds the actions detected until now.
2391  * @param[in] attr
2392  *   Pointer to flow attributes
2393  * @param[out] error
2394  *   Pointer to error structure.
2395  *
2396  * @return
2397  *   0 on success, a negative errno value otherwise and rte_errno is set.
2398  */
2399 static int
2400 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
2401                              const struct rte_flow_action *action,
2402                              uint64_t action_flags,
2403                              const struct rte_flow_attr *attr,
2404                              struct rte_flow_error *error)
2405 {
2406         struct mlx5_priv *priv = dev->data->dev_private;
2407         struct mlx5_dev_config *config = &priv->config;
2408         const struct rte_flow_action_mark *mark = action->conf;
2409         int ret;
2410
2411         /* Fall back if no extended metadata register support. */
2412         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2413                 return mlx5_flow_validate_action_mark(action, action_flags,
2414                                                       attr, error);
2415         /* Extensive metadata mode requires registers. */
2416         if (!mlx5_flow_ext_mreg_supported(dev))
2417                 return rte_flow_error_set(error, ENOTSUP,
2418                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2419                                           "no metadata registers "
2420                                           "to support mark action");
2421         if (!priv->sh->dv_mark_mask)
2422                 return rte_flow_error_set(error, ENOTSUP,
2423                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2424                                           "extended metadata register"
2425                                           " isn't available");
2426         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2427         if (ret < 0)
2428                 return ret;
2429         MLX5_ASSERT(ret > 0);
2430         if (!mark)
2431                 return rte_flow_error_set(error, EINVAL,
2432                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2433                                           "configuration cannot be null");
2434         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
2435                 return rte_flow_error_set(error, EINVAL,
2436                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2437                                           &mark->id,
2438                                           "mark id exceeds the limit");
2439         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2440                 return rte_flow_error_set(error, EINVAL,
2441                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2442                                           "can't flag and mark in same flow");
2443         if (action_flags & MLX5_FLOW_ACTION_MARK)
2444                 return rte_flow_error_set(error, EINVAL,
2445                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2446                                           "can't have 2 mark actions in same"
2447                                           " flow");
2448         return 0;
2449 }
2450
2451 /**
2452  * Validate SET_META action.
2453  *
2454  * @param[in] dev
2455  *   Pointer to the rte_eth_dev structure.
2456  * @param[in] action
2457  *   Pointer to the action structure.
2458  * @param[in] action_flags
2459  *   Holds the actions detected until now.
2460  * @param[in] attr
2461  *   Pointer to flow attributes
2462  * @param[out] error
2463  *   Pointer to error structure.
2464  *
2465  * @return
2466  *   0 on success, a negative errno value otherwise and rte_errno is set.
2467  */
2468 static int
2469 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
2470                                  const struct rte_flow_action *action,
2471                                  uint64_t action_flags __rte_unused,
2472                                  const struct rte_flow_attr *attr,
2473                                  struct rte_flow_error *error)
2474 {
2475         const struct rte_flow_action_set_meta *conf;
2476         uint32_t nic_mask = UINT32_MAX;
2477         int reg;
2478
2479         if (!mlx5_flow_ext_mreg_supported(dev))
2480                 return rte_flow_error_set(error, ENOTSUP,
2481                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2482                                           "extended metadata register"
2483                                           " isn't supported");
2484         reg = flow_dv_get_metadata_reg(dev, attr, error);
2485         if (reg < 0)
2486                 return reg;
2487         if (reg != REG_A && reg != REG_B) {
2488                 struct mlx5_priv *priv = dev->data->dev_private;
2489
2490                 nic_mask = priv->sh->dv_meta_mask;
2491         }
2492         if (!(action->conf))
2493                 return rte_flow_error_set(error, EINVAL,
2494                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2495                                           "configuration cannot be null");
2496         conf = (const struct rte_flow_action_set_meta *)action->conf;
2497         if (!conf->mask)
2498                 return rte_flow_error_set(error, EINVAL,
2499                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2500                                           "zero mask doesn't have any effect");
2501         if (conf->mask & ~nic_mask)
2502                 return rte_flow_error_set(error, EINVAL,
2503                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2504                                           "meta data must be within reg C0");
2505         return 0;
2506 }
2507
2508 /**
2509  * Validate SET_TAG action.
2510  *
2511  * @param[in] dev
2512  *   Pointer to the rte_eth_dev structure.
2513  * @param[in] action
2514  *   Pointer to the action structure.
2515  * @param[in] action_flags
2516  *   Holds the actions detected until now.
2517  * @param[in] attr
2518  *   Pointer to flow attributes
2519  * @param[out] error
2520  *   Pointer to error structure.
2521  *
2522  * @return
2523  *   0 on success, a negative errno value otherwise and rte_errno is set.
2524  */
2525 static int
2526 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
2527                                 const struct rte_flow_action *action,
2528                                 uint64_t action_flags,
2529                                 const struct rte_flow_attr *attr,
2530                                 struct rte_flow_error *error)
2531 {
2532         const struct rte_flow_action_set_tag *conf;
2533         const uint64_t terminal_action_flags =
2534                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
2535                 MLX5_FLOW_ACTION_RSS;
2536         int ret;
2537
2538         if (!mlx5_flow_ext_mreg_supported(dev))
2539                 return rte_flow_error_set(error, ENOTSUP,
2540                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2541                                           "extensive metadata register"
2542                                           " isn't supported");
2543         if (!(action->conf))
2544                 return rte_flow_error_set(error, EINVAL,
2545                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2546                                           "configuration cannot be null");
2547         conf = (const struct rte_flow_action_set_tag *)action->conf;
2548         if (!conf->mask)
2549                 return rte_flow_error_set(error, EINVAL,
2550                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2551                                           "zero mask doesn't have any effect");
2552         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
2553         if (ret < 0)
2554                 return ret;
2555         if (!attr->transfer && attr->ingress &&
2556             (action_flags & terminal_action_flags))
2557                 return rte_flow_error_set(error, EINVAL,
2558                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2559                                           "set_tag has no effect"
2560                                           " with terminal actions");
2561         return 0;
2562 }
2563
2564 /**
2565  * Validate count action.
2566  *
2567  * @param[in] dev
2568  *   Pointer to rte_eth_dev structure.
2569  * @param[out] error
2570  *   Pointer to error structure.
2571  *
2572  * @return
2573  *   0 on success, a negative errno value otherwise and rte_errno is set.
2574  */
2575 static int
2576 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2577                               struct rte_flow_error *error)
2578 {
2579         struct mlx5_priv *priv = dev->data->dev_private;
2580
2581         if (!priv->config.devx)
2582                 goto notsup_err;
2583 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2584         return 0;
2585 #endif
2586 notsup_err:
2587         return rte_flow_error_set
2588                       (error, ENOTSUP,
2589                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2590                        NULL,
2591                        "count action not supported");
2592 }
2593
2594 /**
2595  * Validate the L2 encap action.
2596  *
2597  * @param[in] dev
2598  *   Pointer to the rte_eth_dev structure.
2599  * @param[in] action_flags
2600  *   Holds the actions detected until now.
2601  * @param[in] action
2602  *   Pointer to the action structure.
2603  * @param[in] attr
2604  *   Pointer to flow attributes.
2605  * @param[out] error
2606  *   Pointer to error structure.
2607  *
2608  * @return
2609  *   0 on success, a negative errno value otherwise and rte_errno is set.
2610  */
2611 static int
2612 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
2613                                  uint64_t action_flags,
2614                                  const struct rte_flow_action *action,
2615                                  const struct rte_flow_attr *attr,
2616                                  struct rte_flow_error *error)
2617 {
2618         const struct mlx5_priv *priv = dev->data->dev_private;
2619
2620         if (!(action->conf))
2621                 return rte_flow_error_set(error, EINVAL,
2622                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2623                                           "configuration cannot be null");
2624         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
2625                 return rte_flow_error_set(error, EINVAL,
2626                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2627                                           "can only have a single encap action "
2628                                           "in a flow");
2629         if (!attr->transfer && priv->representor)
2630                 return rte_flow_error_set(error, ENOTSUP,
2631                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2632                                           "encap action for VF representor "
2633                                           "not supported on NIC table");
2634         return 0;
2635 }
2636
2637 /**
2638  * Validate a decap action.
2639  *
2640  * @param[in] dev
2641  *   Pointer to the rte_eth_dev structure.
2642  * @param[in] action_flags
2643  *   Holds the actions detected until now.
2644  * @param[in] attr
2645  *   Pointer to flow attributes
2646  * @param[out] error
2647  *   Pointer to error structure.
2648  *
2649  * @return
2650  *   0 on success, a negative errno value otherwise and rte_errno is set.
2651  */
2652 static int
2653 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
2654                               uint64_t action_flags,
2655                               const struct rte_flow_attr *attr,
2656                               struct rte_flow_error *error)
2657 {
2658         const struct mlx5_priv *priv = dev->data->dev_private;
2659
2660         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
2661             !priv->config.decap_en)
2662                 return rte_flow_error_set(error, ENOTSUP,
2663                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2664                                           "decap is not enabled");
2665         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
2666                 return rte_flow_error_set(error, ENOTSUP,
2667                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2668                                           action_flags &
2669                                           MLX5_FLOW_ACTION_DECAP ? "can only "
2670                                           "have a single decap action" : "decap "
2671                                           "after encap is not supported");
2672         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2673                 return rte_flow_error_set(error, EINVAL,
2674                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2675                                           "can't have decap action after"
2676                                           " modify action");
2677         if (attr->egress)
2678                 return rte_flow_error_set(error, ENOTSUP,
2679                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2680                                           NULL,
2681                                           "decap action not supported for "
2682                                           "egress");
2683         if (!attr->transfer && priv->representor)
2684                 return rte_flow_error_set(error, ENOTSUP,
2685                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2686                                           "decap action for VF representor "
2687                                           "not supported on NIC table");
2688         return 0;
2689 }
2690
2691 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
2692
2693 /**
2694  * Validate the raw encap and decap actions.
2695  *
2696  * @param[in] dev
2697  *   Pointer to the rte_eth_dev structure.
2698  * @param[in] decap
2699  *   Pointer to the decap action.
2700  * @param[in] encap
2701  *   Pointer to the encap action.
2702  * @param[in] attr
2703  *   Pointer to flow attributes
2704  * @param[in/out] action_flags
2705  *   Holds the actions detected until now.
2706  * @param[out] actions_n
2707  *   pointer to the number of actions counter.
2708  * @param[out] error
2709  *   Pointer to error structure.
2710  *
2711  * @return
2712  *   0 on success, a negative errno value otherwise and rte_errno is set.
2713  */
2714 static int
2715 flow_dv_validate_action_raw_encap_decap
2716         (struct rte_eth_dev *dev,
2717          const struct rte_flow_action_raw_decap *decap,
2718          const struct rte_flow_action_raw_encap *encap,
2719          const struct rte_flow_attr *attr, uint64_t *action_flags,
2720          int *actions_n, struct rte_flow_error *error)
2721 {
2722         const struct mlx5_priv *priv = dev->data->dev_private;
2723         int ret;
2724
2725         if (encap && (!encap->size || !encap->data))
2726                 return rte_flow_error_set(error, EINVAL,
2727                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2728                                           "raw encap data cannot be empty");
2729         if (decap && encap) {
2730                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
2731                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2732                         /* L3 encap. */
2733                         decap = NULL;
2734                 else if (encap->size <=
2735                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2736                            decap->size >
2737                            MLX5_ENCAPSULATION_DECISION_SIZE)
2738                         /* L3 decap. */
2739                         encap = NULL;
2740                 else if (encap->size >
2741                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2742                            decap->size >
2743                            MLX5_ENCAPSULATION_DECISION_SIZE)
2744                         /* 2 L2 actions: encap and decap. */
2745                         ;
2746                 else
2747                         return rte_flow_error_set(error,
2748                                 ENOTSUP,
2749                                 RTE_FLOW_ERROR_TYPE_ACTION,
2750                                 NULL, "unsupported too small "
2751                                 "raw decap and too small raw "
2752                                 "encap combination");
2753         }
2754         if (decap) {
2755                 ret = flow_dv_validate_action_decap(dev, *action_flags, attr,
2756                                                     error);
2757                 if (ret < 0)
2758                         return ret;
2759                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
2760                 ++(*actions_n);
2761         }
2762         if (encap) {
2763                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
2764                         return rte_flow_error_set(error, ENOTSUP,
2765                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2766                                                   NULL,
2767                                                   "small raw encap size");
2768                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
2769                         return rte_flow_error_set(error, EINVAL,
2770                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2771                                                   NULL,
2772                                                   "more than one encap action");
2773                 if (!attr->transfer && priv->representor)
2774                         return rte_flow_error_set
2775                                         (error, ENOTSUP,
2776                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2777                                          "encap action for VF representor "
2778                                          "not supported on NIC table");
2779                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
2780                 ++(*actions_n);
2781         }
2782         return 0;
2783 }
2784
2785 /**
2786  * Match encap_decap resource.
2787  *
2788  * @param entry
2789  *   Pointer to exist resource entry object.
2790  * @param ctx
2791  *   Pointer to new encap_decap resource.
2792  *
2793  * @return
2794  *   0 on matching, -1 otherwise.
2795  */
2796 static int
2797 flow_dv_encap_decap_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
2798 {
2799         struct mlx5_flow_dv_encap_decap_resource *resource;
2800         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2801
2802         resource = (struct mlx5_flow_dv_encap_decap_resource *)ctx;
2803         cache_resource = container_of(entry,
2804                                       struct mlx5_flow_dv_encap_decap_resource,
2805                                       entry);
2806         if (resource->entry.key == cache_resource->entry.key &&
2807             resource->reformat_type == cache_resource->reformat_type &&
2808             resource->ft_type == cache_resource->ft_type &&
2809             resource->flags == cache_resource->flags &&
2810             resource->size == cache_resource->size &&
2811             !memcmp((const void *)resource->buf,
2812                     (const void *)cache_resource->buf,
2813                     resource->size))
2814                 return 0;
2815         return -1;
2816 }
2817
2818 /**
2819  * Find existing encap/decap resource or create and register a new one.
2820  *
2821  * @param[in, out] dev
2822  *   Pointer to rte_eth_dev structure.
2823  * @param[in, out] resource
2824  *   Pointer to encap/decap resource.
2825  * @parm[in, out] dev_flow
2826  *   Pointer to the dev_flow.
2827  * @param[out] error
2828  *   pointer to error structure.
2829  *
2830  * @return
2831  *   0 on success otherwise -errno and errno is set.
2832  */
2833 static int
2834 flow_dv_encap_decap_resource_register
2835                         (struct rte_eth_dev *dev,
2836                          struct mlx5_flow_dv_encap_decap_resource *resource,
2837                          struct mlx5_flow *dev_flow,
2838                          struct rte_flow_error *error)
2839 {
2840         struct mlx5_priv *priv = dev->data->dev_private;
2841         struct mlx5_dev_ctx_shared *sh = priv->sh;
2842         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2843         struct mlx5dv_dr_domain *domain;
2844         struct mlx5_hlist_entry *entry;
2845         union mlx5_flow_encap_decap_key encap_decap_key = {
2846                 {
2847                         .ft_type = resource->ft_type,
2848                         .refmt_type = resource->reformat_type,
2849                         .buf_size = resource->size,
2850                         .table_level = !!dev_flow->dv.group,
2851                         .cksum = 0,
2852                 }
2853         };
2854         int ret;
2855
2856         resource->flags = dev_flow->dv.group ? 0 : 1;
2857         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2858                 domain = sh->fdb_domain;
2859         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2860                 domain = sh->rx_domain;
2861         else
2862                 domain = sh->tx_domain;
2863         encap_decap_key.cksum = __rte_raw_cksum(resource->buf,
2864                                                 resource->size, 0);
2865         resource->entry.key = encap_decap_key.v64;
2866         /* Lookup a matching resource from cache. */
2867         entry = mlx5_hlist_lookup_ex(sh->encaps_decaps, resource->entry.key,
2868                                      flow_dv_encap_decap_resource_match,
2869                                      (void *)resource);
2870         if (entry) {
2871                 cache_resource = container_of(entry,
2872                         struct mlx5_flow_dv_encap_decap_resource, entry);
2873                 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2874                         (void *)cache_resource,
2875                         rte_atomic32_read(&cache_resource->refcnt));
2876                 rte_atomic32_inc(&cache_resource->refcnt);
2877                 dev_flow->handle->dvh.rix_encap_decap = cache_resource->idx;
2878                 dev_flow->dv.encap_decap = cache_resource;
2879                 return 0;
2880         }
2881         /* Register new encap/decap resource. */
2882         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2883                                        &dev_flow->handle->dvh.rix_encap_decap);
2884         if (!cache_resource)
2885                 return rte_flow_error_set(error, ENOMEM,
2886                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2887                                           "cannot allocate resource memory");
2888         *cache_resource = *resource;
2889         cache_resource->idx = dev_flow->handle->dvh.rix_encap_decap;
2890         ret = mlx5_flow_os_create_flow_action_packet_reformat
2891                                         (sh->ctx, domain, cache_resource,
2892                                          &cache_resource->action);
2893         if (ret) {
2894                 mlx5_free(cache_resource);
2895                 return rte_flow_error_set(error, ENOMEM,
2896                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2897                                           NULL, "cannot create action");
2898         }
2899         rte_atomic32_init(&cache_resource->refcnt);
2900         rte_atomic32_inc(&cache_resource->refcnt);
2901         if (mlx5_hlist_insert_ex(sh->encaps_decaps, &cache_resource->entry,
2902                                  flow_dv_encap_decap_resource_match,
2903                                  (void *)cache_resource)) {
2904                 claim_zero(mlx5_flow_os_destroy_flow_action
2905                                                 (cache_resource->action));
2906                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2907                                 cache_resource->idx);
2908                 return rte_flow_error_set(error, EEXIST,
2909                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2910                                           NULL, "action exist");
2911         }
2912         dev_flow->dv.encap_decap = cache_resource;
2913         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2914                 (void *)cache_resource,
2915                 rte_atomic32_read(&cache_resource->refcnt));
2916         return 0;
2917 }
2918
2919 /**
2920  * Find existing table jump resource or create and register a new one.
2921  *
2922  * @param[in, out] dev
2923  *   Pointer to rte_eth_dev structure.
2924  * @param[in, out] tbl
2925  *   Pointer to flow table resource.
2926  * @parm[in, out] dev_flow
2927  *   Pointer to the dev_flow.
2928  * @param[out] error
2929  *   pointer to error structure.
2930  *
2931  * @return
2932  *   0 on success otherwise -errno and errno is set.
2933  */
2934 static int
2935 flow_dv_jump_tbl_resource_register
2936                         (struct rte_eth_dev *dev __rte_unused,
2937                          struct mlx5_flow_tbl_resource *tbl,
2938                          struct mlx5_flow *dev_flow,
2939                          struct rte_flow_error *error)
2940 {
2941         struct mlx5_flow_tbl_data_entry *tbl_data =
2942                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2943         int cnt, ret;
2944
2945         MLX5_ASSERT(tbl);
2946         cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
2947         if (!cnt) {
2948                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
2949                                 (tbl->obj, &tbl_data->jump.action);
2950                 if (ret)
2951                         return rte_flow_error_set(error, ENOMEM,
2952                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2953                                         NULL, "cannot create jump action");
2954                 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2955                         (void *)&tbl_data->jump, cnt);
2956         } else {
2957                 /* old jump should not make the table ref++. */
2958                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
2959                 MLX5_ASSERT(tbl_data->jump.action);
2960                 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2961                         (void *)&tbl_data->jump, cnt);
2962         }
2963         rte_atomic32_inc(&tbl_data->jump.refcnt);
2964         dev_flow->handle->rix_jump = tbl_data->idx;
2965         dev_flow->dv.jump = &tbl_data->jump;
2966         return 0;
2967 }
2968
2969 /**
2970  * Find existing default miss resource or create and register a new one.
2971  *
2972  * @param[in, out] dev
2973  *   Pointer to rte_eth_dev structure.
2974  * @param[out] error
2975  *   pointer to error structure.
2976  *
2977  * @return
2978  *   0 on success otherwise -errno and errno is set.
2979  */
2980 static int
2981 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
2982                 struct rte_flow_error *error)
2983 {
2984         struct mlx5_priv *priv = dev->data->dev_private;
2985         struct mlx5_dev_ctx_shared *sh = priv->sh;
2986         struct mlx5_flow_default_miss_resource *cache_resource =
2987                         &sh->default_miss;
2988         int cnt = rte_atomic32_read(&cache_resource->refcnt);
2989
2990         if (!cnt) {
2991                 MLX5_ASSERT(cache_resource->action);
2992                 cache_resource->action =
2993                 mlx5_glue->dr_create_flow_action_default_miss();
2994                 if (!cache_resource->action)
2995                         return rte_flow_error_set(error, ENOMEM,
2996                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2997                                         "cannot create default miss action");
2998                 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++",
2999                                 (void *)cache_resource->action, cnt);
3000         }
3001         rte_atomic32_inc(&cache_resource->refcnt);
3002         return 0;
3003 }
3004
3005 /**
3006  * Find existing table port ID resource or create and register a new one.
3007  *
3008  * @param[in, out] dev
3009  *   Pointer to rte_eth_dev structure.
3010  * @param[in, out] resource
3011  *   Pointer to port ID action resource.
3012  * @parm[in, out] dev_flow
3013  *   Pointer to the dev_flow.
3014  * @param[out] error
3015  *   pointer to error structure.
3016  *
3017  * @return
3018  *   0 on success otherwise -errno and errno is set.
3019  */
3020 static int
3021 flow_dv_port_id_action_resource_register
3022                         (struct rte_eth_dev *dev,
3023                          struct mlx5_flow_dv_port_id_action_resource *resource,
3024                          struct mlx5_flow *dev_flow,
3025                          struct rte_flow_error *error)
3026 {
3027         struct mlx5_priv *priv = dev->data->dev_private;
3028         struct mlx5_dev_ctx_shared *sh = priv->sh;
3029         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
3030         uint32_t idx = 0;
3031         int ret;
3032
3033         /* Lookup a matching resource from cache. */
3034         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list,
3035                       idx, cache_resource, next) {
3036                 if (resource->port_id == cache_resource->port_id) {
3037                         DRV_LOG(DEBUG, "port id action resource resource %p: "
3038                                 "refcnt %d++",
3039                                 (void *)cache_resource,
3040                                 rte_atomic32_read(&cache_resource->refcnt));
3041                         rte_atomic32_inc(&cache_resource->refcnt);
3042                         dev_flow->handle->rix_port_id_action = idx;
3043                         dev_flow->dv.port_id_action = cache_resource;
3044                         return 0;
3045                 }
3046         }
3047         /* Register new port id action resource. */
3048         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID],
3049                                        &dev_flow->handle->rix_port_id_action);
3050         if (!cache_resource)
3051                 return rte_flow_error_set(error, ENOMEM,
3052                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3053                                           "cannot allocate resource memory");
3054         *cache_resource = *resource;
3055         ret = mlx5_flow_os_create_flow_action_dest_port
3056                                 (priv->sh->fdb_domain, resource->port_id,
3057                                  &cache_resource->action);
3058         if (ret) {
3059                 mlx5_free(cache_resource);
3060                 return rte_flow_error_set(error, ENOMEM,
3061                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3062                                           NULL, "cannot create action");
3063         }
3064         rte_atomic32_init(&cache_resource->refcnt);
3065         rte_atomic32_inc(&cache_resource->refcnt);
3066         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
3067                      dev_flow->handle->rix_port_id_action, cache_resource,
3068                      next);
3069         dev_flow->dv.port_id_action = cache_resource;
3070         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
3071                 (void *)cache_resource,
3072                 rte_atomic32_read(&cache_resource->refcnt));
3073         return 0;
3074 }
3075
3076 /**
3077  * Find existing push vlan resource or create and register a new one.
3078  *
3079  * @param [in, out] dev
3080  *   Pointer to rte_eth_dev structure.
3081  * @param[in, out] resource
3082  *   Pointer to port ID action resource.
3083  * @parm[in, out] dev_flow
3084  *   Pointer to the dev_flow.
3085  * @param[out] error
3086  *   pointer to error structure.
3087  *
3088  * @return
3089  *   0 on success otherwise -errno and errno is set.
3090  */
3091 static int
3092 flow_dv_push_vlan_action_resource_register
3093                        (struct rte_eth_dev *dev,
3094                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
3095                         struct mlx5_flow *dev_flow,
3096                         struct rte_flow_error *error)
3097 {
3098         struct mlx5_priv *priv = dev->data->dev_private;
3099         struct mlx5_dev_ctx_shared *sh = priv->sh;
3100         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
3101         struct mlx5dv_dr_domain *domain;
3102         uint32_t idx = 0;
3103         int ret;
3104
3105         /* Lookup a matching resource from cache. */
3106         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3107                       sh->push_vlan_action_list, idx, cache_resource, next) {
3108                 if (resource->vlan_tag == cache_resource->vlan_tag &&
3109                     resource->ft_type == cache_resource->ft_type) {
3110                         DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
3111                                 "refcnt %d++",
3112                                 (void *)cache_resource,
3113                                 rte_atomic32_read(&cache_resource->refcnt));
3114                         rte_atomic32_inc(&cache_resource->refcnt);
3115                         dev_flow->handle->dvh.rix_push_vlan = idx;
3116                         dev_flow->dv.push_vlan_res = cache_resource;
3117                         return 0;
3118                 }
3119         }
3120         /* Register new push_vlan action resource. */
3121         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3122                                        &dev_flow->handle->dvh.rix_push_vlan);
3123         if (!cache_resource)
3124                 return rte_flow_error_set(error, ENOMEM,
3125                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3126                                           "cannot allocate resource memory");
3127         *cache_resource = *resource;
3128         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3129                 domain = sh->fdb_domain;
3130         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3131                 domain = sh->rx_domain;
3132         else
3133                 domain = sh->tx_domain;
3134         ret = mlx5_flow_os_create_flow_action_push_vlan
3135                                         (domain, resource->vlan_tag,
3136                                          &cache_resource->action);
3137         if (ret) {
3138                 mlx5_free(cache_resource);
3139                 return rte_flow_error_set(error, ENOMEM,
3140                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3141                                           NULL, "cannot create action");
3142         }
3143         rte_atomic32_init(&cache_resource->refcnt);
3144         rte_atomic32_inc(&cache_resource->refcnt);
3145         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3146                      &sh->push_vlan_action_list,
3147                      dev_flow->handle->dvh.rix_push_vlan,
3148                      cache_resource, next);
3149         dev_flow->dv.push_vlan_res = cache_resource;
3150         DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
3151                 (void *)cache_resource,
3152                 rte_atomic32_read(&cache_resource->refcnt));
3153         return 0;
3154 }
3155 /**
3156  * Get the size of specific rte_flow_item_type hdr size
3157  *
3158  * @param[in] item_type
3159  *   Tested rte_flow_item_type.
3160  *
3161  * @return
3162  *   sizeof struct item_type, 0 if void or irrelevant.
3163  */
3164 static size_t
3165 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
3166 {
3167         size_t retval;
3168
3169         switch (item_type) {
3170         case RTE_FLOW_ITEM_TYPE_ETH:
3171                 retval = sizeof(struct rte_ether_hdr);
3172                 break;
3173         case RTE_FLOW_ITEM_TYPE_VLAN:
3174                 retval = sizeof(struct rte_vlan_hdr);
3175                 break;
3176         case RTE_FLOW_ITEM_TYPE_IPV4:
3177                 retval = sizeof(struct rte_ipv4_hdr);
3178                 break;
3179         case RTE_FLOW_ITEM_TYPE_IPV6:
3180                 retval = sizeof(struct rte_ipv6_hdr);
3181                 break;
3182         case RTE_FLOW_ITEM_TYPE_UDP:
3183                 retval = sizeof(struct rte_udp_hdr);
3184                 break;
3185         case RTE_FLOW_ITEM_TYPE_TCP:
3186                 retval = sizeof(struct rte_tcp_hdr);
3187                 break;
3188         case RTE_FLOW_ITEM_TYPE_VXLAN:
3189         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3190                 retval = sizeof(struct rte_vxlan_hdr);
3191                 break;
3192         case RTE_FLOW_ITEM_TYPE_GRE:
3193         case RTE_FLOW_ITEM_TYPE_NVGRE:
3194                 retval = sizeof(struct rte_gre_hdr);
3195                 break;
3196         case RTE_FLOW_ITEM_TYPE_MPLS:
3197                 retval = sizeof(struct rte_mpls_hdr);
3198                 break;
3199         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
3200         default:
3201                 retval = 0;
3202                 break;
3203         }
3204         return retval;
3205 }
3206
3207 #define MLX5_ENCAP_IPV4_VERSION         0x40
3208 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
3209 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
3210 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
3211 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
3212 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
3213 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
3214
3215 /**
3216  * Convert the encap action data from list of rte_flow_item to raw buffer
3217  *
3218  * @param[in] items
3219  *   Pointer to rte_flow_item objects list.
3220  * @param[out] buf
3221  *   Pointer to the output buffer.
3222  * @param[out] size
3223  *   Pointer to the output buffer size.
3224  * @param[out] error
3225  *   Pointer to the error structure.
3226  *
3227  * @return
3228  *   0 on success, a negative errno value otherwise and rte_errno is set.
3229  */
3230 static int
3231 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
3232                            size_t *size, struct rte_flow_error *error)
3233 {
3234         struct rte_ether_hdr *eth = NULL;
3235         struct rte_vlan_hdr *vlan = NULL;
3236         struct rte_ipv4_hdr *ipv4 = NULL;
3237         struct rte_ipv6_hdr *ipv6 = NULL;
3238         struct rte_udp_hdr *udp = NULL;
3239         struct rte_vxlan_hdr *vxlan = NULL;
3240         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
3241         struct rte_gre_hdr *gre = NULL;
3242         size_t len;
3243         size_t temp_size = 0;
3244
3245         if (!items)
3246                 return rte_flow_error_set(error, EINVAL,
3247                                           RTE_FLOW_ERROR_TYPE_ACTION,
3248                                           NULL, "invalid empty data");
3249         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3250                 len = flow_dv_get_item_hdr_len(items->type);
3251                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
3252                         return rte_flow_error_set(error, EINVAL,
3253                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3254                                                   (void *)items->type,
3255                                                   "items total size is too big"
3256                                                   " for encap action");
3257                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
3258                 switch (items->type) {
3259                 case RTE_FLOW_ITEM_TYPE_ETH:
3260                         eth = (struct rte_ether_hdr *)&buf[temp_size];
3261                         break;
3262                 case RTE_FLOW_ITEM_TYPE_VLAN:
3263                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
3264                         if (!eth)
3265                                 return rte_flow_error_set(error, EINVAL,
3266                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3267                                                 (void *)items->type,
3268                                                 "eth header not found");
3269                         if (!eth->ether_type)
3270                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
3271                         break;
3272                 case RTE_FLOW_ITEM_TYPE_IPV4:
3273                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
3274                         if (!vlan && !eth)
3275                                 return rte_flow_error_set(error, EINVAL,
3276                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3277                                                 (void *)items->type,
3278                                                 "neither eth nor vlan"
3279                                                 " header found");
3280                         if (vlan && !vlan->eth_proto)
3281                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3282                         else if (eth && !eth->ether_type)
3283                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3284                         if (!ipv4->version_ihl)
3285                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
3286                                                     MLX5_ENCAP_IPV4_IHL_MIN;
3287                         if (!ipv4->time_to_live)
3288                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
3289                         break;
3290                 case RTE_FLOW_ITEM_TYPE_IPV6:
3291                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
3292                         if (!vlan && !eth)
3293                                 return rte_flow_error_set(error, EINVAL,
3294                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3295                                                 (void *)items->type,
3296                                                 "neither eth nor vlan"
3297                                                 " header found");
3298                         if (vlan && !vlan->eth_proto)
3299                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3300                         else if (eth && !eth->ether_type)
3301                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3302                         if (!ipv6->vtc_flow)
3303                                 ipv6->vtc_flow =
3304                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
3305                         if (!ipv6->hop_limits)
3306                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
3307                         break;
3308                 case RTE_FLOW_ITEM_TYPE_UDP:
3309                         udp = (struct rte_udp_hdr *)&buf[temp_size];
3310                         if (!ipv4 && !ipv6)
3311                                 return rte_flow_error_set(error, EINVAL,
3312                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3313                                                 (void *)items->type,
3314                                                 "ip header not found");
3315                         if (ipv4 && !ipv4->next_proto_id)
3316                                 ipv4->next_proto_id = IPPROTO_UDP;
3317                         else if (ipv6 && !ipv6->proto)
3318                                 ipv6->proto = IPPROTO_UDP;
3319                         break;
3320                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3321                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
3322                         if (!udp)
3323                                 return rte_flow_error_set(error, EINVAL,
3324                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3325                                                 (void *)items->type,
3326                                                 "udp header not found");
3327                         if (!udp->dst_port)
3328                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
3329                         if (!vxlan->vx_flags)
3330                                 vxlan->vx_flags =
3331                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
3332                         break;
3333                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3334                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
3335                         if (!udp)
3336                                 return rte_flow_error_set(error, EINVAL,
3337                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3338                                                 (void *)items->type,
3339                                                 "udp header not found");
3340                         if (!vxlan_gpe->proto)
3341                                 return rte_flow_error_set(error, EINVAL,
3342                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3343                                                 (void *)items->type,
3344                                                 "next protocol not found");
3345                         if (!udp->dst_port)
3346                                 udp->dst_port =
3347                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
3348                         if (!vxlan_gpe->vx_flags)
3349                                 vxlan_gpe->vx_flags =
3350                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
3351                         break;
3352                 case RTE_FLOW_ITEM_TYPE_GRE:
3353                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3354                         gre = (struct rte_gre_hdr *)&buf[temp_size];
3355                         if (!gre->proto)
3356                                 return rte_flow_error_set(error, EINVAL,
3357                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3358                                                 (void *)items->type,
3359                                                 "next protocol not found");
3360                         if (!ipv4 && !ipv6)
3361                                 return rte_flow_error_set(error, EINVAL,
3362                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3363                                                 (void *)items->type,
3364                                                 "ip header not found");
3365                         if (ipv4 && !ipv4->next_proto_id)
3366                                 ipv4->next_proto_id = IPPROTO_GRE;
3367                         else if (ipv6 && !ipv6->proto)
3368                                 ipv6->proto = IPPROTO_GRE;
3369                         break;
3370                 case RTE_FLOW_ITEM_TYPE_VOID:
3371                         break;
3372                 default:
3373                         return rte_flow_error_set(error, EINVAL,
3374                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3375                                                   (void *)items->type,
3376                                                   "unsupported item type");
3377                         break;
3378                 }
3379                 temp_size += len;
3380         }
3381         *size = temp_size;
3382         return 0;
3383 }
3384
3385 static int
3386 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
3387 {
3388         struct rte_ether_hdr *eth = NULL;
3389         struct rte_vlan_hdr *vlan = NULL;
3390         struct rte_ipv6_hdr *ipv6 = NULL;
3391         struct rte_udp_hdr *udp = NULL;
3392         char *next_hdr;
3393         uint16_t proto;
3394
3395         eth = (struct rte_ether_hdr *)data;
3396         next_hdr = (char *)(eth + 1);
3397         proto = RTE_BE16(eth->ether_type);
3398
3399         /* VLAN skipping */
3400         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
3401                 vlan = (struct rte_vlan_hdr *)next_hdr;
3402                 proto = RTE_BE16(vlan->eth_proto);
3403                 next_hdr += sizeof(struct rte_vlan_hdr);
3404         }
3405
3406         /* HW calculates IPv4 csum. no need to proceed */
3407         if (proto == RTE_ETHER_TYPE_IPV4)
3408                 return 0;
3409
3410         /* non IPv4/IPv6 header. not supported */
3411         if (proto != RTE_ETHER_TYPE_IPV6) {
3412                 return rte_flow_error_set(error, ENOTSUP,
3413                                           RTE_FLOW_ERROR_TYPE_ACTION,
3414                                           NULL, "Cannot offload non IPv4/IPv6");
3415         }
3416
3417         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
3418
3419         /* ignore non UDP */
3420         if (ipv6->proto != IPPROTO_UDP)
3421                 return 0;
3422
3423         udp = (struct rte_udp_hdr *)(ipv6 + 1);
3424         udp->dgram_cksum = 0;
3425
3426         return 0;
3427 }
3428
3429 /**
3430  * Convert L2 encap action to DV specification.
3431  *
3432  * @param[in] dev
3433  *   Pointer to rte_eth_dev structure.
3434  * @param[in] action
3435  *   Pointer to action structure.
3436  * @param[in, out] dev_flow
3437  *   Pointer to the mlx5_flow.
3438  * @param[in] transfer
3439  *   Mark if the flow is E-Switch flow.
3440  * @param[out] error
3441  *   Pointer to the error structure.
3442  *
3443  * @return
3444  *   0 on success, a negative errno value otherwise and rte_errno is set.
3445  */
3446 static int
3447 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
3448                                const struct rte_flow_action *action,
3449                                struct mlx5_flow *dev_flow,
3450                                uint8_t transfer,
3451                                struct rte_flow_error *error)
3452 {
3453         const struct rte_flow_item *encap_data;
3454         const struct rte_flow_action_raw_encap *raw_encap_data;
3455         struct mlx5_flow_dv_encap_decap_resource res = {
3456                 .reformat_type =
3457                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
3458                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3459                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
3460         };
3461
3462         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3463                 raw_encap_data =
3464                         (const struct rte_flow_action_raw_encap *)action->conf;
3465                 res.size = raw_encap_data->size;
3466                 memcpy(res.buf, raw_encap_data->data, res.size);
3467         } else {
3468                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
3469                         encap_data =
3470                                 ((const struct rte_flow_action_vxlan_encap *)
3471                                                 action->conf)->definition;
3472                 else
3473                         encap_data =
3474                                 ((const struct rte_flow_action_nvgre_encap *)
3475                                                 action->conf)->definition;
3476                 if (flow_dv_convert_encap_data(encap_data, res.buf,
3477                                                &res.size, error))
3478                         return -rte_errno;
3479         }
3480         if (flow_dv_zero_encap_udp_csum(res.buf, error))
3481                 return -rte_errno;
3482         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3483                 return rte_flow_error_set(error, EINVAL,
3484                                           RTE_FLOW_ERROR_TYPE_ACTION,
3485                                           NULL, "can't create L2 encap action");
3486         return 0;
3487 }
3488
3489 /**
3490  * Convert L2 decap action to DV specification.
3491  *
3492  * @param[in] dev
3493  *   Pointer to rte_eth_dev structure.
3494  * @param[in, out] dev_flow
3495  *   Pointer to the mlx5_flow.
3496  * @param[in] transfer
3497  *   Mark if the flow is E-Switch flow.
3498  * @param[out] error
3499  *   Pointer to the error structure.
3500  *
3501  * @return
3502  *   0 on success, a negative errno value otherwise and rte_errno is set.
3503  */
3504 static int
3505 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
3506                                struct mlx5_flow *dev_flow,
3507                                uint8_t transfer,
3508                                struct rte_flow_error *error)
3509 {
3510         struct mlx5_flow_dv_encap_decap_resource res = {
3511                 .size = 0,
3512                 .reformat_type =
3513                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
3514                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3515                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
3516         };
3517
3518         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3519                 return rte_flow_error_set(error, EINVAL,
3520                                           RTE_FLOW_ERROR_TYPE_ACTION,
3521                                           NULL, "can't create L2 decap action");
3522         return 0;
3523 }
3524
3525 /**
3526  * Convert raw decap/encap (L3 tunnel) action to DV specification.
3527  *
3528  * @param[in] dev
3529  *   Pointer to rte_eth_dev structure.
3530  * @param[in] action
3531  *   Pointer to action structure.
3532  * @param[in, out] dev_flow
3533  *   Pointer to the mlx5_flow.
3534  * @param[in] attr
3535  *   Pointer to the flow attributes.
3536  * @param[out] error
3537  *   Pointer to the error structure.
3538  *
3539  * @return
3540  *   0 on success, a negative errno value otherwise and rte_errno is set.
3541  */
3542 static int
3543 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
3544                                 const struct rte_flow_action *action,
3545                                 struct mlx5_flow *dev_flow,
3546                                 const struct rte_flow_attr *attr,
3547                                 struct rte_flow_error *error)
3548 {
3549         const struct rte_flow_action_raw_encap *encap_data;
3550         struct mlx5_flow_dv_encap_decap_resource res;
3551
3552         memset(&res, 0, sizeof(res));
3553         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
3554         res.size = encap_data->size;
3555         memcpy(res.buf, encap_data->data, res.size);
3556         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
3557                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
3558                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3559         if (attr->transfer)
3560                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3561         else
3562                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3563                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3564         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3565                 return rte_flow_error_set(error, EINVAL,
3566                                           RTE_FLOW_ERROR_TYPE_ACTION,
3567                                           NULL, "can't create encap action");
3568         return 0;
3569 }
3570
3571 /**
3572  * Create action push VLAN.
3573  *
3574  * @param[in] dev
3575  *   Pointer to rte_eth_dev structure.
3576  * @param[in] attr
3577  *   Pointer to the flow attributes.
3578  * @param[in] vlan
3579  *   Pointer to the vlan to push to the Ethernet header.
3580  * @param[in, out] dev_flow
3581  *   Pointer to the mlx5_flow.
3582  * @param[out] error
3583  *   Pointer to the error structure.
3584  *
3585  * @return
3586  *   0 on success, a negative errno value otherwise and rte_errno is set.
3587  */
3588 static int
3589 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
3590                                 const struct rte_flow_attr *attr,
3591                                 const struct rte_vlan_hdr *vlan,
3592                                 struct mlx5_flow *dev_flow,
3593                                 struct rte_flow_error *error)
3594 {
3595         struct mlx5_flow_dv_push_vlan_action_resource res;
3596
3597         memset(&res, 0, sizeof(res));
3598         res.vlan_tag =
3599                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
3600                                  vlan->vlan_tci);
3601         if (attr->transfer)
3602                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3603         else
3604                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3605                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3606         return flow_dv_push_vlan_action_resource_register
3607                                             (dev, &res, dev_flow, error);
3608 }
3609
3610 static int fdb_mirror;
3611
3612 /**
3613  * Validate the modify-header actions.
3614  *
3615  * @param[in] action_flags
3616  *   Holds the actions detected until now.
3617  * @param[in] action
3618  *   Pointer to the modify action.
3619  * @param[out] error
3620  *   Pointer to error structure.
3621  *
3622  * @return
3623  *   0 on success, a negative errno value otherwise and rte_errno is set.
3624  */
3625 static int
3626 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
3627                                    const struct rte_flow_action *action,
3628                                    struct rte_flow_error *error)
3629 {
3630         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
3631                 return rte_flow_error_set(error, EINVAL,
3632                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3633                                           NULL, "action configuration not set");
3634         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3635                 return rte_flow_error_set(error, EINVAL,
3636                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3637                                           "can't have encap action before"
3638                                           " modify action");
3639         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror)
3640                 return rte_flow_error_set(error, EINVAL,
3641                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3642                                           "can't support sample action before"
3643                                           " modify action for E-Switch"
3644                                           " mirroring");
3645         return 0;
3646 }
3647
3648 /**
3649  * Validate the modify-header MAC address actions.
3650  *
3651  * @param[in] action_flags
3652  *   Holds the actions detected until now.
3653  * @param[in] action
3654  *   Pointer to the modify action.
3655  * @param[in] item_flags
3656  *   Holds the items detected.
3657  * @param[out] error
3658  *   Pointer to error structure.
3659  *
3660  * @return
3661  *   0 on success, a negative errno value otherwise and rte_errno is set.
3662  */
3663 static int
3664 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
3665                                    const struct rte_flow_action *action,
3666                                    const uint64_t item_flags,
3667                                    struct rte_flow_error *error)
3668 {
3669         int ret = 0;
3670
3671         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3672         if (!ret) {
3673                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
3674                         return rte_flow_error_set(error, EINVAL,
3675                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3676                                                   NULL,
3677                                                   "no L2 item in pattern");
3678         }
3679         return ret;
3680 }
3681
3682 /**
3683  * Validate the modify-header IPv4 address actions.
3684  *
3685  * @param[in] action_flags
3686  *   Holds the actions detected until now.
3687  * @param[in] action
3688  *   Pointer to the modify action.
3689  * @param[in] item_flags
3690  *   Holds the items detected.
3691  * @param[out] error
3692  *   Pointer to error structure.
3693  *
3694  * @return
3695  *   0 on success, a negative errno value otherwise and rte_errno is set.
3696  */
3697 static int
3698 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3699                                     const struct rte_flow_action *action,
3700                                     const uint64_t item_flags,
3701                                     struct rte_flow_error *error)
3702 {
3703         int ret = 0;
3704         uint64_t layer;
3705
3706         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3707         if (!ret) {
3708                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3709                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3710                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3711                 if (!(item_flags & layer))
3712                         return rte_flow_error_set(error, EINVAL,
3713                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3714                                                   NULL,
3715                                                   "no ipv4 item in pattern");
3716         }
3717         return ret;
3718 }
3719
3720 /**
3721  * Validate the modify-header IPv6 address actions.
3722  *
3723  * @param[in] action_flags
3724  *   Holds the actions detected until now.
3725  * @param[in] action
3726  *   Pointer to the modify action.
3727  * @param[in] item_flags
3728  *   Holds the items detected.
3729  * @param[out] error
3730  *   Pointer to error structure.
3731  *
3732  * @return
3733  *   0 on success, a negative errno value otherwise and rte_errno is set.
3734  */
3735 static int
3736 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3737                                     const struct rte_flow_action *action,
3738                                     const uint64_t item_flags,
3739                                     struct rte_flow_error *error)
3740 {
3741         int ret = 0;
3742         uint64_t layer;
3743
3744         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3745         if (!ret) {
3746                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3747                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3748                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3749                 if (!(item_flags & layer))
3750                         return rte_flow_error_set(error, EINVAL,
3751                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3752                                                   NULL,
3753                                                   "no ipv6 item in pattern");
3754         }
3755         return ret;
3756 }
3757
3758 /**
3759  * Validate the modify-header TP actions.
3760  *
3761  * @param[in] action_flags
3762  *   Holds the actions detected until now.
3763  * @param[in] action
3764  *   Pointer to the modify action.
3765  * @param[in] item_flags
3766  *   Holds the items detected.
3767  * @param[out] error
3768  *   Pointer to error structure.
3769  *
3770  * @return
3771  *   0 on success, a negative errno value otherwise and rte_errno is set.
3772  */
3773 static int
3774 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3775                                   const struct rte_flow_action *action,
3776                                   const uint64_t item_flags,
3777                                   struct rte_flow_error *error)
3778 {
3779         int ret = 0;
3780         uint64_t layer;
3781
3782         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3783         if (!ret) {
3784                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3785                                  MLX5_FLOW_LAYER_INNER_L4 :
3786                                  MLX5_FLOW_LAYER_OUTER_L4;
3787                 if (!(item_flags & layer))
3788                         return rte_flow_error_set(error, EINVAL,
3789                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3790                                                   NULL, "no transport layer "
3791                                                   "in pattern");
3792         }
3793         return ret;
3794 }
3795
3796 /**
3797  * Validate the modify-header actions of increment/decrement
3798  * TCP Sequence-number.
3799  *
3800  * @param[in] action_flags
3801  *   Holds the actions detected until now.
3802  * @param[in] action
3803  *   Pointer to the modify action.
3804  * @param[in] item_flags
3805  *   Holds the items detected.
3806  * @param[out] error
3807  *   Pointer to error structure.
3808  *
3809  * @return
3810  *   0 on success, a negative errno value otherwise and rte_errno is set.
3811  */
3812 static int
3813 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3814                                        const struct rte_flow_action *action,
3815                                        const uint64_t item_flags,
3816                                        struct rte_flow_error *error)
3817 {
3818         int ret = 0;
3819         uint64_t layer;
3820
3821         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3822         if (!ret) {
3823                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3824                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3825                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3826                 if (!(item_flags & layer))
3827                         return rte_flow_error_set(error, EINVAL,
3828                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3829                                                   NULL, "no TCP item in"
3830                                                   " pattern");
3831                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3832                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3833                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3834                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3835                         return rte_flow_error_set(error, EINVAL,
3836                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3837                                                   NULL,
3838                                                   "cannot decrease and increase"
3839                                                   " TCP sequence number"
3840                                                   " at the same time");
3841         }
3842         return ret;
3843 }
3844
3845 /**
3846  * Validate the modify-header actions of increment/decrement
3847  * TCP Acknowledgment number.
3848  *
3849  * @param[in] action_flags
3850  *   Holds the actions detected until now.
3851  * @param[in] action
3852  *   Pointer to the modify action.
3853  * @param[in] item_flags
3854  *   Holds the items detected.
3855  * @param[out] error
3856  *   Pointer to error structure.
3857  *
3858  * @return
3859  *   0 on success, a negative errno value otherwise and rte_errno is set.
3860  */
3861 static int
3862 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3863                                        const struct rte_flow_action *action,
3864                                        const uint64_t item_flags,
3865                                        struct rte_flow_error *error)
3866 {
3867         int ret = 0;
3868         uint64_t layer;
3869
3870         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3871         if (!ret) {
3872                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3873                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3874                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3875                 if (!(item_flags & layer))
3876                         return rte_flow_error_set(error, EINVAL,
3877                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3878                                                   NULL, "no TCP item in"
3879                                                   " pattern");
3880                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3881                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3882                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3883                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3884                         return rte_flow_error_set(error, EINVAL,
3885                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3886                                                   NULL,
3887                                                   "cannot decrease and increase"
3888                                                   " TCP acknowledgment number"
3889                                                   " at the same time");
3890         }
3891         return ret;
3892 }
3893
3894 /**
3895  * Validate the modify-header TTL actions.
3896  *
3897  * @param[in] action_flags
3898  *   Holds the actions detected until now.
3899  * @param[in] action
3900  *   Pointer to the modify action.
3901  * @param[in] item_flags
3902  *   Holds the items detected.
3903  * @param[out] error
3904  *   Pointer to error structure.
3905  *
3906  * @return
3907  *   0 on success, a negative errno value otherwise and rte_errno is set.
3908  */
3909 static int
3910 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3911                                    const struct rte_flow_action *action,
3912                                    const uint64_t item_flags,
3913                                    struct rte_flow_error *error)
3914 {
3915         int ret = 0;
3916         uint64_t layer;
3917
3918         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3919         if (!ret) {
3920                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3921                                  MLX5_FLOW_LAYER_INNER_L3 :
3922                                  MLX5_FLOW_LAYER_OUTER_L3;
3923                 if (!(item_flags & layer))
3924                         return rte_flow_error_set(error, EINVAL,
3925                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3926                                                   NULL,
3927                                                   "no IP protocol in pattern");
3928         }
3929         return ret;
3930 }
3931
3932 /**
3933  * Validate jump action.
3934  *
3935  * @param[in] action
3936  *   Pointer to the jump action.
3937  * @param[in] action_flags
3938  *   Holds the actions detected until now.
3939  * @param[in] attributes
3940  *   Pointer to flow attributes
3941  * @param[in] external
3942  *   Action belongs to flow rule created by request external to PMD.
3943  * @param[out] error
3944  *   Pointer to error structure.
3945  *
3946  * @return
3947  *   0 on success, a negative errno value otherwise and rte_errno is set.
3948  */
3949 static int
3950 flow_dv_validate_action_jump(const struct rte_flow_action *action,
3951                              uint64_t action_flags,
3952                              const struct rte_flow_attr *attributes,
3953                              bool external, struct rte_flow_error *error)
3954 {
3955         uint32_t target_group, table;
3956         int ret = 0;
3957
3958         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3959                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3960                 return rte_flow_error_set(error, EINVAL,
3961                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3962                                           "can't have 2 fate actions in"
3963                                           " same flow");
3964         if (action_flags & MLX5_FLOW_ACTION_METER)
3965                 return rte_flow_error_set(error, ENOTSUP,
3966                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3967                                           "jump with meter not support");
3968         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror)
3969                 return rte_flow_error_set(error, EINVAL,
3970                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3971                                           "E-Switch mirroring can't support"
3972                                           " Sample action and jump action in"
3973                                           " same flow now");
3974         if (!action->conf)
3975                 return rte_flow_error_set(error, EINVAL,
3976                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3977                                           NULL, "action configuration not set");
3978         target_group =
3979                 ((const struct rte_flow_action_jump *)action->conf)->group;
3980         ret = mlx5_flow_group_to_table(attributes, external, target_group,
3981                                        true, &table, error);
3982         if (ret)
3983                 return ret;
3984         if (attributes->group == target_group)
3985                 return rte_flow_error_set(error, EINVAL,
3986                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3987                                           "target group must be other than"
3988                                           " the current flow group");
3989         return 0;
3990 }
3991
3992 /*
3993  * Validate the port_id action.
3994  *
3995  * @param[in] dev
3996  *   Pointer to rte_eth_dev structure.
3997  * @param[in] action_flags
3998  *   Bit-fields that holds the actions detected until now.
3999  * @param[in] action
4000  *   Port_id RTE action structure.
4001  * @param[in] attr
4002  *   Attributes of flow that includes this action.
4003  * @param[out] error
4004  *   Pointer to error structure.
4005  *
4006  * @return
4007  *   0 on success, a negative errno value otherwise and rte_errno is set.
4008  */
4009 static int
4010 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
4011                                 uint64_t action_flags,
4012                                 const struct rte_flow_action *action,
4013                                 const struct rte_flow_attr *attr,
4014                                 struct rte_flow_error *error)
4015 {
4016         const struct rte_flow_action_port_id *port_id;
4017         struct mlx5_priv *act_priv;
4018         struct mlx5_priv *dev_priv;
4019         uint16_t port;
4020
4021         if (!attr->transfer)
4022                 return rte_flow_error_set(error, ENOTSUP,
4023                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4024                                           NULL,
4025                                           "port id action is valid in transfer"
4026                                           " mode only");
4027         if (!action || !action->conf)
4028                 return rte_flow_error_set(error, ENOTSUP,
4029                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4030                                           NULL,
4031                                           "port id action parameters must be"
4032                                           " specified");
4033         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
4034                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4035                 return rte_flow_error_set(error, EINVAL,
4036                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4037                                           "can have only one fate actions in"
4038                                           " a flow");
4039         dev_priv = mlx5_dev_to_eswitch_info(dev);
4040         if (!dev_priv)
4041                 return rte_flow_error_set(error, rte_errno,
4042                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4043                                           NULL,
4044                                           "failed to obtain E-Switch info");
4045         port_id = action->conf;
4046         port = port_id->original ? dev->data->port_id : port_id->id;
4047         act_priv = mlx5_port_to_eswitch_info(port, false);
4048         if (!act_priv)
4049                 return rte_flow_error_set
4050                                 (error, rte_errno,
4051                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
4052                                  "failed to obtain E-Switch port id for port");
4053         if (act_priv->domain_id != dev_priv->domain_id)
4054                 return rte_flow_error_set
4055                                 (error, EINVAL,
4056                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4057                                  "port does not belong to"
4058                                  " E-Switch being configured");
4059         return 0;
4060 }
4061
4062 /**
4063  * Get the maximum number of modify header actions.
4064  *
4065  * @param dev
4066  *   Pointer to rte_eth_dev structure.
4067  * @param flags
4068  *   Flags bits to check if root level.
4069  *
4070  * @return
4071  *   Max number of modify header actions device can support.
4072  */
4073 static inline unsigned int
4074 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
4075                               uint64_t flags)
4076 {
4077         /*
4078          * There's no way to directly query the max capacity from FW.
4079          * The maximal value on root table should be assumed to be supported.
4080          */
4081         if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
4082                 return MLX5_MAX_MODIFY_NUM;
4083         else
4084                 return MLX5_ROOT_TBL_MODIFY_NUM;
4085 }
4086
4087 /**
4088  * Validate the meter action.
4089  *
4090  * @param[in] dev
4091  *   Pointer to rte_eth_dev structure.
4092  * @param[in] action_flags
4093  *   Bit-fields that holds the actions detected until now.
4094  * @param[in] action
4095  *   Pointer to the meter action.
4096  * @param[in] attr
4097  *   Attributes of flow that includes this action.
4098  * @param[out] error
4099  *   Pointer to error structure.
4100  *
4101  * @return
4102  *   0 on success, a negative errno value otherwise and rte_ernno is set.
4103  */
4104 static int
4105 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
4106                                 uint64_t action_flags,
4107                                 const struct rte_flow_action *action,
4108                                 const struct rte_flow_attr *attr,
4109                                 struct rte_flow_error *error)
4110 {
4111         struct mlx5_priv *priv = dev->data->dev_private;
4112         const struct rte_flow_action_meter *am = action->conf;
4113         struct mlx5_flow_meter *fm;
4114
4115         if (!am)
4116                 return rte_flow_error_set(error, EINVAL,
4117                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4118                                           "meter action conf is NULL");
4119
4120         if (action_flags & MLX5_FLOW_ACTION_METER)
4121                 return rte_flow_error_set(error, ENOTSUP,
4122                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4123                                           "meter chaining not support");
4124         if (action_flags & MLX5_FLOW_ACTION_JUMP)
4125                 return rte_flow_error_set(error, ENOTSUP,
4126                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4127                                           "meter with jump not support");
4128         if (!priv->mtr_en)
4129                 return rte_flow_error_set(error, ENOTSUP,
4130                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4131                                           NULL,
4132                                           "meter action not supported");
4133         fm = mlx5_flow_meter_find(priv, am->mtr_id);
4134         if (!fm)
4135                 return rte_flow_error_set(error, EINVAL,
4136                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4137                                           "Meter not found");
4138         if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
4139               (!fm->ingress && !attr->ingress && attr->egress) ||
4140               (!fm->egress && !attr->egress && attr->ingress))))
4141                 return rte_flow_error_set(error, EINVAL,
4142                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4143                                           "Flow attributes are either invalid "
4144                                           "or have a conflict with current "
4145                                           "meter attributes");
4146         return 0;
4147 }
4148
4149 /**
4150  * Validate the age action.
4151  *
4152  * @param[in] action_flags
4153  *   Holds the actions detected until now.
4154  * @param[in] action
4155  *   Pointer to the age action.
4156  * @param[in] dev
4157  *   Pointer to the Ethernet device structure.
4158  * @param[out] error
4159  *   Pointer to error structure.
4160  *
4161  * @return
4162  *   0 on success, a negative errno value otherwise and rte_errno is set.
4163  */
4164 static int
4165 flow_dv_validate_action_age(uint64_t action_flags,
4166                             const struct rte_flow_action *action,
4167                             struct rte_eth_dev *dev,
4168                             struct rte_flow_error *error)
4169 {
4170         struct mlx5_priv *priv = dev->data->dev_private;
4171         const struct rte_flow_action_age *age = action->conf;
4172
4173         if (!priv->config.devx || priv->counter_fallback)
4174                 return rte_flow_error_set(error, ENOTSUP,
4175                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4176                                           NULL,
4177                                           "age action not supported");
4178         if (!(action->conf))
4179                 return rte_flow_error_set(error, EINVAL,
4180                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4181                                           "configuration cannot be null");
4182         if (!(age->timeout))
4183                 return rte_flow_error_set(error, EINVAL,
4184                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4185                                           "invalid timeout value 0");
4186         if (action_flags & MLX5_FLOW_ACTION_AGE)
4187                 return rte_flow_error_set(error, EINVAL,
4188                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4189                                           "duplicate age actions set");
4190         return 0;
4191 }
4192
4193 /**
4194  * Validate the modify-header IPv4 DSCP actions.
4195  *
4196  * @param[in] action_flags
4197  *   Holds the actions detected until now.
4198  * @param[in] action
4199  *   Pointer to the modify action.
4200  * @param[in] item_flags
4201  *   Holds the items detected.
4202  * @param[out] error
4203  *   Pointer to error structure.
4204  *
4205  * @return
4206  *   0 on success, a negative errno value otherwise and rte_errno is set.
4207  */
4208 static int
4209 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
4210                                          const struct rte_flow_action *action,
4211                                          const uint64_t item_flags,
4212                                          struct rte_flow_error *error)
4213 {
4214         int ret = 0;
4215
4216         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4217         if (!ret) {
4218                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
4219                         return rte_flow_error_set(error, EINVAL,
4220                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4221                                                   NULL,
4222                                                   "no ipv4 item in pattern");
4223         }
4224         return ret;
4225 }
4226
4227 /**
4228  * Validate the modify-header IPv6 DSCP actions.
4229  *
4230  * @param[in] action_flags
4231  *   Holds the actions detected until now.
4232  * @param[in] action
4233  *   Pointer to the modify action.
4234  * @param[in] item_flags
4235  *   Holds the items detected.
4236  * @param[out] error
4237  *   Pointer to error structure.
4238  *
4239  * @return
4240  *   0 on success, a negative errno value otherwise and rte_errno is set.
4241  */
4242 static int
4243 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
4244                                          const struct rte_flow_action *action,
4245                                          const uint64_t item_flags,
4246                                          struct rte_flow_error *error)
4247 {
4248         int ret = 0;
4249
4250         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4251         if (!ret) {
4252                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
4253                         return rte_flow_error_set(error, EINVAL,
4254                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4255                                                   NULL,
4256                                                   "no ipv6 item in pattern");
4257         }
4258         return ret;
4259 }
4260
4261 /**
4262  * Match modify-header resource.
4263  *
4264  * @param entry
4265  *   Pointer to exist resource entry object.
4266  * @param ctx
4267  *   Pointer to new modify-header resource.
4268  *
4269  * @return
4270  *   0 on matching, -1 otherwise.
4271  */
4272 static int
4273 flow_dv_modify_hdr_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
4274 {
4275         struct mlx5_flow_dv_modify_hdr_resource *resource;
4276         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4277         uint32_t actions_len;
4278
4279         resource = (struct mlx5_flow_dv_modify_hdr_resource *)ctx;
4280         cache_resource = container_of(entry,
4281                                       struct mlx5_flow_dv_modify_hdr_resource,
4282                                       entry);
4283         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4284         if (resource->entry.key == cache_resource->entry.key &&
4285             resource->ft_type == cache_resource->ft_type &&
4286             resource->actions_num == cache_resource->actions_num &&
4287             resource->flags == cache_resource->flags &&
4288             !memcmp((const void *)resource->actions,
4289                     (const void *)cache_resource->actions,
4290                     actions_len))
4291                 return 0;
4292         return -1;
4293 }
4294
4295 /**
4296  * Validate the sample action.
4297  *
4298  * @param[in] action_flags
4299  *   Holds the actions detected until now.
4300  * @param[in] action
4301  *   Pointer to the sample action.
4302  * @param[in] dev
4303  *   Pointer to the Ethernet device structure.
4304  * @param[in] attr
4305  *   Attributes of flow that includes this action.
4306  * @param[out] error
4307  *   Pointer to error structure.
4308  *
4309  * @return
4310  *   0 on success, a negative errno value otherwise and rte_errno is set.
4311  */
4312 static int
4313 flow_dv_validate_action_sample(uint64_t action_flags,
4314                                const struct rte_flow_action *action,
4315                                struct rte_eth_dev *dev,
4316                                const struct rte_flow_attr *attr,
4317                                struct rte_flow_error *error)
4318 {
4319         struct mlx5_priv *priv = dev->data->dev_private;
4320         struct mlx5_dev_config *dev_conf = &priv->config;
4321         const struct rte_flow_action_sample *sample = action->conf;
4322         const struct rte_flow_action *act;
4323         uint64_t sub_action_flags = 0;
4324         uint16_t queue_index = 0xFFFF;
4325         int actions_n = 0;
4326         int ret;
4327         fdb_mirror = 0;
4328
4329         if (!sample)
4330                 return rte_flow_error_set(error, EINVAL,
4331                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4332                                           "configuration cannot be NULL");
4333         if (sample->ratio == 0)
4334                 return rte_flow_error_set(error, EINVAL,
4335                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4336                                           "ratio value starts from 1");
4337         if (!priv->config.devx || (sample->ratio > 0 && !priv->sampler_en))
4338                 return rte_flow_error_set(error, ENOTSUP,
4339                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4340                                           NULL,
4341                                           "sample action not supported");
4342         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
4343                 return rte_flow_error_set(error, EINVAL,
4344                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4345                                           "Multiple sample actions not "
4346                                           "supported");
4347         if (action_flags & MLX5_FLOW_ACTION_METER)
4348                 return rte_flow_error_set(error, EINVAL,
4349                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4350                                           "wrong action order, meter should "
4351                                           "be after sample action");
4352         if (action_flags & MLX5_FLOW_ACTION_JUMP)
4353                 return rte_flow_error_set(error, EINVAL,
4354                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4355                                           "wrong action order, jump should "
4356                                           "be after sample action");
4357         act = sample->actions;
4358         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
4359                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
4360                         return rte_flow_error_set(error, ENOTSUP,
4361                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4362                                                   act, "too many actions");
4363                 switch (act->type) {
4364                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4365                         ret = mlx5_flow_validate_action_queue(act,
4366                                                               sub_action_flags,
4367                                                               dev,
4368                                                               attr, error);
4369                         if (ret < 0)
4370                                 return ret;
4371                         queue_index = ((const struct rte_flow_action_queue *)
4372                                                         (act->conf))->index;
4373                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
4374                         ++actions_n;
4375                         break;
4376                 case RTE_FLOW_ACTION_TYPE_MARK:
4377                         ret = flow_dv_validate_action_mark(dev, act,
4378                                                            sub_action_flags,
4379                                                            attr, error);
4380                         if (ret < 0)
4381                                 return ret;
4382                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
4383                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
4384                                                 MLX5_FLOW_ACTION_MARK_EXT;
4385                         else
4386                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
4387                         ++actions_n;
4388                         break;
4389                 case RTE_FLOW_ACTION_TYPE_COUNT:
4390                         ret = flow_dv_validate_action_count(dev, error);
4391                         if (ret < 0)
4392                                 return ret;
4393                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
4394                         ++actions_n;
4395                         break;
4396                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
4397                         ret = flow_dv_validate_action_port_id(dev,
4398                                                               sub_action_flags,
4399                                                               act,
4400                                                               attr,
4401                                                               error);
4402                         if (ret)
4403                                 return ret;
4404                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
4405                         ++actions_n;
4406                         break;
4407                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4408                         ret = flow_dv_validate_action_raw_encap_decap
4409                                 (dev, NULL, act->conf, attr, &sub_action_flags,
4410                                  &actions_n, error);
4411                         if (ret < 0)
4412                                 return ret;
4413                         ++actions_n;
4414                         break;
4415                 default:
4416                         return rte_flow_error_set(error, ENOTSUP,
4417                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4418                                                   NULL,
4419                                                   "Doesn't support optional "
4420                                                   "action");
4421                 }
4422         }
4423         if (attr->ingress && !attr->transfer) {
4424                 if (!(sub_action_flags & MLX5_FLOW_ACTION_QUEUE))
4425                         return rte_flow_error_set(error, EINVAL,
4426                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4427                                                   NULL,
4428                                                   "Ingress must has a dest "
4429                                                   "QUEUE for Sample");
4430         } else if (attr->egress && !attr->transfer) {
4431                 return rte_flow_error_set(error, ENOTSUP,
4432                                           RTE_FLOW_ERROR_TYPE_ACTION,
4433                                           NULL,
4434                                           "Sample Only support Ingress "
4435                                           "or E-Switch");
4436         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
4437                 MLX5_ASSERT(attr->transfer);
4438                 if (sample->ratio > 1)
4439                         return rte_flow_error_set(error, ENOTSUP,
4440                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4441                                                   NULL,
4442                                                   "E-Switch doesn't support "
4443                                                   "any optional action "
4444                                                   "for sampling");
4445                 fdb_mirror = 1;
4446                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
4447                         return rte_flow_error_set(error, ENOTSUP,
4448                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4449                                                   NULL,
4450                                                   "unsupported action QUEUE");
4451                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
4452                         return rte_flow_error_set(error, EINVAL,
4453                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4454                                                   NULL,
4455                                                   "E-Switch must has a dest "
4456                                                   "port for mirroring");
4457         }
4458         /* Continue validation for Xcap actions.*/
4459         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
4460             (queue_index == 0xFFFF ||
4461              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
4462                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
4463                      MLX5_FLOW_XCAP_ACTIONS)
4464                         return rte_flow_error_set(error, ENOTSUP,
4465                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4466                                                   NULL, "encap and decap "
4467                                                   "combination aren't "
4468                                                   "supported");
4469                 if (!attr->transfer && attr->ingress && (sub_action_flags &
4470                                                         MLX5_FLOW_ACTION_ENCAP))
4471                         return rte_flow_error_set(error, ENOTSUP,
4472                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4473                                                   NULL, "encap is not supported"
4474                                                   " for ingress traffic");
4475         }
4476         return 0;
4477 }
4478
4479 /**
4480  * Find existing modify-header resource or create and register a new one.
4481  *
4482  * @param dev[in, out]
4483  *   Pointer to rte_eth_dev structure.
4484  * @param[in, out] resource
4485  *   Pointer to modify-header resource.
4486  * @parm[in, out] dev_flow
4487  *   Pointer to the dev_flow.
4488  * @param[out] error
4489  *   pointer to error structure.
4490  *
4491  * @return
4492  *   0 on success otherwise -errno and errno is set.
4493  */
4494 static int
4495 flow_dv_modify_hdr_resource_register
4496                         (struct rte_eth_dev *dev,
4497                          struct mlx5_flow_dv_modify_hdr_resource *resource,
4498                          struct mlx5_flow *dev_flow,
4499                          struct rte_flow_error *error)
4500 {
4501         struct mlx5_priv *priv = dev->data->dev_private;
4502         struct mlx5_dev_ctx_shared *sh = priv->sh;
4503         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4504         struct mlx5dv_dr_domain *ns;
4505         uint32_t actions_len;
4506         struct mlx5_hlist_entry *entry;
4507         union mlx5_flow_modify_hdr_key hdr_mod_key = {
4508                 {
4509                         .ft_type = resource->ft_type,
4510                         .actions_num = resource->actions_num,
4511                         .group = dev_flow->dv.group,
4512                         .cksum = 0,
4513                 }
4514         };
4515         int ret;
4516
4517         resource->flags = dev_flow->dv.group ? 0 :
4518                           MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4519         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
4520                                     resource->flags))
4521                 return rte_flow_error_set(error, EOVERFLOW,
4522                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4523                                           "too many modify header items");
4524         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
4525                 ns = sh->fdb_domain;
4526         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
4527                 ns = sh->tx_domain;
4528         else
4529                 ns = sh->rx_domain;
4530         /* Lookup a matching resource from cache. */
4531         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4532         hdr_mod_key.cksum = __rte_raw_cksum(resource->actions, actions_len, 0);
4533         resource->entry.key = hdr_mod_key.v64;
4534         entry = mlx5_hlist_lookup_ex(sh->modify_cmds, resource->entry.key,
4535                                      flow_dv_modify_hdr_resource_match,
4536                                      (void *)resource);
4537         if (entry) {
4538                 cache_resource = container_of(entry,
4539                                         struct mlx5_flow_dv_modify_hdr_resource,
4540                                         entry);
4541                 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
4542                         (void *)cache_resource,
4543                         rte_atomic32_read(&cache_resource->refcnt));
4544                 rte_atomic32_inc(&cache_resource->refcnt);
4545                 dev_flow->handle->dvh.modify_hdr = cache_resource;
4546                 return 0;
4547
4548         }
4549         /* Register new modify-header resource. */
4550         cache_resource = mlx5_malloc(MLX5_MEM_ZERO,
4551                                     sizeof(*cache_resource) + actions_len, 0,
4552                                     SOCKET_ID_ANY);
4553         if (!cache_resource)
4554                 return rte_flow_error_set(error, ENOMEM,
4555                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4556                                           "cannot allocate resource memory");
4557         *cache_resource = *resource;
4558         rte_memcpy(cache_resource->actions, resource->actions, actions_len);
4559         ret = mlx5_flow_os_create_flow_action_modify_header
4560                                         (sh->ctx, ns, cache_resource,
4561                                          actions_len, &cache_resource->action);
4562         if (ret) {
4563                 mlx5_free(cache_resource);
4564                 return rte_flow_error_set(error, ENOMEM,
4565                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4566                                           NULL, "cannot create action");
4567         }
4568         rte_atomic32_init(&cache_resource->refcnt);
4569         rte_atomic32_inc(&cache_resource->refcnt);
4570         if (mlx5_hlist_insert_ex(sh->modify_cmds, &cache_resource->entry,
4571                                  flow_dv_modify_hdr_resource_match,
4572                                  (void *)cache_resource)) {
4573                 claim_zero(mlx5_flow_os_destroy_flow_action
4574                                                 (cache_resource->action));
4575                 mlx5_free(cache_resource);
4576                 return rte_flow_error_set(error, EEXIST,
4577                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4578                                           NULL, "action exist");
4579         }
4580         dev_flow->handle->dvh.modify_hdr = cache_resource;
4581         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
4582                 (void *)cache_resource,
4583                 rte_atomic32_read(&cache_resource->refcnt));
4584         return 0;
4585 }
4586
4587 /**
4588  * Get DV flow counter by index.
4589  *
4590  * @param[in] dev
4591  *   Pointer to the Ethernet device structure.
4592  * @param[in] idx
4593  *   mlx5 flow counter index in the container.
4594  * @param[out] ppool
4595  *   mlx5 flow counter pool in the container,
4596  *
4597  * @return
4598  *   Pointer to the counter, NULL otherwise.
4599  */
4600 static struct mlx5_flow_counter *
4601 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
4602                            uint32_t idx,
4603                            struct mlx5_flow_counter_pool **ppool)
4604 {
4605         struct mlx5_priv *priv = dev->data->dev_private;
4606         struct mlx5_pools_container *cont;
4607         struct mlx5_flow_counter_pool *pool;
4608         uint32_t batch = 0;
4609
4610         idx--;
4611         if (idx >= MLX5_CNT_BATCH_OFFSET) {
4612                 idx -= MLX5_CNT_BATCH_OFFSET;
4613                 batch = 1;
4614         }
4615         cont = MLX5_CNT_CONTAINER(priv->sh, batch);
4616         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n);
4617         pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
4618         MLX5_ASSERT(pool);
4619         if (ppool)
4620                 *ppool = pool;
4621         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
4622 }
4623
4624 /**
4625  * Check the devx counter belongs to the pool.
4626  *
4627  * @param[in] pool
4628  *   Pointer to the counter pool.
4629  * @param[in] id
4630  *   The counter devx ID.
4631  *
4632  * @return
4633  *   True if counter belongs to the pool, false otherwise.
4634  */
4635 static bool
4636 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
4637 {
4638         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
4639                    MLX5_COUNTERS_PER_POOL;
4640
4641         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
4642                 return true;
4643         return false;
4644 }
4645
4646 /**
4647  * Get a pool by devx counter ID.
4648  *
4649  * @param[in] cont
4650  *   Pointer to the counter container.
4651  * @param[in] id
4652  *   The counter devx ID.
4653  *
4654  * @return
4655  *   The counter pool pointer if exists, NULL otherwise,
4656  */
4657 static struct mlx5_flow_counter_pool *
4658 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
4659 {
4660         uint32_t i;
4661
4662         /* Check last used pool. */
4663         if (cont->last_pool_idx != POOL_IDX_INVALID &&
4664             flow_dv_is_counter_in_pool(cont->pools[cont->last_pool_idx], id))
4665                 return cont->pools[cont->last_pool_idx];
4666         /* ID out of range means no suitable pool in the container. */
4667         if (id > cont->max_id || id < cont->min_id)
4668                 return NULL;
4669         /*
4670          * Find the pool from the end of the container, since mostly counter
4671          * ID is sequence increasing, and the last pool should be the needed
4672          * one.
4673          */
4674         i = rte_atomic16_read(&cont->n_valid);
4675         while (i--) {
4676                 struct mlx5_flow_counter_pool *pool = cont->pools[i];
4677
4678                 if (flow_dv_is_counter_in_pool(pool, id))
4679                         return pool;
4680         }
4681         return NULL;
4682 }
4683
4684 /**
4685  * Allocate a new memory for the counter values wrapped by all the needed
4686  * management.
4687  *
4688  * @param[in] dev
4689  *   Pointer to the Ethernet device structure.
4690  * @param[in] raws_n
4691  *   The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
4692  *
4693  * @return
4694  *   The new memory management pointer on success, otherwise NULL and rte_errno
4695  *   is set.
4696  */
4697 static struct mlx5_counter_stats_mem_mng *
4698 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
4699 {
4700         struct mlx5_priv *priv = dev->data->dev_private;
4701         struct mlx5_dev_ctx_shared *sh = priv->sh;
4702         struct mlx5_devx_mkey_attr mkey_attr;
4703         struct mlx5_counter_stats_mem_mng *mem_mng;
4704         volatile struct flow_counter_stats *raw_data;
4705         int size = (sizeof(struct flow_counter_stats) *
4706                         MLX5_COUNTERS_PER_POOL +
4707                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
4708                         sizeof(struct mlx5_counter_stats_mem_mng);
4709         size_t pgsize = rte_mem_page_size();
4710         if (pgsize == (size_t)-1) {
4711                 DRV_LOG(ERR, "Failed to get mem page size");
4712                 rte_errno = ENOMEM;
4713                 return NULL;
4714         }
4715         uint8_t *mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize,
4716                                   SOCKET_ID_ANY);
4717         int i;
4718
4719         if (!mem) {
4720                 rte_errno = ENOMEM;
4721                 return NULL;
4722         }
4723         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
4724         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
4725         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
4726                                                  IBV_ACCESS_LOCAL_WRITE);
4727         if (!mem_mng->umem) {
4728                 rte_errno = errno;
4729                 mlx5_free(mem);
4730                 return NULL;
4731         }
4732         mkey_attr.addr = (uintptr_t)mem;
4733         mkey_attr.size = size;
4734         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
4735         mkey_attr.pd = sh->pdn;
4736         mkey_attr.log_entity_size = 0;
4737         mkey_attr.pg_access = 0;
4738         mkey_attr.klm_array = NULL;
4739         mkey_attr.klm_num = 0;
4740         if (priv->config.hca_attr.relaxed_ordering_write &&
4741                 priv->config.hca_attr.relaxed_ordering_read  &&
4742                 !haswell_broadwell_cpu)
4743                 mkey_attr.relaxed_ordering = 1;
4744         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
4745         if (!mem_mng->dm) {
4746                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
4747                 rte_errno = errno;
4748                 mlx5_free(mem);
4749                 return NULL;
4750         }
4751         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
4752         raw_data = (volatile struct flow_counter_stats *)mem;
4753         for (i = 0; i < raws_n; ++i) {
4754                 mem_mng->raws[i].mem_mng = mem_mng;
4755                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
4756         }
4757         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
4758         return mem_mng;
4759 }
4760
4761 /**
4762  * Resize a counter container.
4763  *
4764  * @param[in] dev
4765  *   Pointer to the Ethernet device structure.
4766  * @param[in] batch
4767  *   Whether the pool is for counter that was allocated by batch command.
4768  *
4769  * @return
4770  *   0 on success, otherwise negative errno value and rte_errno is set.
4771  */
4772 static int
4773 flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch)
4774 {
4775         struct mlx5_priv *priv = dev->data->dev_private;
4776         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch);
4777         struct mlx5_counter_stats_mem_mng *mem_mng = NULL;
4778         void *old_pools = cont->pools;
4779         uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
4780         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
4781         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
4782
4783         if (!pools) {
4784                 rte_errno = ENOMEM;
4785                 return -ENOMEM;
4786         }
4787         if (old_pools)
4788                 memcpy(pools, old_pools, cont->n *
4789                                        sizeof(struct mlx5_flow_counter_pool *));
4790         /*
4791          * Fallback mode query the counter directly, no background query
4792          * resources are needed.
4793          */
4794         if (!priv->counter_fallback) {
4795                 int i;
4796
4797                 mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
4798                           MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
4799                 if (!mem_mng) {
4800                         mlx5_free(pools);
4801                         return -ENOMEM;
4802                 }
4803                 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
4804                         LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
4805                                          mem_mng->raws +
4806                                          MLX5_CNT_CONTAINER_RESIZE +
4807                                          i, next);
4808         }
4809         rte_spinlock_lock(&cont->resize_sl);
4810         cont->n = resize;
4811         cont->mem_mng = mem_mng;
4812         cont->pools = pools;
4813         rte_spinlock_unlock(&cont->resize_sl);
4814         if (old_pools)
4815                 mlx5_free(old_pools);
4816         return 0;
4817 }
4818
4819 /**
4820  * Query a devx flow counter.
4821  *
4822  * @param[in] dev
4823  *   Pointer to the Ethernet device structure.
4824  * @param[in] cnt
4825  *   Index to the flow counter.
4826  * @param[out] pkts
4827  *   The statistics value of packets.
4828  * @param[out] bytes
4829  *   The statistics value of bytes.
4830  *
4831  * @return
4832  *   0 on success, otherwise a negative errno value and rte_errno is set.
4833  */
4834 static inline int
4835 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
4836                      uint64_t *bytes)
4837 {
4838         struct mlx5_priv *priv = dev->data->dev_private;
4839         struct mlx5_flow_counter_pool *pool = NULL;
4840         struct mlx5_flow_counter *cnt;
4841         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4842         int offset;
4843
4844         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4845         MLX5_ASSERT(pool);
4846         if (counter < MLX5_CNT_BATCH_OFFSET) {
4847                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4848                 if (priv->counter_fallback)
4849                         return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
4850                                         0, pkts, bytes, 0, NULL, NULL, 0);
4851         }
4852
4853         rte_spinlock_lock(&pool->sl);
4854         /*
4855          * The single counters allocation may allocate smaller ID than the
4856          * current allocated in parallel to the host reading.
4857          * In this case the new counter values must be reported as 0.
4858          */
4859         if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) {
4860                 *pkts = 0;
4861                 *bytes = 0;
4862         } else {
4863                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
4864                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
4865                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
4866         }
4867         rte_spinlock_unlock(&pool->sl);
4868         return 0;
4869 }
4870
4871 /**
4872  * Create and initialize a new counter pool.
4873  *
4874  * @param[in] dev
4875  *   Pointer to the Ethernet device structure.
4876  * @param[out] dcs
4877  *   The devX counter handle.
4878  * @param[in] batch
4879  *   Whether the pool is for counter that was allocated by batch command.
4880  * @param[in] age
4881  *   Whether the pool is for counter that was allocated for aging.
4882  * @param[in/out] cont_cur
4883  *   Pointer to the container pointer, it will be update in pool resize.
4884  *
4885  * @return
4886  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
4887  */
4888 static struct mlx5_flow_counter_pool *
4889 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
4890                     uint32_t batch, uint32_t age)
4891 {
4892         struct mlx5_priv *priv = dev->data->dev_private;
4893         struct mlx5_flow_counter_pool *pool;
4894         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch);
4895         int16_t n_valid = rte_atomic16_read(&cont->n_valid);
4896         uint32_t size = sizeof(*pool);
4897
4898         if (cont->n == n_valid && flow_dv_container_resize(dev, batch))
4899                 return NULL;
4900         size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
4901         size += (batch ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
4902         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE);
4903         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
4904         if (!pool) {
4905                 rte_errno = ENOMEM;
4906                 return NULL;
4907         }
4908         pool->min_dcs = dcs;
4909         if (!priv->counter_fallback)
4910                 pool->raw = cont->mem_mng->raws + n_valid %
4911                                                       MLX5_CNT_CONTAINER_RESIZE;
4912         pool->raw_hw = NULL;
4913         pool->type = 0;
4914         pool->type |= (batch ? 0 :  CNT_POOL_TYPE_EXT);
4915         pool->type |= (!age ? 0 :  CNT_POOL_TYPE_AGE);
4916         pool->query_gen = 0;
4917         rte_spinlock_init(&pool->sl);
4918         TAILQ_INIT(&pool->counters[0]);
4919         TAILQ_INIT(&pool->counters[1]);
4920         TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
4921         pool->index = n_valid;
4922         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
4923         cont->pools[n_valid] = pool;
4924         if (!batch) {
4925                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
4926
4927                 if (base < cont->min_id)
4928                         cont->min_id = base;
4929                 if (base > cont->max_id)
4930                         cont->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
4931                 cont->last_pool_idx = pool->index;
4932         }
4933         /* Pool initialization must be updated before host thread access. */
4934         rte_io_wmb();
4935         rte_atomic16_add(&cont->n_valid, 1);
4936         return pool;
4937 }
4938
4939 /**
4940  * Restore skipped counters in the pool.
4941  *
4942  * As counter pool query requires the first counter dcs
4943  * ID start with 4 alinged, if the pool counters with
4944  * min_dcs ID are not aligned with 4, the counters will
4945  * be skipped.
4946  * Once other min_dcs ID less than these skipped counter
4947  * dcs ID appears, the skipped counters will be safe to
4948  * use.
4949  * Should be called when min_dcs is updated.
4950  *
4951  * @param[in] pool
4952  *   Current counter pool.
4953  * @param[in] last_min_dcs
4954  *   Last min_dcs.
4955  */
4956 static void
4957 flow_dv_counter_restore(struct mlx5_flow_counter_pool *pool,
4958                         struct mlx5_devx_obj *last_min_dcs)
4959 {
4960         struct mlx5_flow_counter_ext *cnt_ext;
4961         uint32_t offset, new_offset;
4962         uint32_t skip_cnt = 0;
4963         uint32_t i;
4964
4965         if (!pool->skip_cnt)
4966                 return;
4967         /*
4968          * If last min_dcs is not valid. The skipped counter may even after
4969          * last min_dcs, set the offset to the whole pool.
4970          */
4971         if (last_min_dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))
4972                 offset = MLX5_COUNTERS_PER_POOL;
4973         else
4974                 offset = last_min_dcs->id % MLX5_COUNTERS_PER_POOL;
4975         new_offset = pool->min_dcs->id % MLX5_COUNTERS_PER_POOL;
4976         /*
4977          * Check the counters from 1 to the last_min_dcs range. Counters
4978          * before new min_dcs indicates pool still has skipped counters.
4979          * Counters be skipped after new min_dcs will be ready to use.
4980          * Offset 0 counter must be empty or min_dcs, start from 1.
4981          */
4982         for (i = 1; i < offset; i++) {
4983                 cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
4984                 if (cnt_ext->skipped) {
4985                         if (i > new_offset) {
4986                                 cnt_ext->skipped = 0;
4987                                 TAILQ_INSERT_TAIL
4988                                         (&pool->counters[pool->query_gen],
4989                                          MLX5_POOL_GET_CNT(pool, i), next);
4990                         } else {
4991                                 skip_cnt++;
4992                         }
4993                 }
4994         }
4995         if (!skip_cnt)
4996                 pool->skip_cnt = 0;
4997 }
4998
4999 /**
5000  * Prepare a new counter and/or a new counter pool.
5001  *
5002  * @param[in] dev
5003  *   Pointer to the Ethernet device structure.
5004  * @param[out] cnt_free
5005  *   Where to put the pointer of a new counter.
5006  * @param[in] batch
5007  *   Whether the pool is for counter that was allocated by batch command.
5008  * @param[in] age
5009  *   Whether the pool is for counter that was allocated for aging.
5010  *
5011  * @return
5012  *   The counter pool pointer and @p cnt_free is set on success,
5013  *   NULL otherwise and rte_errno is set.
5014  */
5015 static struct mlx5_flow_counter_pool *
5016 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
5017                              struct mlx5_flow_counter **cnt_free,
5018                              uint32_t batch, uint32_t age)
5019 {
5020         struct mlx5_priv *priv = dev->data->dev_private;
5021         struct mlx5_pools_container *cont;
5022         struct mlx5_flow_counter_pool *pool;
5023         struct mlx5_counters tmp_tq;
5024         struct mlx5_devx_obj *last_min_dcs;
5025         struct mlx5_devx_obj *dcs = NULL;
5026         struct mlx5_flow_counter *cnt;
5027         enum mlx5_counter_type cnt_type =
5028                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
5029         uint32_t add2other;
5030         uint32_t i;
5031
5032         cont = MLX5_CNT_CONTAINER(priv->sh, batch);
5033         if (!batch) {
5034 retry:
5035                 add2other = 0;
5036                 /* bulk_bitmap must be 0 for single counter allocation. */
5037                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
5038                 if (!dcs)
5039                         return NULL;
5040                 pool = flow_dv_find_pool_by_id(cont, dcs->id);
5041                 /*
5042                  * If pool eixsts but with other type, counter will be added
5043                  * to the other pool, need to reallocate new counter in the
5044                  * ragne with same type later.
5045                  */
5046                 if (!pool) {
5047                         pool = flow_dv_pool_create(dev, dcs, batch,
5048                                                    age);
5049                         if (!pool) {
5050                                 mlx5_devx_cmd_destroy(dcs);
5051                                 return NULL;
5052                         }
5053                 } else if ((!!IS_AGE_POOL(pool)) != age) {
5054                         add2other = 1;
5055                 }
5056                 if ((dcs->id < pool->min_dcs->id ||
5057                     pool->min_dcs->id &
5058                     (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1)) &&
5059                     !(dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))) {
5060                         /*
5061                          * Update the pool min_dcs only if current dcs is
5062                          * valid and exist min_dcs is not valid or greater
5063                          * than new dcs.
5064                          */
5065                         last_min_dcs = pool->min_dcs;
5066                         rte_atomic64_set(&pool->a64_dcs,
5067                                          (int64_t)(uintptr_t)dcs);
5068                         /*
5069                          * Restore any skipped counters if the new min_dcs
5070                          * ID is smaller or min_dcs is not valid.
5071                          */
5072                         if (dcs->id < last_min_dcs->id ||
5073                             last_min_dcs->id &
5074                             (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))
5075                                 flow_dv_counter_restore(pool, last_min_dcs);
5076                 }
5077                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
5078                 cnt = MLX5_POOL_GET_CNT(pool, i);
5079                 cnt->pool = pool;
5080                 MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs;
5081                 /*
5082                  * If min_dcs is not valid, it means the new allocated dcs
5083                  * also fail to become the valid min_dcs, just skip it.
5084                  * Or if min_dcs is valid, and new dcs ID is smaller than
5085                  * min_dcs, but not become the min_dcs, also skip it.
5086                  */
5087                 if (pool->min_dcs->id &
5088                     (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1) ||
5089                     dcs->id < pool->min_dcs->id) {
5090                         MLX5_GET_POOL_CNT_EXT(pool, i)->skipped = 1;
5091                         pool->skip_cnt = 1;
5092                         goto retry;
5093                 }
5094                 if (add2other) {
5095                         TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen],
5096                                           cnt, next);
5097                         goto retry;
5098                 }
5099                 *cnt_free = cnt;
5100                 return pool;
5101         }
5102         /* bulk_bitmap is in 128 counters units. */
5103         if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
5104                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
5105         if (!dcs) {
5106                 rte_errno = ENODATA;
5107                 return NULL;
5108         }
5109         pool = flow_dv_pool_create(dev, dcs, batch, age);
5110         if (!pool) {
5111                 mlx5_devx_cmd_destroy(dcs);
5112                 return NULL;
5113         }
5114         TAILQ_INIT(&tmp_tq);
5115         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
5116                 cnt = MLX5_POOL_GET_CNT(pool, i);
5117                 cnt->pool = pool;
5118                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
5119         }
5120         rte_spinlock_lock(&cont->csl);
5121         TAILQ_CONCAT(&cont->counters[cnt_type], &tmp_tq, next);
5122         rte_spinlock_unlock(&cont->csl);
5123         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
5124         (*cnt_free)->pool = pool;
5125         return pool;
5126 }
5127
5128 /**
5129  * Search for existed shared counter.
5130  *
5131  * @param[in] dev
5132  *   Pointer to the Ethernet device structure.
5133  * @param[in] id
5134  *   The shared counter ID to search.
5135  * @param[out] ppool
5136  *   mlx5 flow counter pool in the container,
5137  *
5138  * @return
5139  *   NULL if not existed, otherwise pointer to the shared extend counter.
5140  */
5141 static struct mlx5_flow_counter_ext *
5142 flow_dv_counter_shared_search(struct rte_eth_dev *dev, uint32_t id,
5143                               struct mlx5_flow_counter_pool **ppool)
5144 {
5145         struct mlx5_priv *priv = dev->data->dev_private;
5146         union mlx5_l3t_data data;
5147         uint32_t cnt_idx;
5148
5149         if (mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) || !data.dword)
5150                 return NULL;
5151         cnt_idx = data.dword;
5152         /*
5153          * Shared counters don't have age info. The counter extend is after
5154          * the counter datat structure.
5155          */
5156         return (struct mlx5_flow_counter_ext *)
5157                ((flow_dv_counter_get_by_idx(dev, cnt_idx, ppool)) + 1);
5158 }
5159
5160 /**
5161  * Allocate a flow counter.
5162  *
5163  * @param[in] dev
5164  *   Pointer to the Ethernet device structure.
5165  * @param[in] shared
5166  *   Indicate if this counter is shared with other flows.
5167  * @param[in] id
5168  *   Counter identifier.
5169  * @param[in] group
5170  *   Counter flow group.
5171  * @param[in] age
5172  *   Whether the counter was allocated for aging.
5173  *
5174  * @return
5175  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
5176  */
5177 static uint32_t
5178 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
5179                       uint16_t group, uint32_t age)
5180 {
5181         struct mlx5_priv *priv = dev->data->dev_private;
5182         struct mlx5_flow_counter_pool *pool = NULL;
5183         struct mlx5_flow_counter *cnt_free = NULL;
5184         struct mlx5_flow_counter_ext *cnt_ext = NULL;
5185         /*
5186          * Currently group 0 flow counter cannot be assigned to a flow if it is
5187          * not the first one in the batch counter allocation, so it is better
5188          * to allocate counters one by one for these flows in a separate
5189          * container.
5190          * A counter can be shared between different groups so need to take
5191          * shared counters from the single container.
5192          */
5193         uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0;
5194         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch);
5195         enum mlx5_counter_type cnt_type =
5196                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
5197         uint32_t cnt_idx;
5198
5199         if (!priv->config.devx) {
5200                 rte_errno = ENOTSUP;
5201                 return 0;
5202         }
5203         if (shared) {
5204                 cnt_ext = flow_dv_counter_shared_search(dev, id, &pool);
5205                 if (cnt_ext) {
5206                         if (cnt_ext->ref_cnt + 1 == 0) {
5207                                 rte_errno = E2BIG;
5208                                 return 0;
5209                         }
5210                         cnt_ext->ref_cnt++;
5211                         cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL +
5212                                   (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL)
5213                                   + 1;
5214                         return cnt_idx;
5215                 }
5216         }
5217         /* Get free counters from container. */
5218         rte_spinlock_lock(&cont->csl);
5219         cnt_free = TAILQ_FIRST(&cont->counters[cnt_type]);
5220         if (cnt_free)
5221                 TAILQ_REMOVE(&cont->counters[cnt_type], cnt_free, next);
5222         rte_spinlock_unlock(&cont->csl);
5223         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free,
5224                                                        batch, age))
5225                 goto err;
5226         pool = cnt_free->pool;
5227         if (!batch)
5228                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free);
5229         /* Create a DV counter action only in the first time usage. */
5230         if (!cnt_free->action) {
5231                 uint16_t offset;
5232                 struct mlx5_devx_obj *dcs;
5233                 int ret;
5234
5235                 if (batch) {
5236                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
5237                         dcs = pool->min_dcs;
5238                 } else {
5239                         offset = 0;
5240                         dcs = cnt_ext->dcs;
5241                 }
5242                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
5243                                                             &cnt_free->action);
5244                 if (ret) {
5245                         rte_errno = errno;
5246                         goto err;
5247                 }
5248         }
5249         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
5250                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
5251         cnt_idx += batch * MLX5_CNT_BATCH_OFFSET;
5252         /* Update the counter reset values. */
5253         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
5254                                  &cnt_free->bytes))
5255                 goto err;
5256         if (cnt_ext) {
5257                 cnt_ext->shared = shared;
5258                 cnt_ext->ref_cnt = 1;
5259                 cnt_ext->id = id;
5260                 if (shared) {
5261                         union mlx5_l3t_data data;
5262
5263                         data.dword = cnt_idx;
5264                         if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
5265                                 return 0;
5266                 }
5267         }
5268         if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on)
5269                 /* Start the asynchronous batch query by the host thread. */
5270                 mlx5_set_query_alarm(priv->sh);
5271         return cnt_idx;
5272 err:
5273         if (cnt_free) {
5274                 cnt_free->pool = pool;
5275                 rte_spinlock_lock(&cont->csl);
5276                 TAILQ_INSERT_TAIL(&cont->counters[cnt_type], cnt_free, next);
5277                 rte_spinlock_unlock(&cont->csl);
5278         }
5279         return 0;
5280 }
5281
5282 /**
5283  * Get age param from counter index.
5284  *
5285  * @param[in] dev
5286  *   Pointer to the Ethernet device structure.
5287  * @param[in] counter
5288  *   Index to the counter handler.
5289  *
5290  * @return
5291  *   The aging parameter specified for the counter index.
5292  */
5293 static struct mlx5_age_param*
5294 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
5295                                 uint32_t counter)
5296 {
5297         struct mlx5_flow_counter *cnt;
5298         struct mlx5_flow_counter_pool *pool = NULL;
5299
5300         flow_dv_counter_get_by_idx(dev, counter, &pool);
5301         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
5302         cnt = MLX5_POOL_GET_CNT(pool, counter);
5303         return MLX5_CNT_TO_AGE(cnt);
5304 }
5305
5306 /**
5307  * Remove a flow counter from aged counter list.
5308  *
5309  * @param[in] dev
5310  *   Pointer to the Ethernet device structure.
5311  * @param[in] counter
5312  *   Index to the counter handler.
5313  * @param[in] cnt
5314  *   Pointer to the counter handler.
5315  */
5316 static void
5317 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
5318                                 uint32_t counter, struct mlx5_flow_counter *cnt)
5319 {
5320         struct mlx5_age_info *age_info;
5321         struct mlx5_age_param *age_param;
5322         struct mlx5_priv *priv = dev->data->dev_private;
5323         uint16_t expected = AGE_CANDIDATE;
5324
5325         age_info = GET_PORT_AGE_INFO(priv);
5326         age_param = flow_dv_counter_idx_get_age(dev, counter);
5327         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
5328                                          AGE_FREE, false, __ATOMIC_RELAXED,
5329                                          __ATOMIC_RELAXED)) {
5330                 /**
5331                  * We need the lock even it is age timeout,
5332                  * since counter may still in process.
5333                  */
5334                 rte_spinlock_lock(&age_info->aged_sl);
5335                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
5336                 rte_spinlock_unlock(&age_info->aged_sl);
5337                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
5338         }
5339 }
5340
5341 /**
5342  * Release a flow counter.
5343  *
5344  * @param[in] dev
5345  *   Pointer to the Ethernet device structure.
5346  * @param[in] counter
5347  *   Index to the counter handler.
5348  */
5349 static void
5350 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
5351 {
5352         struct mlx5_priv *priv = dev->data->dev_private;
5353         struct mlx5_flow_counter_pool *pool = NULL;
5354         struct mlx5_flow_counter *cnt;
5355         struct mlx5_flow_counter_ext *cnt_ext = NULL;
5356         enum mlx5_counter_type cnt_type;
5357
5358         if (!counter)
5359                 return;
5360         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
5361         MLX5_ASSERT(pool);
5362         if (counter < MLX5_CNT_BATCH_OFFSET) {
5363                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
5364                 if (cnt_ext) {
5365                         if (--cnt_ext->ref_cnt)
5366                                 return;
5367                         if (cnt_ext->shared)
5368                                 mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
5369                                                      cnt_ext->id);
5370                 }
5371         }
5372         if (IS_AGE_POOL(pool))
5373                 flow_dv_counter_remove_from_age(dev, counter, cnt);
5374         cnt->pool = pool;
5375         /*
5376          * Put the counter back to list to be updated in none fallback mode.
5377          * Currently, we are using two list alternately, while one is in query,
5378          * add the freed counter to the other list based on the pool query_gen
5379          * value. After query finishes, add counter the list to the global
5380          * container counter list. The list changes while query starts. In
5381          * this case, lock will not be needed as query callback and release
5382          * function both operate with the different list.
5383          *
5384          */
5385         if (!priv->counter_fallback) {
5386                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
5387         } else {
5388                 cnt_type = IS_AGE_POOL(pool) ? MLX5_COUNTER_TYPE_AGE :
5389                                                MLX5_COUNTER_TYPE_ORIGIN;
5390                 TAILQ_INSERT_TAIL(&((MLX5_CNT_CONTAINER
5391                                   (priv->sh, 0))->counters[cnt_type]),
5392                                   cnt, next);
5393         }
5394 }
5395
5396 /**
5397  * Verify the @p attributes will be correctly understood by the NIC and store
5398  * them in the @p flow if everything is correct.
5399  *
5400  * @param[in] dev
5401  *   Pointer to dev struct.
5402  * @param[in] attributes
5403  *   Pointer to flow attributes
5404  * @param[in] external
5405  *   This flow rule is created by request external to PMD.
5406  * @param[out] error
5407  *   Pointer to error structure.
5408  *
5409  * @return
5410  *   - 0 on success and non root table.
5411  *   - 1 on success and root table.
5412  *   - a negative errno value otherwise and rte_errno is set.
5413  */
5414 static int
5415 flow_dv_validate_attributes(struct rte_eth_dev *dev,
5416                             const struct rte_flow_attr *attributes,
5417                             bool external __rte_unused,
5418                             struct rte_flow_error *error)
5419 {
5420         struct mlx5_priv *priv = dev->data->dev_private;
5421         uint32_t priority_max = priv->config.flow_prio - 1;
5422         int ret = 0;
5423
5424 #ifndef HAVE_MLX5DV_DR
5425         if (attributes->group)
5426                 return rte_flow_error_set(error, ENOTSUP,
5427                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
5428                                           NULL,
5429                                           "groups are not supported");
5430 #else
5431         uint32_t table = 0;
5432
5433         ret = mlx5_flow_group_to_table(attributes, external,
5434                                        attributes->group, !!priv->fdb_def_rule,
5435                                        &table, error);
5436         if (ret)
5437                 return ret;
5438         if (!table)
5439                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
5440 #endif
5441         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
5442             attributes->priority >= priority_max)
5443                 return rte_flow_error_set(error, ENOTSUP,
5444                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
5445                                           NULL,
5446                                           "priority out of range");
5447         if (attributes->transfer) {
5448                 if (!priv->config.dv_esw_en)
5449                         return rte_flow_error_set
5450                                 (error, ENOTSUP,
5451                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5452                                  "E-Switch dr is not supported");
5453                 if (!(priv->representor || priv->master))
5454                         return rte_flow_error_set
5455                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5456                                  NULL, "E-Switch configuration can only be"
5457                                  " done by a master or a representor device");
5458                 if (attributes->egress)
5459                         return rte_flow_error_set
5460                                 (error, ENOTSUP,
5461                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
5462                                  "egress is not supported");
5463         }
5464         if (!(attributes->egress ^ attributes->ingress))
5465                 return rte_flow_error_set(error, ENOTSUP,
5466                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
5467                                           "must specify exactly one of "
5468                                           "ingress or egress");
5469         return ret;
5470 }
5471
5472 /**
5473  * Internal validation function. For validating both actions and items.
5474  *
5475  * @param[in] dev
5476  *   Pointer to the rte_eth_dev structure.
5477  * @param[in] attr
5478  *   Pointer to the flow attributes.
5479  * @param[in] items
5480  *   Pointer to the list of items.
5481  * @param[in] actions
5482  *   Pointer to the list of actions.
5483  * @param[in] external
5484  *   This flow rule is created by request external to PMD.
5485  * @param[in] hairpin
5486  *   Number of hairpin TX actions, 0 means classic flow.
5487  * @param[out] error
5488  *   Pointer to the error structure.
5489  *
5490  * @return
5491  *   0 on success, a negative errno value otherwise and rte_errno is set.
5492  */
5493 static int
5494 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
5495                  const struct rte_flow_item items[],
5496                  const struct rte_flow_action actions[],
5497                  bool external, int hairpin, struct rte_flow_error *error)
5498 {
5499         int ret;
5500         uint64_t action_flags = 0;
5501         uint64_t item_flags = 0;
5502         uint64_t last_item = 0;
5503         uint8_t next_protocol = 0xff;
5504         uint16_t ether_type = 0;
5505         int actions_n = 0;
5506         uint8_t item_ipv6_proto = 0;
5507         const struct rte_flow_item *gre_item = NULL;
5508         const struct rte_flow_action_raw_decap *decap;
5509         const struct rte_flow_action_raw_encap *encap;
5510         const struct rte_flow_action_rss *rss;
5511         const struct rte_flow_item_tcp nic_tcp_mask = {
5512                 .hdr = {
5513                         .tcp_flags = 0xFF,
5514                         .src_port = RTE_BE16(UINT16_MAX),
5515                         .dst_port = RTE_BE16(UINT16_MAX),
5516                 }
5517         };
5518         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
5519                 .hdr = {
5520                         .src_addr =
5521                         "\xff\xff\xff\xff\xff\xff\xff\xff"
5522                         "\xff\xff\xff\xff\xff\xff\xff\xff",
5523                         .dst_addr =
5524                         "\xff\xff\xff\xff\xff\xff\xff\xff"
5525                         "\xff\xff\xff\xff\xff\xff\xff\xff",
5526                         .vtc_flow = RTE_BE32(0xffffffff),
5527                         .proto = 0xff,
5528                         .hop_limits = 0xff,
5529                 },
5530                 .has_frag_ext = 1,
5531         };
5532         const struct rte_flow_item_ecpri nic_ecpri_mask = {
5533                 .hdr = {
5534                         .common = {
5535                                 .u32 =
5536                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
5537                                         .type = 0xFF,
5538                                         }).u32),
5539                         },
5540                         .dummy[0] = 0xffffffff,
5541                 },
5542         };
5543         struct mlx5_priv *priv = dev->data->dev_private;
5544         struct mlx5_dev_config *dev_conf = &priv->config;
5545         uint16_t queue_index = 0xFFFF;
5546         const struct rte_flow_item_vlan *vlan_m = NULL;
5547         int16_t rw_act_num = 0;
5548         uint64_t is_root;
5549
5550         if (items == NULL)
5551                 return -1;
5552         ret = flow_dv_validate_attributes(dev, attr, external, error);
5553         if (ret < 0)
5554                 return ret;
5555         is_root = (uint64_t)ret;
5556         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5557                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
5558                 int type = items->type;
5559
5560                 if (!mlx5_flow_os_item_supported(type))
5561                         return rte_flow_error_set(error, ENOTSUP,
5562                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5563                                                   NULL, "item not supported");
5564                 switch (type) {
5565                 case RTE_FLOW_ITEM_TYPE_VOID:
5566                         break;
5567                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
5568                         ret = flow_dv_validate_item_port_id
5569                                         (dev, items, attr, item_flags, error);
5570                         if (ret < 0)
5571                                 return ret;
5572                         last_item = MLX5_FLOW_ITEM_PORT_ID;
5573                         break;
5574                 case RTE_FLOW_ITEM_TYPE_ETH:
5575                         ret = mlx5_flow_validate_item_eth(items, item_flags,
5576                                                           error);
5577                         if (ret < 0)
5578                                 return ret;
5579                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
5580                                              MLX5_FLOW_LAYER_OUTER_L2;
5581                         if (items->mask != NULL && items->spec != NULL) {
5582                                 ether_type =
5583                                         ((const struct rte_flow_item_eth *)
5584                                          items->spec)->type;
5585                                 ether_type &=
5586                                         ((const struct rte_flow_item_eth *)
5587                                          items->mask)->type;
5588                                 ether_type = rte_be_to_cpu_16(ether_type);
5589                         } else {
5590                                 ether_type = 0;
5591                         }
5592                         break;
5593                 case RTE_FLOW_ITEM_TYPE_VLAN:
5594                         ret = flow_dv_validate_item_vlan(items, item_flags,
5595                                                          dev, error);
5596                         if (ret < 0)
5597                                 return ret;
5598                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
5599                                              MLX5_FLOW_LAYER_OUTER_VLAN;
5600                         if (items->mask != NULL && items->spec != NULL) {
5601                                 ether_type =
5602                                         ((const struct rte_flow_item_vlan *)
5603                                          items->spec)->inner_type;
5604                                 ether_type &=
5605                                         ((const struct rte_flow_item_vlan *)
5606                                          items->mask)->inner_type;
5607                                 ether_type = rte_be_to_cpu_16(ether_type);
5608                         } else {
5609                                 ether_type = 0;
5610                         }
5611                         /* Store outer VLAN mask for of_push_vlan action. */
5612                         if (!tunnel)
5613                                 vlan_m = items->mask;
5614                         break;
5615                 case RTE_FLOW_ITEM_TYPE_IPV4:
5616                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5617                                                   &item_flags, &tunnel);
5618                         ret = flow_dv_validate_item_ipv4(items, item_flags,
5619                                                          last_item, ether_type,
5620                                                          error);
5621                         if (ret < 0)
5622                                 return ret;
5623                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5624                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5625                         if (items->mask != NULL &&
5626                             ((const struct rte_flow_item_ipv4 *)
5627                              items->mask)->hdr.next_proto_id) {
5628                                 next_protocol =
5629                                         ((const struct rte_flow_item_ipv4 *)
5630                                          (items->spec))->hdr.next_proto_id;
5631                                 next_protocol &=
5632                                         ((const struct rte_flow_item_ipv4 *)
5633                                          (items->mask))->hdr.next_proto_id;
5634                         } else {
5635                                 /* Reset for inner layer. */
5636                                 next_protocol = 0xff;
5637                         }
5638                         break;
5639                 case RTE_FLOW_ITEM_TYPE_IPV6:
5640                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5641                                                   &item_flags, &tunnel);
5642                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
5643                                                            last_item,
5644                                                            ether_type,
5645                                                            &nic_ipv6_mask,
5646                                                            error);
5647                         if (ret < 0)
5648                                 return ret;
5649                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5650                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5651                         if (items->mask != NULL &&
5652                             ((const struct rte_flow_item_ipv6 *)
5653                              items->mask)->hdr.proto) {
5654                                 item_ipv6_proto =
5655                                         ((const struct rte_flow_item_ipv6 *)
5656                                          items->spec)->hdr.proto;
5657                                 next_protocol =
5658                                         ((const struct rte_flow_item_ipv6 *)
5659                                          items->spec)->hdr.proto;
5660                                 next_protocol &=
5661                                         ((const struct rte_flow_item_ipv6 *)
5662                                          items->mask)->hdr.proto;
5663                         } else {
5664                                 /* Reset for inner layer. */
5665                                 next_protocol = 0xff;
5666                         }
5667                         break;
5668                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
5669                         ret = flow_dv_validate_item_ipv6_frag_ext(items,
5670                                                                   item_flags,
5671                                                                   error);
5672                         if (ret < 0)
5673                                 return ret;
5674                         last_item = tunnel ?
5675                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
5676                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
5677                         if (items->mask != NULL &&
5678                             ((const struct rte_flow_item_ipv6_frag_ext *)
5679                              items->mask)->hdr.next_header) {
5680                                 next_protocol =
5681                                 ((const struct rte_flow_item_ipv6_frag_ext *)
5682                                  items->spec)->hdr.next_header;
5683                                 next_protocol &=
5684                                 ((const struct rte_flow_item_ipv6_frag_ext *)
5685                                  items->mask)->hdr.next_header;
5686                         } else {
5687                                 /* Reset for inner layer. */
5688                                 next_protocol = 0xff;
5689                         }
5690                         break;
5691                 case RTE_FLOW_ITEM_TYPE_TCP:
5692                         ret = mlx5_flow_validate_item_tcp
5693                                                 (items, item_flags,
5694                                                  next_protocol,
5695                                                  &nic_tcp_mask,
5696                                                  error);
5697                         if (ret < 0)
5698                                 return ret;
5699                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5700                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5701                         break;
5702                 case RTE_FLOW_ITEM_TYPE_UDP:
5703                         ret = mlx5_flow_validate_item_udp(items, item_flags,
5704                                                           next_protocol,
5705                                                           error);
5706                         if (ret < 0)
5707                                 return ret;
5708                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5709                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5710                         break;
5711                 case RTE_FLOW_ITEM_TYPE_GRE:
5712                         ret = mlx5_flow_validate_item_gre(items, item_flags,
5713                                                           next_protocol, error);
5714                         if (ret < 0)
5715                                 return ret;
5716                         gre_item = items;
5717                         last_item = MLX5_FLOW_LAYER_GRE;
5718                         break;
5719                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5720                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
5721                                                             next_protocol,
5722                                                             error);
5723                         if (ret < 0)
5724                                 return ret;
5725                         last_item = MLX5_FLOW_LAYER_NVGRE;
5726                         break;
5727                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5728                         ret = mlx5_flow_validate_item_gre_key
5729                                 (items, item_flags, gre_item, error);
5730                         if (ret < 0)
5731                                 return ret;
5732                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5733                         break;
5734                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5735                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
5736                                                             error);
5737                         if (ret < 0)
5738                                 return ret;
5739                         last_item = MLX5_FLOW_LAYER_VXLAN;
5740                         break;
5741                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5742                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
5743                                                                 item_flags, dev,
5744                                                                 error);
5745                         if (ret < 0)
5746                                 return ret;
5747                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5748                         break;
5749                 case RTE_FLOW_ITEM_TYPE_GENEVE:
5750                         ret = mlx5_flow_validate_item_geneve(items,
5751                                                              item_flags, dev,
5752                                                              error);
5753                         if (ret < 0)
5754                                 return ret;
5755                         last_item = MLX5_FLOW_LAYER_GENEVE;
5756                         break;
5757                 case RTE_FLOW_ITEM_TYPE_MPLS:
5758                         ret = mlx5_flow_validate_item_mpls(dev, items,
5759                                                            item_flags,
5760                                                            last_item, error);
5761                         if (ret < 0)
5762                                 return ret;
5763                         last_item = MLX5_FLOW_LAYER_MPLS;
5764                         break;
5765
5766                 case RTE_FLOW_ITEM_TYPE_MARK:
5767                         ret = flow_dv_validate_item_mark(dev, items, attr,
5768                                                          error);
5769                         if (ret < 0)
5770                                 return ret;
5771                         last_item = MLX5_FLOW_ITEM_MARK;
5772                         break;
5773                 case RTE_FLOW_ITEM_TYPE_META:
5774                         ret = flow_dv_validate_item_meta(dev, items, attr,
5775                                                          error);
5776                         if (ret < 0)
5777                                 return ret;
5778                         last_item = MLX5_FLOW_ITEM_METADATA;
5779                         break;
5780                 case RTE_FLOW_ITEM_TYPE_ICMP:
5781                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
5782                                                            next_protocol,
5783                                                            error);
5784                         if (ret < 0)
5785                                 return ret;
5786                         last_item = MLX5_FLOW_LAYER_ICMP;
5787                         break;
5788                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5789                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
5790                                                             next_protocol,
5791                                                             error);
5792                         if (ret < 0)
5793                                 return ret;
5794                         item_ipv6_proto = IPPROTO_ICMPV6;
5795                         last_item = MLX5_FLOW_LAYER_ICMP6;
5796                         break;
5797                 case RTE_FLOW_ITEM_TYPE_TAG:
5798                         ret = flow_dv_validate_item_tag(dev, items,
5799                                                         attr, error);
5800                         if (ret < 0)
5801                                 return ret;
5802                         last_item = MLX5_FLOW_ITEM_TAG;
5803                         break;
5804                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
5805                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
5806                         break;
5807                 case RTE_FLOW_ITEM_TYPE_GTP:
5808                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
5809                                                         error);
5810                         if (ret < 0)
5811                                 return ret;
5812                         last_item = MLX5_FLOW_LAYER_GTP;
5813                         break;
5814                 case RTE_FLOW_ITEM_TYPE_ECPRI:
5815                         /* Capacity will be checked in the translate stage. */
5816                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
5817                                                             last_item,
5818                                                             ether_type,
5819                                                             &nic_ecpri_mask,
5820                                                             error);
5821                         if (ret < 0)
5822                                 return ret;
5823                         last_item = MLX5_FLOW_LAYER_ECPRI;
5824                         break;
5825                 default:
5826                         return rte_flow_error_set(error, ENOTSUP,
5827                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5828                                                   NULL, "item not supported");
5829                 }
5830                 item_flags |= last_item;
5831         }
5832         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5833                 int type = actions->type;
5834
5835                 if (!mlx5_flow_os_action_supported(type))
5836                         return rte_flow_error_set(error, ENOTSUP,
5837                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5838                                                   actions,
5839                                                   "action not supported");
5840                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5841                         return rte_flow_error_set(error, ENOTSUP,
5842                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5843                                                   actions, "too many actions");
5844                 switch (type) {
5845                 case RTE_FLOW_ACTION_TYPE_VOID:
5846                         break;
5847                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5848                         ret = flow_dv_validate_action_port_id(dev,
5849                                                               action_flags,
5850                                                               actions,
5851                                                               attr,
5852                                                               error);
5853                         if (ret)
5854                                 return ret;
5855                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5856                         ++actions_n;
5857                         break;
5858                 case RTE_FLOW_ACTION_TYPE_FLAG:
5859                         ret = flow_dv_validate_action_flag(dev, action_flags,
5860                                                            attr, error);
5861                         if (ret < 0)
5862                                 return ret;
5863                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5864                                 /* Count all modify-header actions as one. */
5865                                 if (!(action_flags &
5866                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5867                                         ++actions_n;
5868                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
5869                                                 MLX5_FLOW_ACTION_MARK_EXT;
5870                         } else {
5871                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
5872                                 ++actions_n;
5873                         }
5874                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5875                         break;
5876                 case RTE_FLOW_ACTION_TYPE_MARK:
5877                         ret = flow_dv_validate_action_mark(dev, actions,
5878                                                            action_flags,
5879                                                            attr, error);
5880                         if (ret < 0)
5881                                 return ret;
5882                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5883                                 /* Count all modify-header actions as one. */
5884                                 if (!(action_flags &
5885                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5886                                         ++actions_n;
5887                                 action_flags |= MLX5_FLOW_ACTION_MARK |
5888                                                 MLX5_FLOW_ACTION_MARK_EXT;
5889                         } else {
5890                                 action_flags |= MLX5_FLOW_ACTION_MARK;
5891                                 ++actions_n;
5892                         }
5893                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5894                         break;
5895                 case RTE_FLOW_ACTION_TYPE_SET_META:
5896                         ret = flow_dv_validate_action_set_meta(dev, actions,
5897                                                                action_flags,
5898                                                                attr, error);
5899                         if (ret < 0)
5900                                 return ret;
5901                         /* Count all modify-header actions as one action. */
5902                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5903                                 ++actions_n;
5904                         action_flags |= MLX5_FLOW_ACTION_SET_META;
5905                         rw_act_num += MLX5_ACT_NUM_SET_META;
5906                         break;
5907                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5908                         ret = flow_dv_validate_action_set_tag(dev, actions,
5909                                                               action_flags,
5910                                                               attr, error);
5911                         if (ret < 0)
5912                                 return ret;
5913                         /* Count all modify-header actions as one action. */
5914                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5915                                 ++actions_n;
5916                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
5917                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5918                         break;
5919                 case RTE_FLOW_ACTION_TYPE_DROP:
5920                         ret = mlx5_flow_validate_action_drop(action_flags,
5921                                                              attr, error);
5922                         if (ret < 0)
5923                                 return ret;
5924                         action_flags |= MLX5_FLOW_ACTION_DROP;
5925                         ++actions_n;
5926                         break;
5927                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5928                         ret = mlx5_flow_validate_action_queue(actions,
5929                                                               action_flags, dev,
5930                                                               attr, error);
5931                         if (ret < 0)
5932                                 return ret;
5933                         queue_index = ((const struct rte_flow_action_queue *)
5934                                                         (actions->conf))->index;
5935                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
5936                         ++actions_n;
5937                         break;
5938                 case RTE_FLOW_ACTION_TYPE_RSS:
5939                         rss = actions->conf;
5940                         ret = mlx5_flow_validate_action_rss(actions,
5941                                                             action_flags, dev,
5942                                                             attr, item_flags,
5943                                                             error);
5944                         if (ret < 0)
5945                                 return ret;
5946                         if (rss != NULL && rss->queue_num)
5947                                 queue_index = rss->queue[0];
5948                         action_flags |= MLX5_FLOW_ACTION_RSS;
5949                         ++actions_n;
5950                         break;
5951                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
5952                         ret =
5953                         mlx5_flow_validate_action_default_miss(action_flags,
5954                                         attr, error);
5955                         if (ret < 0)
5956                                 return ret;
5957                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
5958                         ++actions_n;
5959                         break;
5960                 case RTE_FLOW_ACTION_TYPE_COUNT:
5961                         ret = flow_dv_validate_action_count(dev, error);
5962                         if (ret < 0)
5963                                 return ret;
5964                         action_flags |= MLX5_FLOW_ACTION_COUNT;
5965                         ++actions_n;
5966                         break;
5967                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5968                         if (flow_dv_validate_action_pop_vlan(dev,
5969                                                              action_flags,
5970                                                              actions,
5971                                                              item_flags, attr,
5972                                                              error))
5973                                 return -rte_errno;
5974                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5975                         ++actions_n;
5976                         break;
5977                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5978                         ret = flow_dv_validate_action_push_vlan(dev,
5979                                                                 action_flags,
5980                                                                 vlan_m,
5981                                                                 actions, attr,
5982                                                                 error);
5983                         if (ret < 0)
5984                                 return ret;
5985                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5986                         ++actions_n;
5987                         break;
5988                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5989                         ret = flow_dv_validate_action_set_vlan_pcp
5990                                                 (action_flags, actions, error);
5991                         if (ret < 0)
5992                                 return ret;
5993                         /* Count PCP with push_vlan command. */
5994                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
5995                         break;
5996                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5997                         ret = flow_dv_validate_action_set_vlan_vid
5998                                                 (item_flags, action_flags,
5999                                                  actions, error);
6000                         if (ret < 0)
6001                                 return ret;
6002                         /* Count VID with push_vlan command. */
6003                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
6004                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
6005                         break;
6006                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
6007                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
6008                         ret = flow_dv_validate_action_l2_encap(dev,
6009                                                                action_flags,
6010                                                                actions, attr,
6011                                                                error);
6012                         if (ret < 0)
6013                                 return ret;
6014                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
6015                         ++actions_n;
6016                         break;
6017                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
6018                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
6019                         ret = flow_dv_validate_action_decap(dev, action_flags,
6020                                                             attr, error);
6021                         if (ret < 0)
6022                                 return ret;
6023                         action_flags |= MLX5_FLOW_ACTION_DECAP;
6024                         ++actions_n;
6025                         break;
6026                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
6027                         ret = flow_dv_validate_action_raw_encap_decap
6028                                 (dev, NULL, actions->conf, attr, &action_flags,
6029                                  &actions_n, error);
6030                         if (ret < 0)
6031                                 return ret;
6032                         break;
6033                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
6034                         decap = actions->conf;
6035                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
6036                                 ;
6037                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
6038                                 encap = NULL;
6039                                 actions--;
6040                         } else {
6041                                 encap = actions->conf;
6042                         }
6043                         ret = flow_dv_validate_action_raw_encap_decap
6044                                            (dev,
6045                                             decap ? decap : &empty_decap, encap,
6046                                             attr, &action_flags, &actions_n,
6047                                             error);
6048                         if (ret < 0)
6049                                 return ret;
6050                         break;
6051                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
6052                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
6053                         ret = flow_dv_validate_action_modify_mac(action_flags,
6054                                                                  actions,
6055                                                                  item_flags,
6056                                                                  error);
6057                         if (ret < 0)
6058                                 return ret;
6059                         /* Count all modify-header actions as one action. */
6060                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6061                                 ++actions_n;
6062                         action_flags |= actions->type ==
6063                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
6064                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
6065                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
6066                         /*
6067                          * Even if the source and destination MAC addresses have
6068                          * overlap in the header with 4B alignment, the convert
6069                          * function will handle them separately and 4 SW actions
6070                          * will be created. And 2 actions will be added each
6071                          * time no matter how many bytes of address will be set.
6072                          */
6073                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
6074                         break;
6075                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
6076                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
6077                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
6078                                                                   actions,
6079                                                                   item_flags,
6080                                                                   error);
6081                         if (ret < 0)
6082                                 return ret;
6083                         /* Count all modify-header actions as one action. */
6084                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6085                                 ++actions_n;
6086                         action_flags |= actions->type ==
6087                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
6088                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
6089                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
6090                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
6091                         break;
6092                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
6093                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
6094                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
6095                                                                   actions,
6096                                                                   item_flags,
6097                                                                   error);
6098                         if (ret < 0)
6099                                 return ret;
6100                         if (item_ipv6_proto == IPPROTO_ICMPV6)
6101                                 return rte_flow_error_set(error, ENOTSUP,
6102                                         RTE_FLOW_ERROR_TYPE_ACTION,
6103                                         actions,
6104                                         "Can't change header "
6105                                         "with ICMPv6 proto");
6106                         /* Count all modify-header actions as one action. */
6107                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6108                                 ++actions_n;
6109                         action_flags |= actions->type ==
6110                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
6111                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
6112                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
6113                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
6114                         break;
6115                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
6116                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
6117                         ret = flow_dv_validate_action_modify_tp(action_flags,
6118                                                                 actions,
6119                                                                 item_flags,
6120                                                                 error);
6121                         if (ret < 0)
6122                                 return ret;
6123                         /* Count all modify-header actions as one action. */
6124                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6125                                 ++actions_n;
6126                         action_flags |= actions->type ==
6127                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
6128                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
6129                                                 MLX5_FLOW_ACTION_SET_TP_DST;
6130                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
6131                         break;
6132                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
6133                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
6134                         ret = flow_dv_validate_action_modify_ttl(action_flags,
6135                                                                  actions,
6136                                                                  item_flags,
6137                                                                  error);
6138                         if (ret < 0)
6139                                 return ret;
6140                         /* Count all modify-header actions as one action. */
6141                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6142                                 ++actions_n;
6143                         action_flags |= actions->type ==
6144                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
6145                                                 MLX5_FLOW_ACTION_SET_TTL :
6146                                                 MLX5_FLOW_ACTION_DEC_TTL;
6147                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
6148                         break;
6149                 case RTE_FLOW_ACTION_TYPE_JUMP:
6150                         ret = flow_dv_validate_action_jump(actions,
6151                                                            action_flags,
6152                                                            attr, external,
6153                                                            error);
6154                         if (ret)
6155                                 return ret;
6156                         ++actions_n;
6157                         action_flags |= MLX5_FLOW_ACTION_JUMP;
6158                         break;
6159                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
6160                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
6161                         ret = flow_dv_validate_action_modify_tcp_seq
6162                                                                 (action_flags,
6163                                                                  actions,
6164                                                                  item_flags,
6165                                                                  error);
6166                         if (ret < 0)
6167                                 return ret;
6168                         /* Count all modify-header actions as one action. */
6169                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6170                                 ++actions_n;
6171                         action_flags |= actions->type ==
6172                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
6173                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
6174                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
6175                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
6176                         break;
6177                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
6178                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
6179                         ret = flow_dv_validate_action_modify_tcp_ack
6180                                                                 (action_flags,
6181                                                                  actions,
6182                                                                  item_flags,
6183                                                                  error);
6184                         if (ret < 0)
6185                                 return ret;
6186                         /* Count all modify-header actions as one action. */
6187                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6188                                 ++actions_n;
6189                         action_flags |= actions->type ==
6190                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
6191                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
6192                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
6193                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
6194                         break;
6195                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
6196                         break;
6197                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
6198                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
6199                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
6200                         break;
6201                 case RTE_FLOW_ACTION_TYPE_METER:
6202                         ret = mlx5_flow_validate_action_meter(dev,
6203                                                               action_flags,
6204                                                               actions, attr,
6205                                                               error);
6206                         if (ret < 0)
6207                                 return ret;
6208                         action_flags |= MLX5_FLOW_ACTION_METER;
6209                         ++actions_n;
6210                         /* Meter action will add one more TAG action. */
6211                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
6212                         break;
6213                 case RTE_FLOW_ACTION_TYPE_AGE:
6214                         ret = flow_dv_validate_action_age(action_flags,
6215                                                           actions, dev,
6216                                                           error);
6217                         if (ret < 0)
6218                                 return ret;
6219                         action_flags |= MLX5_FLOW_ACTION_AGE;
6220                         ++actions_n;
6221                         break;
6222                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
6223                         ret = flow_dv_validate_action_modify_ipv4_dscp
6224                                                          (action_flags,
6225                                                           actions,
6226                                                           item_flags,
6227                                                           error);
6228                         if (ret < 0)
6229                                 return ret;
6230                         /* Count all modify-header actions as one action. */
6231                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6232                                 ++actions_n;
6233                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
6234                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
6235                         break;
6236                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
6237                         ret = flow_dv_validate_action_modify_ipv6_dscp
6238                                                                 (action_flags,
6239                                                                  actions,
6240                                                                  item_flags,
6241                                                                  error);
6242                         if (ret < 0)
6243                                 return ret;
6244                         /* Count all modify-header actions as one action. */
6245                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6246                                 ++actions_n;
6247                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
6248                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
6249                         break;
6250                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
6251                         ret = flow_dv_validate_action_sample(action_flags,
6252                                                              actions, dev,
6253                                                              attr, error);
6254                         if (ret < 0)
6255                                 return ret;
6256                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
6257                         ++actions_n;
6258                         break;
6259                 default:
6260                         return rte_flow_error_set(error, ENOTSUP,
6261                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6262                                                   actions,
6263                                                   "action not supported");
6264                 }
6265         }
6266         /*
6267          * Validate the drop action mutual exclusion with other actions.
6268          * Drop action is mutually-exclusive with any other action, except for
6269          * Count action.
6270          */
6271         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
6272             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
6273                 return rte_flow_error_set(error, EINVAL,
6274                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
6275                                           "Drop action is mutually-exclusive "
6276                                           "with any other action, except for "
6277                                           "Count action");
6278         /* Eswitch has few restrictions on using items and actions */
6279         if (attr->transfer) {
6280                 if (!mlx5_flow_ext_mreg_supported(dev) &&
6281                     action_flags & MLX5_FLOW_ACTION_FLAG)
6282                         return rte_flow_error_set(error, ENOTSUP,
6283                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6284                                                   NULL,
6285                                                   "unsupported action FLAG");
6286                 if (!mlx5_flow_ext_mreg_supported(dev) &&
6287                     action_flags & MLX5_FLOW_ACTION_MARK)
6288                         return rte_flow_error_set(error, ENOTSUP,
6289                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6290                                                   NULL,
6291                                                   "unsupported action MARK");
6292                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
6293                         return rte_flow_error_set(error, ENOTSUP,
6294                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6295                                                   NULL,
6296                                                   "unsupported action QUEUE");
6297                 if (action_flags & MLX5_FLOW_ACTION_RSS)
6298                         return rte_flow_error_set(error, ENOTSUP,
6299                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6300                                                   NULL,
6301                                                   "unsupported action RSS");
6302                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
6303                         return rte_flow_error_set(error, EINVAL,
6304                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6305                                                   actions,
6306                                                   "no fate action is found");
6307         } else {
6308                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
6309                         return rte_flow_error_set(error, EINVAL,
6310                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6311                                                   actions,
6312                                                   "no fate action is found");
6313         }
6314         /* Continue validation for Xcap and VLAN actions.*/
6315         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
6316                              MLX5_FLOW_VLAN_ACTIONS)) &&
6317             (queue_index == 0xFFFF ||
6318              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
6319                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
6320                     MLX5_FLOW_XCAP_ACTIONS)
6321                         return rte_flow_error_set(error, ENOTSUP,
6322                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6323                                                   NULL, "encap and decap "
6324                                                   "combination aren't supported");
6325                 if (!attr->transfer && attr->ingress) {
6326                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
6327                                 return rte_flow_error_set
6328                                                 (error, ENOTSUP,
6329                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6330                                                  NULL, "encap is not supported"
6331                                                  " for ingress traffic");
6332                         else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
6333                                 return rte_flow_error_set
6334                                                 (error, ENOTSUP,
6335                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6336                                                  NULL, "push VLAN action not "
6337                                                  "supported for ingress");
6338                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
6339                                         MLX5_FLOW_VLAN_ACTIONS)
6340                                 return rte_flow_error_set
6341                                                 (error, ENOTSUP,
6342                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6343                                                  NULL, "no support for "
6344                                                  "multiple VLAN actions");
6345                 }
6346         }
6347         /* Hairpin flow will add one more TAG action. */
6348         if (hairpin > 0)
6349                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6350         /* extra metadata enabled: one more TAG action will be add. */
6351         if (dev_conf->dv_flow_en &&
6352             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
6353             mlx5_flow_ext_mreg_supported(dev))
6354                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6355         if ((uint32_t)rw_act_num >
6356                         flow_dv_modify_hdr_action_max(dev, is_root)) {
6357                 return rte_flow_error_set(error, ENOTSUP,
6358                                           RTE_FLOW_ERROR_TYPE_ACTION,
6359                                           NULL, "too many header modify"
6360                                           " actions to support");
6361         }
6362         return 0;
6363 }
6364
6365 /**
6366  * Internal preparation function. Allocates the DV flow size,
6367  * this size is constant.
6368  *
6369  * @param[in] dev
6370  *   Pointer to the rte_eth_dev structure.
6371  * @param[in] attr
6372  *   Pointer to the flow attributes.
6373  * @param[in] items
6374  *   Pointer to the list of items.
6375  * @param[in] actions
6376  *   Pointer to the list of actions.
6377  * @param[out] error
6378  *   Pointer to the error structure.
6379  *
6380  * @return
6381  *   Pointer to mlx5_flow object on success,
6382  *   otherwise NULL and rte_errno is set.
6383  */
6384 static struct mlx5_flow *
6385 flow_dv_prepare(struct rte_eth_dev *dev,
6386                 const struct rte_flow_attr *attr __rte_unused,
6387                 const struct rte_flow_item items[] __rte_unused,
6388                 const struct rte_flow_action actions[] __rte_unused,
6389                 struct rte_flow_error *error)
6390 {
6391         uint32_t handle_idx = 0;
6392         struct mlx5_flow *dev_flow;
6393         struct mlx5_flow_handle *dev_handle;
6394         struct mlx5_priv *priv = dev->data->dev_private;
6395
6396         /* In case of corrupting the memory. */
6397         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
6398                 rte_flow_error_set(error, ENOSPC,
6399                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6400                                    "not free temporary device flow");
6401                 return NULL;
6402         }
6403         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
6404                                    &handle_idx);
6405         if (!dev_handle) {
6406                 rte_flow_error_set(error, ENOMEM,
6407                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6408                                    "not enough memory to create flow handle");
6409                 return NULL;
6410         }
6411         /* No multi-thread supporting. */
6412         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
6413         dev_flow->handle = dev_handle;
6414         dev_flow->handle_idx = handle_idx;
6415         /*
6416          * In some old rdma-core releases, before continuing, a check of the
6417          * length of matching parameter will be done at first. It needs to use
6418          * the length without misc4 param. If the flow has misc4 support, then
6419          * the length needs to be adjusted accordingly. Each param member is
6420          * aligned with a 64B boundary naturally.
6421          */
6422         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) -
6423                                   MLX5_ST_SZ_BYTES(fte_match_set_misc4);
6424         /*
6425          * The matching value needs to be cleared to 0 before using. In the
6426          * past, it will be automatically cleared when using rte_*alloc
6427          * API. The time consumption will be almost the same as before.
6428          */
6429         memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
6430         dev_flow->ingress = attr->ingress;
6431         dev_flow->dv.transfer = attr->transfer;
6432         return dev_flow;
6433 }
6434
6435 #ifdef RTE_LIBRTE_MLX5_DEBUG
6436 /**
6437  * Sanity check for match mask and value. Similar to check_valid_spec() in
6438  * kernel driver. If unmasked bit is present in value, it returns failure.
6439  *
6440  * @param match_mask
6441  *   pointer to match mask buffer.
6442  * @param match_value
6443  *   pointer to match value buffer.
6444  *
6445  * @return
6446  *   0 if valid, -EINVAL otherwise.
6447  */
6448 static int
6449 flow_dv_check_valid_spec(void *match_mask, void *match_value)
6450 {
6451         uint8_t *m = match_mask;
6452         uint8_t *v = match_value;
6453         unsigned int i;
6454
6455         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
6456                 if (v[i] & ~m[i]) {
6457                         DRV_LOG(ERR,
6458                                 "match_value differs from match_criteria"
6459                                 " %p[%u] != %p[%u]",
6460                                 match_value, i, match_mask, i);
6461                         return -EINVAL;
6462                 }
6463         }
6464         return 0;
6465 }
6466 #endif
6467
6468 /**
6469  * Add match of ip_version.
6470  *
6471  * @param[in] group
6472  *   Flow group.
6473  * @param[in] headers_v
6474  *   Values header pointer.
6475  * @param[in] headers_m
6476  *   Masks header pointer.
6477  * @param[in] ip_version
6478  *   The IP version to set.
6479  */
6480 static inline void
6481 flow_dv_set_match_ip_version(uint32_t group,
6482                              void *headers_v,
6483                              void *headers_m,
6484                              uint8_t ip_version)
6485 {
6486         if (group == 0)
6487                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
6488         else
6489                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
6490                          ip_version);
6491         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
6492         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
6493         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
6494 }
6495
6496 /**
6497  * Add Ethernet item to matcher and to the value.
6498  *
6499  * @param[in, out] matcher
6500  *   Flow matcher.
6501  * @param[in, out] key
6502  *   Flow matcher value.
6503  * @param[in] item
6504  *   Flow pattern to translate.
6505  * @param[in] inner
6506  *   Item is inner pattern.
6507  */
6508 static void
6509 flow_dv_translate_item_eth(void *matcher, void *key,
6510                            const struct rte_flow_item *item, int inner,
6511                            uint32_t group)
6512 {
6513         const struct rte_flow_item_eth *eth_m = item->mask;
6514         const struct rte_flow_item_eth *eth_v = item->spec;
6515         const struct rte_flow_item_eth nic_mask = {
6516                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6517                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6518                 .type = RTE_BE16(0xffff),
6519         };
6520         void *headers_m;
6521         void *headers_v;
6522         char *l24_v;
6523         unsigned int i;
6524
6525         if (!eth_v)
6526                 return;
6527         if (!eth_m)
6528                 eth_m = &nic_mask;
6529         if (inner) {
6530                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6531                                          inner_headers);
6532                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6533         } else {
6534                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6535                                          outer_headers);
6536                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6537         }
6538         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
6539                &eth_m->dst, sizeof(eth_m->dst));
6540         /* The value must be in the range of the mask. */
6541         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
6542         for (i = 0; i < sizeof(eth_m->dst); ++i)
6543                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
6544         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
6545                &eth_m->src, sizeof(eth_m->src));
6546         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
6547         /* The value must be in the range of the mask. */
6548         for (i = 0; i < sizeof(eth_m->dst); ++i)
6549                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
6550         if (eth_v->type) {
6551                 /* When ethertype is present set mask for tagged VLAN. */
6552                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6553                 /* Set value for tagged VLAN if ethertype is 802.1Q. */
6554                 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
6555                     eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
6556                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag,
6557                                  1);
6558                         /* Return here to avoid setting match on ethertype. */
6559                         return;
6560                 }
6561         }
6562         /*
6563          * HW supports match on one Ethertype, the Ethertype following the last
6564          * VLAN tag of the packet (see PRM).
6565          * Set match on ethertype only if ETH header is not followed by VLAN.
6566          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6567          * ethertype, and use ip_version field instead.
6568          * eCPRI over Ether layer will use type value 0xAEFE.
6569          */
6570         if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6571             eth_m->type == 0xFFFF) {
6572                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6573         } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6574                    eth_m->type == 0xFFFF) {
6575                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6576         } else {
6577                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6578                          rte_be_to_cpu_16(eth_m->type));
6579                 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6580                                      ethertype);
6581                 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
6582         }
6583 }
6584
6585 /**
6586  * Add VLAN item to matcher and to the value.
6587  *
6588  * @param[in, out] dev_flow
6589  *   Flow descriptor.
6590  * @param[in, out] matcher
6591  *   Flow matcher.
6592  * @param[in, out] key
6593  *   Flow matcher value.
6594  * @param[in] item
6595  *   Flow pattern to translate.
6596  * @param[in] inner
6597  *   Item is inner pattern.
6598  */
6599 static void
6600 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
6601                             void *matcher, void *key,
6602                             const struct rte_flow_item *item,
6603                             int inner, uint32_t group)
6604 {
6605         const struct rte_flow_item_vlan *vlan_m = item->mask;
6606         const struct rte_flow_item_vlan *vlan_v = item->spec;
6607         void *headers_m;
6608         void *headers_v;
6609         uint16_t tci_m;
6610         uint16_t tci_v;
6611
6612         if (inner) {
6613                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6614                                          inner_headers);
6615                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6616         } else {
6617                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6618                                          outer_headers);
6619                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6620                 /*
6621                  * This is workaround, masks are not supported,
6622                  * and pre-validated.
6623                  */
6624                 if (vlan_v)
6625                         dev_flow->handle->vf_vlan.tag =
6626                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
6627         }
6628         /*
6629          * When VLAN item exists in flow, mark packet as tagged,
6630          * even if TCI is not specified.
6631          */
6632         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6633         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
6634         if (!vlan_v)
6635                 return;
6636         if (!vlan_m)
6637                 vlan_m = &rte_flow_item_vlan_mask;
6638         tci_m = rte_be_to_cpu_16(vlan_m->tci);
6639         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
6640         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
6641         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
6642         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
6643         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
6644         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
6645         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
6646         /*
6647          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6648          * ethertype, and use ip_version field instead.
6649          */
6650         if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6651             vlan_m->inner_type == 0xFFFF) {
6652                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6653         } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6654                    vlan_m->inner_type == 0xFFFF) {
6655                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6656         } else {
6657                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6658                          rte_be_to_cpu_16(vlan_m->inner_type));
6659                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
6660                          rte_be_to_cpu_16(vlan_m->inner_type &
6661                                           vlan_v->inner_type));
6662         }
6663 }
6664
6665 /**
6666  * Add IPV4 item to matcher and to the value.
6667  *
6668  * @param[in, out] matcher
6669  *   Flow matcher.
6670  * @param[in, out] key
6671  *   Flow matcher value.
6672  * @param[in] item
6673  *   Flow pattern to translate.
6674  * @param[in] item_flags
6675  *   Bit-fields that holds the items detected until now.
6676  * @param[in] inner
6677  *   Item is inner pattern.
6678  * @param[in] group
6679  *   The group to insert the rule.
6680  */
6681 static void
6682 flow_dv_translate_item_ipv4(void *matcher, void *key,
6683                             const struct rte_flow_item *item,
6684                             const uint64_t item_flags,
6685                             int inner, uint32_t group)
6686 {
6687         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
6688         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
6689         const struct rte_flow_item_ipv4 nic_mask = {
6690                 .hdr = {
6691                         .src_addr = RTE_BE32(0xffffffff),
6692                         .dst_addr = RTE_BE32(0xffffffff),
6693                         .type_of_service = 0xff,
6694                         .next_proto_id = 0xff,
6695                         .time_to_live = 0xff,
6696                 },
6697         };
6698         void *headers_m;
6699         void *headers_v;
6700         char *l24_m;
6701         char *l24_v;
6702         uint8_t tos;
6703
6704         if (inner) {
6705                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6706                                          inner_headers);
6707                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6708         } else {
6709                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6710                                          outer_headers);
6711                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6712         }
6713         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6714         /*
6715          * On outer header (which must contains L2), or inner header with L2,
6716          * set cvlan_tag mask bit to mark this packet as untagged.
6717          * This should be done even if item->spec is empty.
6718          */
6719         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6720                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6721         if (!ipv4_v)
6722                 return;
6723         if (!ipv4_m)
6724                 ipv4_m = &nic_mask;
6725         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6726                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6727         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6728                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6729         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
6730         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
6731         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6732                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6733         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6734                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6735         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
6736         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
6737         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
6738         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
6739                  ipv4_m->hdr.type_of_service);
6740         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
6741         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
6742                  ipv4_m->hdr.type_of_service >> 2);
6743         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
6744         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6745                  ipv4_m->hdr.next_proto_id);
6746         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6747                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
6748         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6749                  ipv4_m->hdr.time_to_live);
6750         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6751                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
6752         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
6753                  !!(ipv4_m->hdr.fragment_offset));
6754         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
6755                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
6756 }
6757
6758 /**
6759  * Add IPV6 item to matcher and to the value.
6760  *
6761  * @param[in, out] matcher
6762  *   Flow matcher.
6763  * @param[in, out] key
6764  *   Flow matcher value.
6765  * @param[in] item
6766  *   Flow pattern to translate.
6767  * @param[in] item_flags
6768  *   Bit-fields that holds the items detected until now.
6769  * @param[in] inner
6770  *   Item is inner pattern.
6771  * @param[in] group
6772  *   The group to insert the rule.
6773  */
6774 static void
6775 flow_dv_translate_item_ipv6(void *matcher, void *key,
6776                             const struct rte_flow_item *item,
6777                             const uint64_t item_flags,
6778                             int inner, uint32_t group)
6779 {
6780         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
6781         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
6782         const struct rte_flow_item_ipv6 nic_mask = {
6783                 .hdr = {
6784                         .src_addr =
6785                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6786                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6787                         .dst_addr =
6788                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6789                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6790                         .vtc_flow = RTE_BE32(0xffffffff),
6791                         .proto = 0xff,
6792                         .hop_limits = 0xff,
6793                 },
6794         };
6795         void *headers_m;
6796         void *headers_v;
6797         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6798         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6799         char *l24_m;
6800         char *l24_v;
6801         uint32_t vtc_m;
6802         uint32_t vtc_v;
6803         int i;
6804         int size;
6805
6806         if (inner) {
6807                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6808                                          inner_headers);
6809                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6810         } else {
6811                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6812                                          outer_headers);
6813                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6814         }
6815         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6816         /*
6817          * On outer header (which must contains L2), or inner header with L2,
6818          * set cvlan_tag mask bit to mark this packet as untagged.
6819          * This should be done even if item->spec is empty.
6820          */
6821         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6822                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6823         if (!ipv6_v)
6824                 return;
6825         if (!ipv6_m)
6826                 ipv6_m = &nic_mask;
6827         size = sizeof(ipv6_m->hdr.dst_addr);
6828         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6829                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6830         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6831                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6832         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
6833         for (i = 0; i < size; ++i)
6834                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
6835         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6836                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6837         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6838                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6839         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
6840         for (i = 0; i < size; ++i)
6841                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
6842         /* TOS. */
6843         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
6844         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
6845         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
6846         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
6847         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
6848         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
6849         /* Label. */
6850         if (inner) {
6851                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
6852                          vtc_m);
6853                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
6854                          vtc_v);
6855         } else {
6856                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
6857                          vtc_m);
6858                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
6859                          vtc_v);
6860         }
6861         /* Protocol. */
6862         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6863                  ipv6_m->hdr.proto);
6864         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6865                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
6866         /* Hop limit. */
6867         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6868                  ipv6_m->hdr.hop_limits);
6869         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6870                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
6871         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
6872                  !!(ipv6_m->has_frag_ext));
6873         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
6874                  !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext));
6875 }
6876
6877 /**
6878  * Add IPV6 fragment extension item to matcher and to the value.
6879  *
6880  * @param[in, out] matcher
6881  *   Flow matcher.
6882  * @param[in, out] key
6883  *   Flow matcher value.
6884  * @param[in] item
6885  *   Flow pattern to translate.
6886  * @param[in] inner
6887  *   Item is inner pattern.
6888  */
6889 static void
6890 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key,
6891                                      const struct rte_flow_item *item,
6892                                      int inner)
6893 {
6894         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask;
6895         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec;
6896         const struct rte_flow_item_ipv6_frag_ext nic_mask = {
6897                 .hdr = {
6898                         .next_header = 0xff,
6899                         .frag_data = RTE_BE16(0xffff),
6900                 },
6901         };
6902         void *headers_m;
6903         void *headers_v;
6904
6905         if (inner) {
6906                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6907                                          inner_headers);
6908                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6909         } else {
6910                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6911                                          outer_headers);
6912                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6913         }
6914         /* IPv6 fragment extension item exists, so packet is IP fragment. */
6915         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
6916         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1);
6917         if (!ipv6_frag_ext_v)
6918                 return;
6919         if (!ipv6_frag_ext_m)
6920                 ipv6_frag_ext_m = &nic_mask;
6921         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6922                  ipv6_frag_ext_m->hdr.next_header);
6923         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6924                  ipv6_frag_ext_v->hdr.next_header &
6925                  ipv6_frag_ext_m->hdr.next_header);
6926 }
6927
6928 /**
6929  * Add TCP item to matcher and to the value.
6930  *
6931  * @param[in, out] matcher
6932  *   Flow matcher.
6933  * @param[in, out] key
6934  *   Flow matcher value.
6935  * @param[in] item
6936  *   Flow pattern to translate.
6937  * @param[in] inner
6938  *   Item is inner pattern.
6939  */
6940 static void
6941 flow_dv_translate_item_tcp(void *matcher, void *key,
6942                            const struct rte_flow_item *item,
6943                            int inner)
6944 {
6945         const struct rte_flow_item_tcp *tcp_m = item->mask;
6946         const struct rte_flow_item_tcp *tcp_v = item->spec;
6947         void *headers_m;
6948         void *headers_v;
6949
6950         if (inner) {
6951                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6952                                          inner_headers);
6953                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6954         } else {
6955                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6956                                          outer_headers);
6957                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6958         }
6959         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6960         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
6961         if (!tcp_v)
6962                 return;
6963         if (!tcp_m)
6964                 tcp_m = &rte_flow_item_tcp_mask;
6965         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
6966                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
6967         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
6968                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
6969         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
6970                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
6971         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
6972                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
6973         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
6974                  tcp_m->hdr.tcp_flags);
6975         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
6976                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
6977 }
6978
6979 /**
6980  * Add UDP item to matcher and to the value.
6981  *
6982  * @param[in, out] matcher
6983  *   Flow matcher.
6984  * @param[in, out] key
6985  *   Flow matcher value.
6986  * @param[in] item
6987  *   Flow pattern to translate.
6988  * @param[in] inner
6989  *   Item is inner pattern.
6990  */
6991 static void
6992 flow_dv_translate_item_udp(void *matcher, void *key,
6993                            const struct rte_flow_item *item,
6994                            int inner)
6995 {
6996         const struct rte_flow_item_udp *udp_m = item->mask;
6997         const struct rte_flow_item_udp *udp_v = item->spec;
6998         void *headers_m;
6999         void *headers_v;
7000
7001         if (inner) {
7002                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7003                                          inner_headers);
7004                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7005         } else {
7006                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7007                                          outer_headers);
7008                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7009         }
7010         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
7011         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
7012         if (!udp_v)
7013                 return;
7014         if (!udp_m)
7015                 udp_m = &rte_flow_item_udp_mask;
7016         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
7017                  rte_be_to_cpu_16(udp_m->hdr.src_port));
7018         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
7019                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
7020         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
7021                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
7022         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
7023                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
7024 }
7025
7026 /**
7027  * Add GRE optional Key item to matcher and to the value.
7028  *
7029  * @param[in, out] matcher
7030  *   Flow matcher.
7031  * @param[in, out] key
7032  *   Flow matcher value.
7033  * @param[in] item
7034  *   Flow pattern to translate.
7035  * @param[in] inner
7036  *   Item is inner pattern.
7037  */
7038 static void
7039 flow_dv_translate_item_gre_key(void *matcher, void *key,
7040                                    const struct rte_flow_item *item)
7041 {
7042         const rte_be32_t *key_m = item->mask;
7043         const rte_be32_t *key_v = item->spec;
7044         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7045         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7046         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
7047
7048         /* GRE K bit must be on and should already be validated */
7049         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
7050         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
7051         if (!key_v)
7052                 return;
7053         if (!key_m)
7054                 key_m = &gre_key_default_mask;
7055         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
7056                  rte_be_to_cpu_32(*key_m) >> 8);
7057         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
7058                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
7059         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
7060                  rte_be_to_cpu_32(*key_m) & 0xFF);
7061         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
7062                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
7063 }
7064
7065 /**
7066  * Add GRE item to matcher and to the value.
7067  *
7068  * @param[in, out] matcher
7069  *   Flow matcher.
7070  * @param[in, out] key
7071  *   Flow matcher value.
7072  * @param[in] item
7073  *   Flow pattern to translate.
7074  * @param[in] inner
7075  *   Item is inner pattern.
7076  */
7077 static void
7078 flow_dv_translate_item_gre(void *matcher, void *key,
7079                            const struct rte_flow_item *item,
7080                            int inner)
7081 {
7082         const struct rte_flow_item_gre *gre_m = item->mask;
7083         const struct rte_flow_item_gre *gre_v = item->spec;
7084         void *headers_m;
7085         void *headers_v;
7086         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7087         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7088         struct {
7089                 union {
7090                         __extension__
7091                         struct {
7092                                 uint16_t version:3;
7093                                 uint16_t rsvd0:9;
7094                                 uint16_t s_present:1;
7095                                 uint16_t k_present:1;
7096                                 uint16_t rsvd_bit1:1;
7097                                 uint16_t c_present:1;
7098                         };
7099                         uint16_t value;
7100                 };
7101         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
7102
7103         if (inner) {
7104                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7105                                          inner_headers);
7106                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7107         } else {
7108                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7109                                          outer_headers);
7110                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7111         }
7112         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
7113         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
7114         if (!gre_v)
7115                 return;
7116         if (!gre_m)
7117                 gre_m = &rte_flow_item_gre_mask;
7118         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
7119                  rte_be_to_cpu_16(gre_m->protocol));
7120         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
7121                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
7122         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
7123         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
7124         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
7125                  gre_crks_rsvd0_ver_m.c_present);
7126         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
7127                  gre_crks_rsvd0_ver_v.c_present &
7128                  gre_crks_rsvd0_ver_m.c_present);
7129         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
7130                  gre_crks_rsvd0_ver_m.k_present);
7131         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
7132                  gre_crks_rsvd0_ver_v.k_present &
7133                  gre_crks_rsvd0_ver_m.k_present);
7134         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
7135                  gre_crks_rsvd0_ver_m.s_present);
7136         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
7137                  gre_crks_rsvd0_ver_v.s_present &
7138                  gre_crks_rsvd0_ver_m.s_present);
7139 }
7140
7141 /**
7142  * Add NVGRE item to matcher and to the value.
7143  *
7144  * @param[in, out] matcher
7145  *   Flow matcher.
7146  * @param[in, out] key
7147  *   Flow matcher value.
7148  * @param[in] item
7149  *   Flow pattern to translate.
7150  * @param[in] inner
7151  *   Item is inner pattern.
7152  */
7153 static void
7154 flow_dv_translate_item_nvgre(void *matcher, void *key,
7155                              const struct rte_flow_item *item,
7156                              int inner)
7157 {
7158         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
7159         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
7160         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7161         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7162         const char *tni_flow_id_m;
7163         const char *tni_flow_id_v;
7164         char *gre_key_m;
7165         char *gre_key_v;
7166         int size;
7167         int i;
7168
7169         /* For NVGRE, GRE header fields must be set with defined values. */
7170         const struct rte_flow_item_gre gre_spec = {
7171                 .c_rsvd0_ver = RTE_BE16(0x2000),
7172                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
7173         };
7174         const struct rte_flow_item_gre gre_mask = {
7175                 .c_rsvd0_ver = RTE_BE16(0xB000),
7176                 .protocol = RTE_BE16(UINT16_MAX),
7177         };
7178         const struct rte_flow_item gre_item = {
7179                 .spec = &gre_spec,
7180                 .mask = &gre_mask,
7181                 .last = NULL,
7182         };
7183         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
7184         if (!nvgre_v)
7185                 return;
7186         if (!nvgre_m)
7187                 nvgre_m = &rte_flow_item_nvgre_mask;
7188         tni_flow_id_m = (const char *)nvgre_m->tni;
7189         tni_flow_id_v = (const char *)nvgre_v->tni;
7190         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
7191         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
7192         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
7193         memcpy(gre_key_m, tni_flow_id_m, size);
7194         for (i = 0; i < size; ++i)
7195                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
7196 }
7197
7198 /**
7199  * Add VXLAN item to matcher and to the value.
7200  *
7201  * @param[in, out] matcher
7202  *   Flow matcher.
7203  * @param[in, out] key
7204  *   Flow matcher value.
7205  * @param[in] item
7206  *   Flow pattern to translate.
7207  * @param[in] inner
7208  *   Item is inner pattern.
7209  */
7210 static void
7211 flow_dv_translate_item_vxlan(void *matcher, void *key,
7212                              const struct rte_flow_item *item,
7213                              int inner)
7214 {
7215         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
7216         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
7217         void *headers_m;
7218         void *headers_v;
7219         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7220         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7221         char *vni_m;
7222         char *vni_v;
7223         uint16_t dport;
7224         int size;
7225         int i;
7226
7227         if (inner) {
7228                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7229                                          inner_headers);
7230                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7231         } else {
7232                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7233                                          outer_headers);
7234                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7235         }
7236         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
7237                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
7238         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7239                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7240                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7241         }
7242         if (!vxlan_v)
7243                 return;
7244         if (!vxlan_m)
7245                 vxlan_m = &rte_flow_item_vxlan_mask;
7246         size = sizeof(vxlan_m->vni);
7247         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
7248         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
7249         memcpy(vni_m, vxlan_m->vni, size);
7250         for (i = 0; i < size; ++i)
7251                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
7252 }
7253
7254 /**
7255  * Add VXLAN-GPE item to matcher and to the value.
7256  *
7257  * @param[in, out] matcher
7258  *   Flow matcher.
7259  * @param[in, out] key
7260  *   Flow matcher value.
7261  * @param[in] item
7262  *   Flow pattern to translate.
7263  * @param[in] inner
7264  *   Item is inner pattern.
7265  */
7266
7267 static void
7268 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
7269                                  const struct rte_flow_item *item, int inner)
7270 {
7271         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
7272         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
7273         void *headers_m;
7274         void *headers_v;
7275         void *misc_m =
7276                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
7277         void *misc_v =
7278                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7279         char *vni_m;
7280         char *vni_v;
7281         uint16_t dport;
7282         int size;
7283         int i;
7284         uint8_t flags_m = 0xff;
7285         uint8_t flags_v = 0xc;
7286
7287         if (inner) {
7288                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7289                                          inner_headers);
7290                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7291         } else {
7292                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7293                                          outer_headers);
7294                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7295         }
7296         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
7297                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
7298         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7299                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7300                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7301         }
7302         if (!vxlan_v)
7303                 return;
7304         if (!vxlan_m)
7305                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
7306         size = sizeof(vxlan_m->vni);
7307         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
7308         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
7309         memcpy(vni_m, vxlan_m->vni, size);
7310         for (i = 0; i < size; ++i)
7311                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
7312         if (vxlan_m->flags) {
7313                 flags_m = vxlan_m->flags;
7314                 flags_v = vxlan_v->flags;
7315         }
7316         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
7317         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
7318         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
7319                  vxlan_m->protocol);
7320         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
7321                  vxlan_v->protocol);
7322 }
7323
7324 /**
7325  * Add Geneve item to matcher and to the value.
7326  *
7327  * @param[in, out] matcher
7328  *   Flow matcher.
7329  * @param[in, out] key
7330  *   Flow matcher value.
7331  * @param[in] item
7332  *   Flow pattern to translate.
7333  * @param[in] inner
7334  *   Item is inner pattern.
7335  */
7336
7337 static void
7338 flow_dv_translate_item_geneve(void *matcher, void *key,
7339                               const struct rte_flow_item *item, int inner)
7340 {
7341         const struct rte_flow_item_geneve *geneve_m = item->mask;
7342         const struct rte_flow_item_geneve *geneve_v = item->spec;
7343         void *headers_m;
7344         void *headers_v;
7345         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7346         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7347         uint16_t dport;
7348         uint16_t gbhdr_m;
7349         uint16_t gbhdr_v;
7350         char *vni_m;
7351         char *vni_v;
7352         size_t size, i;
7353
7354         if (inner) {
7355                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7356                                          inner_headers);
7357                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7358         } else {
7359                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7360                                          outer_headers);
7361                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7362         }
7363         dport = MLX5_UDP_PORT_GENEVE;
7364         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7365                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7366                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7367         }
7368         if (!geneve_v)
7369                 return;
7370         if (!geneve_m)
7371                 geneve_m = &rte_flow_item_geneve_mask;
7372         size = sizeof(geneve_m->vni);
7373         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
7374         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
7375         memcpy(vni_m, geneve_m->vni, size);
7376         for (i = 0; i < size; ++i)
7377                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
7378         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
7379                  rte_be_to_cpu_16(geneve_m->protocol));
7380         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
7381                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
7382         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
7383         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
7384         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
7385                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7386         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
7387                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7388         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
7389                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7390         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
7391                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
7392                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7393 }
7394
7395 /**
7396  * Add MPLS item to matcher and to the value.
7397  *
7398  * @param[in, out] matcher
7399  *   Flow matcher.
7400  * @param[in, out] key
7401  *   Flow matcher value.
7402  * @param[in] item
7403  *   Flow pattern to translate.
7404  * @param[in] prev_layer
7405  *   The protocol layer indicated in previous item.
7406  * @param[in] inner
7407  *   Item is inner pattern.
7408  */
7409 static void
7410 flow_dv_translate_item_mpls(void *matcher, void *key,
7411                             const struct rte_flow_item *item,
7412                             uint64_t prev_layer,
7413                             int inner)
7414 {
7415         const uint32_t *in_mpls_m = item->mask;
7416         const uint32_t *in_mpls_v = item->spec;
7417         uint32_t *out_mpls_m = 0;
7418         uint32_t *out_mpls_v = 0;
7419         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7420         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7421         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
7422                                      misc_parameters_2);
7423         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7424         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
7425         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7426
7427         switch (prev_layer) {
7428         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7429                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
7430                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
7431                          MLX5_UDP_PORT_MPLS);
7432                 break;
7433         case MLX5_FLOW_LAYER_GRE:
7434                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
7435                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
7436                          RTE_ETHER_TYPE_MPLS);
7437                 break;
7438         default:
7439                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
7440                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
7441                          IPPROTO_MPLS);
7442                 break;
7443         }
7444         if (!in_mpls_v)
7445                 return;
7446         if (!in_mpls_m)
7447                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
7448         switch (prev_layer) {
7449         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7450                 out_mpls_m =
7451                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7452                                                  outer_first_mpls_over_udp);
7453                 out_mpls_v =
7454                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7455                                                  outer_first_mpls_over_udp);
7456                 break;
7457         case MLX5_FLOW_LAYER_GRE:
7458                 out_mpls_m =
7459                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7460                                                  outer_first_mpls_over_gre);
7461                 out_mpls_v =
7462                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7463                                                  outer_first_mpls_over_gre);
7464                 break;
7465         default:
7466                 /* Inner MPLS not over GRE is not supported. */
7467                 if (!inner) {
7468                         out_mpls_m =
7469                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7470                                                          misc2_m,
7471                                                          outer_first_mpls);
7472                         out_mpls_v =
7473                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7474                                                          misc2_v,
7475                                                          outer_first_mpls);
7476                 }
7477                 break;
7478         }
7479         if (out_mpls_m && out_mpls_v) {
7480                 *out_mpls_m = *in_mpls_m;
7481                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
7482         }
7483 }
7484
7485 /**
7486  * Add metadata register item to matcher
7487  *
7488  * @param[in, out] matcher
7489  *   Flow matcher.
7490  * @param[in, out] key
7491  *   Flow matcher value.
7492  * @param[in] reg_type
7493  *   Type of device metadata register
7494  * @param[in] value
7495  *   Register value
7496  * @param[in] mask
7497  *   Register mask
7498  */
7499 static void
7500 flow_dv_match_meta_reg(void *matcher, void *key,
7501                        enum modify_reg reg_type,
7502                        uint32_t data, uint32_t mask)
7503 {
7504         void *misc2_m =
7505                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
7506         void *misc2_v =
7507                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7508         uint32_t temp;
7509
7510         data &= mask;
7511         switch (reg_type) {
7512         case REG_A:
7513                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
7514                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
7515                 break;
7516         case REG_B:
7517                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
7518                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
7519                 break;
7520         case REG_C_0:
7521                 /*
7522                  * The metadata register C0 field might be divided into
7523                  * source vport index and META item value, we should set
7524                  * this field according to specified mask, not as whole one.
7525                  */
7526                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
7527                 temp |= mask;
7528                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
7529                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
7530                 temp &= ~mask;
7531                 temp |= data;
7532                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
7533                 break;
7534         case REG_C_1:
7535                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
7536                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
7537                 break;
7538         case REG_C_2:
7539                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
7540                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
7541                 break;
7542         case REG_C_3:
7543                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
7544                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
7545                 break;
7546         case REG_C_4:
7547                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
7548                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
7549                 break;
7550         case REG_C_5:
7551                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
7552                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
7553                 break;
7554         case REG_C_6:
7555                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
7556                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
7557                 break;
7558         case REG_C_7:
7559                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
7560                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
7561                 break;
7562         default:
7563                 MLX5_ASSERT(false);
7564                 break;
7565         }
7566 }
7567
7568 /**
7569  * Add MARK item to matcher
7570  *
7571  * @param[in] dev
7572  *   The device to configure through.
7573  * @param[in, out] matcher
7574  *   Flow matcher.
7575  * @param[in, out] key
7576  *   Flow matcher value.
7577  * @param[in] item
7578  *   Flow pattern to translate.
7579  */
7580 static void
7581 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
7582                             void *matcher, void *key,
7583                             const struct rte_flow_item *item)
7584 {
7585         struct mlx5_priv *priv = dev->data->dev_private;
7586         const struct rte_flow_item_mark *mark;
7587         uint32_t value;
7588         uint32_t mask;
7589
7590         mark = item->mask ? (const void *)item->mask :
7591                             &rte_flow_item_mark_mask;
7592         mask = mark->id & priv->sh->dv_mark_mask;
7593         mark = (const void *)item->spec;
7594         MLX5_ASSERT(mark);
7595         value = mark->id & priv->sh->dv_mark_mask & mask;
7596         if (mask) {
7597                 enum modify_reg reg;
7598
7599                 /* Get the metadata register index for the mark. */
7600                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
7601                 MLX5_ASSERT(reg > 0);
7602                 if (reg == REG_C_0) {
7603                         struct mlx5_priv *priv = dev->data->dev_private;
7604                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7605                         uint32_t shl_c0 = rte_bsf32(msk_c0);
7606
7607                         mask &= msk_c0;
7608                         mask <<= shl_c0;
7609                         value <<= shl_c0;
7610                 }
7611                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7612         }
7613 }
7614
7615 /**
7616  * Add META item to matcher
7617  *
7618  * @param[in] dev
7619  *   The devich to configure through.
7620  * @param[in, out] matcher
7621  *   Flow matcher.
7622  * @param[in, out] key
7623  *   Flow matcher value.
7624  * @param[in] attr
7625  *   Attributes of flow that includes this item.
7626  * @param[in] item
7627  *   Flow pattern to translate.
7628  */
7629 static void
7630 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
7631                             void *matcher, void *key,
7632                             const struct rte_flow_attr *attr,
7633                             const struct rte_flow_item *item)
7634 {
7635         const struct rte_flow_item_meta *meta_m;
7636         const struct rte_flow_item_meta *meta_v;
7637
7638         meta_m = (const void *)item->mask;
7639         if (!meta_m)
7640                 meta_m = &rte_flow_item_meta_mask;
7641         meta_v = (const void *)item->spec;
7642         if (meta_v) {
7643                 int reg;
7644                 uint32_t value = meta_v->data;
7645                 uint32_t mask = meta_m->data;
7646
7647                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
7648                 if (reg < 0)
7649                         return;
7650                 /*
7651                  * In datapath code there is no endianness
7652                  * coversions for perfromance reasons, all
7653                  * pattern conversions are done in rte_flow.
7654                  */
7655                 value = rte_cpu_to_be_32(value);
7656                 mask = rte_cpu_to_be_32(mask);
7657                 if (reg == REG_C_0) {
7658                         struct mlx5_priv *priv = dev->data->dev_private;
7659                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7660                         uint32_t shl_c0 = rte_bsf32(msk_c0);
7661 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
7662                         uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
7663
7664                         value >>= shr_c0;
7665                         mask >>= shr_c0;
7666 #endif
7667                         value <<= shl_c0;
7668                         mask <<= shl_c0;
7669                         MLX5_ASSERT(msk_c0);
7670                         MLX5_ASSERT(!(~msk_c0 & mask));
7671                 }
7672                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7673         }
7674 }
7675
7676 /**
7677  * Add vport metadata Reg C0 item to matcher
7678  *
7679  * @param[in, out] matcher
7680  *   Flow matcher.
7681  * @param[in, out] key
7682  *   Flow matcher value.
7683  * @param[in] reg
7684  *   Flow pattern to translate.
7685  */
7686 static void
7687 flow_dv_translate_item_meta_vport(void *matcher, void *key,
7688                                   uint32_t value, uint32_t mask)
7689 {
7690         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
7691 }
7692
7693 /**
7694  * Add tag item to matcher
7695  *
7696  * @param[in] dev
7697  *   The devich to configure through.
7698  * @param[in, out] matcher
7699  *   Flow matcher.
7700  * @param[in, out] key
7701  *   Flow matcher value.
7702  * @param[in] item
7703  *   Flow pattern to translate.
7704  */
7705 static void
7706 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
7707                                 void *matcher, void *key,
7708                                 const struct rte_flow_item *item)
7709 {
7710         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
7711         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
7712         uint32_t mask, value;
7713
7714         MLX5_ASSERT(tag_v);
7715         value = tag_v->data;
7716         mask = tag_m ? tag_m->data : UINT32_MAX;
7717         if (tag_v->id == REG_C_0) {
7718                 struct mlx5_priv *priv = dev->data->dev_private;
7719                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7720                 uint32_t shl_c0 = rte_bsf32(msk_c0);
7721
7722                 mask &= msk_c0;
7723                 mask <<= shl_c0;
7724                 value <<= shl_c0;
7725         }
7726         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
7727 }
7728
7729 /**
7730  * Add TAG item to matcher
7731  *
7732  * @param[in] dev
7733  *   The devich to configure through.
7734  * @param[in, out] matcher
7735  *   Flow matcher.
7736  * @param[in, out] key
7737  *   Flow matcher value.
7738  * @param[in] item
7739  *   Flow pattern to translate.
7740  */
7741 static void
7742 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
7743                            void *matcher, void *key,
7744                            const struct rte_flow_item *item)
7745 {
7746         const struct rte_flow_item_tag *tag_v = item->spec;
7747         const struct rte_flow_item_tag *tag_m = item->mask;
7748         enum modify_reg reg;
7749
7750         MLX5_ASSERT(tag_v);
7751         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
7752         /* Get the metadata register index for the tag. */
7753         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
7754         MLX5_ASSERT(reg > 0);
7755         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
7756 }
7757
7758 /**
7759  * Add source vport match to the specified matcher.
7760  *
7761  * @param[in, out] matcher
7762  *   Flow matcher.
7763  * @param[in, out] key
7764  *   Flow matcher value.
7765  * @param[in] port
7766  *   Source vport value to match
7767  * @param[in] mask
7768  *   Mask
7769  */
7770 static void
7771 flow_dv_translate_item_source_vport(void *matcher, void *key,
7772                                     int16_t port, uint16_t mask)
7773 {
7774         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7775         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7776
7777         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
7778         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
7779 }
7780
7781 /**
7782  * Translate port-id item to eswitch match on  port-id.
7783  *
7784  * @param[in] dev
7785  *   The devich to configure through.
7786  * @param[in, out] matcher
7787  *   Flow matcher.
7788  * @param[in, out] key
7789  *   Flow matcher value.
7790  * @param[in] item
7791  *   Flow pattern to translate.
7792  *
7793  * @return
7794  *   0 on success, a negative errno value otherwise.
7795  */
7796 static int
7797 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
7798                                void *key, const struct rte_flow_item *item)
7799 {
7800         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
7801         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
7802         struct mlx5_priv *priv;
7803         uint16_t mask, id;
7804
7805         mask = pid_m ? pid_m->id : 0xffff;
7806         id = pid_v ? pid_v->id : dev->data->port_id;
7807         priv = mlx5_port_to_eswitch_info(id, item == NULL);
7808         if (!priv)
7809                 return -rte_errno;
7810         /* Translate to vport field or to metadata, depending on mode. */
7811         if (priv->vport_meta_mask)
7812                 flow_dv_translate_item_meta_vport(matcher, key,
7813                                                   priv->vport_meta_tag,
7814                                                   priv->vport_meta_mask);
7815         else
7816                 flow_dv_translate_item_source_vport(matcher, key,
7817                                                     priv->vport_id, mask);
7818         return 0;
7819 }
7820
7821 /**
7822  * Add ICMP6 item to matcher and to the value.
7823  *
7824  * @param[in, out] matcher
7825  *   Flow matcher.
7826  * @param[in, out] key
7827  *   Flow matcher value.
7828  * @param[in] item
7829  *   Flow pattern to translate.
7830  * @param[in] inner
7831  *   Item is inner pattern.
7832  */
7833 static void
7834 flow_dv_translate_item_icmp6(void *matcher, void *key,
7835                               const struct rte_flow_item *item,
7836                               int inner)
7837 {
7838         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
7839         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
7840         void *headers_m;
7841         void *headers_v;
7842         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7843                                      misc_parameters_3);
7844         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7845         if (inner) {
7846                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7847                                          inner_headers);
7848                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7849         } else {
7850                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7851                                          outer_headers);
7852                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7853         }
7854         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7855         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
7856         if (!icmp6_v)
7857                 return;
7858         if (!icmp6_m)
7859                 icmp6_m = &rte_flow_item_icmp6_mask;
7860         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
7861         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
7862                  icmp6_v->type & icmp6_m->type);
7863         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
7864         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
7865                  icmp6_v->code & icmp6_m->code);
7866 }
7867
7868 /**
7869  * Add ICMP item to matcher and to the value.
7870  *
7871  * @param[in, out] matcher
7872  *   Flow matcher.
7873  * @param[in, out] key
7874  *   Flow matcher value.
7875  * @param[in] item
7876  *   Flow pattern to translate.
7877  * @param[in] inner
7878  *   Item is inner pattern.
7879  */
7880 static void
7881 flow_dv_translate_item_icmp(void *matcher, void *key,
7882                             const struct rte_flow_item *item,
7883                             int inner)
7884 {
7885         const struct rte_flow_item_icmp *icmp_m = item->mask;
7886         const struct rte_flow_item_icmp *icmp_v = item->spec;
7887         uint32_t icmp_header_data_m = 0;
7888         uint32_t icmp_header_data_v = 0;
7889         void *headers_m;
7890         void *headers_v;
7891         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7892                                      misc_parameters_3);
7893         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7894         if (inner) {
7895                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7896                                          inner_headers);
7897                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7898         } else {
7899                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7900                                          outer_headers);
7901                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7902         }
7903         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7904         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
7905         if (!icmp_v)
7906                 return;
7907         if (!icmp_m)
7908                 icmp_m = &rte_flow_item_icmp_mask;
7909         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
7910                  icmp_m->hdr.icmp_type);
7911         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
7912                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
7913         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
7914                  icmp_m->hdr.icmp_code);
7915         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
7916                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
7917         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
7918         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
7919         if (icmp_header_data_m) {
7920                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
7921                 icmp_header_data_v |=
7922                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
7923                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
7924                          icmp_header_data_m);
7925                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
7926                          icmp_header_data_v & icmp_header_data_m);
7927         }
7928 }
7929
7930 /**
7931  * Add GTP item to matcher and to the value.
7932  *
7933  * @param[in, out] matcher
7934  *   Flow matcher.
7935  * @param[in, out] key
7936  *   Flow matcher value.
7937  * @param[in] item
7938  *   Flow pattern to translate.
7939  * @param[in] inner
7940  *   Item is inner pattern.
7941  */
7942 static void
7943 flow_dv_translate_item_gtp(void *matcher, void *key,
7944                            const struct rte_flow_item *item, int inner)
7945 {
7946         const struct rte_flow_item_gtp *gtp_m = item->mask;
7947         const struct rte_flow_item_gtp *gtp_v = item->spec;
7948         void *headers_m;
7949         void *headers_v;
7950         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7951                                      misc_parameters_3);
7952         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7953         uint16_t dport = RTE_GTPU_UDP_PORT;
7954
7955         if (inner) {
7956                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7957                                          inner_headers);
7958                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7959         } else {
7960                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7961                                          outer_headers);
7962                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7963         }
7964         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7965                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7966                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7967         }
7968         if (!gtp_v)
7969                 return;
7970         if (!gtp_m)
7971                 gtp_m = &rte_flow_item_gtp_mask;
7972         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
7973                  gtp_m->v_pt_rsv_flags);
7974         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
7975                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
7976         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
7977         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
7978                  gtp_v->msg_type & gtp_m->msg_type);
7979         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
7980                  rte_be_to_cpu_32(gtp_m->teid));
7981         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
7982                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
7983 }
7984
7985 /**
7986  * Add eCPRI item to matcher and to the value.
7987  *
7988  * @param[in] dev
7989  *   The devich to configure through.
7990  * @param[in, out] matcher
7991  *   Flow matcher.
7992  * @param[in, out] key
7993  *   Flow matcher value.
7994  * @param[in] item
7995  *   Flow pattern to translate.
7996  * @param[in] samples
7997  *   Sample IDs to be used in the matching.
7998  */
7999 static void
8000 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
8001                              void *key, const struct rte_flow_item *item)
8002 {
8003         struct mlx5_priv *priv = dev->data->dev_private;
8004         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
8005         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
8006         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
8007                                      misc_parameters_4);
8008         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
8009         uint32_t *samples;
8010         void *dw_m;
8011         void *dw_v;
8012
8013         if (!ecpri_v)
8014                 return;
8015         if (!ecpri_m)
8016                 ecpri_m = &rte_flow_item_ecpri_mask;
8017         /*
8018          * Maximal four DW samples are supported in a single matching now.
8019          * Two are used now for a eCPRI matching:
8020          * 1. Type: one byte, mask should be 0x00ff0000 in network order
8021          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
8022          *    if any.
8023          */
8024         if (!ecpri_m->hdr.common.u32)
8025                 return;
8026         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
8027         /* Need to take the whole DW as the mask to fill the entry. */
8028         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
8029                             prog_sample_field_value_0);
8030         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
8031                             prog_sample_field_value_0);
8032         /* Already big endian (network order) in the header. */
8033         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
8034         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32;
8035         /* Sample#0, used for matching type, offset 0. */
8036         MLX5_SET(fte_match_set_misc4, misc4_m,
8037                  prog_sample_field_id_0, samples[0]);
8038         /* It makes no sense to set the sample ID in the mask field. */
8039         MLX5_SET(fte_match_set_misc4, misc4_v,
8040                  prog_sample_field_id_0, samples[0]);
8041         /*
8042          * Checking if message body part needs to be matched.
8043          * Some wildcard rules only matching type field should be supported.
8044          */
8045         if (ecpri_m->hdr.dummy[0]) {
8046                 switch (ecpri_v->hdr.common.type) {
8047                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
8048                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
8049                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
8050                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
8051                                             prog_sample_field_value_1);
8052                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
8053                                             prog_sample_field_value_1);
8054                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
8055                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0];
8056                         /* Sample#1, to match message body, offset 4. */
8057                         MLX5_SET(fte_match_set_misc4, misc4_m,
8058                                  prog_sample_field_id_1, samples[1]);
8059                         MLX5_SET(fte_match_set_misc4, misc4_v,
8060                                  prog_sample_field_id_1, samples[1]);
8061                         break;
8062                 default:
8063                         /* Others, do not match any sample ID. */
8064                         break;
8065                 }
8066         }
8067 }
8068
8069 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
8070
8071 #define HEADER_IS_ZERO(match_criteria, headers)                              \
8072         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
8073                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
8074
8075 /**
8076  * Calculate flow matcher enable bitmap.
8077  *
8078  * @param match_criteria
8079  *   Pointer to flow matcher criteria.
8080  *
8081  * @return
8082  *   Bitmap of enabled fields.
8083  */
8084 static uint8_t
8085 flow_dv_matcher_enable(uint32_t *match_criteria)
8086 {
8087         uint8_t match_criteria_enable;
8088
8089         match_criteria_enable =
8090                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
8091                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
8092         match_criteria_enable |=
8093                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
8094                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
8095         match_criteria_enable |=
8096                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
8097                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
8098         match_criteria_enable |=
8099                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
8100                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
8101         match_criteria_enable |=
8102                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
8103                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
8104         match_criteria_enable |=
8105                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
8106                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
8107         return match_criteria_enable;
8108 }
8109
8110
8111 /**
8112  * Get a flow table.
8113  *
8114  * @param[in, out] dev
8115  *   Pointer to rte_eth_dev structure.
8116  * @param[in] table_id
8117  *   Table id to use.
8118  * @param[in] egress
8119  *   Direction of the table.
8120  * @param[in] transfer
8121  *   E-Switch or NIC flow.
8122  * @param[out] error
8123  *   pointer to error structure.
8124  *
8125  * @return
8126  *   Returns tables resource based on the index, NULL in case of failed.
8127  */
8128 static struct mlx5_flow_tbl_resource *
8129 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
8130                          uint32_t table_id, uint8_t egress,
8131                          uint8_t transfer,
8132                          struct rte_flow_error *error)
8133 {
8134         struct mlx5_priv *priv = dev->data->dev_private;
8135         struct mlx5_dev_ctx_shared *sh = priv->sh;
8136         struct mlx5_flow_tbl_resource *tbl;
8137         union mlx5_flow_tbl_key table_key = {
8138                 {
8139                         .table_id = table_id,
8140                         .reserved = 0,
8141                         .domain = !!transfer,
8142                         .direction = !!egress,
8143                 }
8144         };
8145         struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
8146                                                          table_key.v64);
8147         struct mlx5_flow_tbl_data_entry *tbl_data;
8148         uint32_t idx = 0;
8149         int ret;
8150         void *domain;
8151
8152         if (pos) {
8153                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
8154                                         entry);
8155                 tbl = &tbl_data->tbl;
8156                 rte_atomic32_inc(&tbl->refcnt);
8157                 return tbl;
8158         }
8159         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
8160         if (!tbl_data) {
8161                 rte_flow_error_set(error, ENOMEM,
8162                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8163                                    NULL,
8164                                    "cannot allocate flow table data entry");
8165                 return NULL;
8166         }
8167         tbl_data->idx = idx;
8168         tbl = &tbl_data->tbl;
8169         pos = &tbl_data->entry;
8170         if (transfer)
8171                 domain = sh->fdb_domain;
8172         else if (egress)
8173                 domain = sh->tx_domain;
8174         else
8175                 domain = sh->rx_domain;
8176         ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj);
8177         if (ret) {
8178                 rte_flow_error_set(error, ENOMEM,
8179                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8180                                    NULL, "cannot create flow table object");
8181                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
8182                 return NULL;
8183         }
8184         /*
8185          * No multi-threads now, but still better to initialize the reference
8186          * count before insert it into the hash list.
8187          */
8188         rte_atomic32_init(&tbl->refcnt);
8189         /* Jump action reference count is initialized here. */
8190         rte_atomic32_init(&tbl_data->jump.refcnt);
8191         pos->key = table_key.v64;
8192         ret = mlx5_hlist_insert(sh->flow_tbls, pos);
8193         if (ret < 0) {
8194                 rte_flow_error_set(error, -ret,
8195                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8196                                    "cannot insert flow table data entry");
8197                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
8198                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
8199         }
8200         rte_atomic32_inc(&tbl->refcnt);
8201         return tbl;
8202 }
8203
8204 /**
8205  * Release a flow table.
8206  *
8207  * @param[in] dev
8208  *   Pointer to rte_eth_dev structure.
8209  * @param[in] tbl
8210  *   Table resource to be released.
8211  *
8212  * @return
8213  *   Returns 0 if table was released, else return 1;
8214  */
8215 static int
8216 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
8217                              struct mlx5_flow_tbl_resource *tbl)
8218 {
8219         struct mlx5_priv *priv = dev->data->dev_private;
8220         struct mlx5_dev_ctx_shared *sh = priv->sh;
8221         struct mlx5_flow_tbl_data_entry *tbl_data =
8222                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
8223
8224         if (!tbl)
8225                 return 0;
8226         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
8227                 struct mlx5_hlist_entry *pos = &tbl_data->entry;
8228
8229                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
8230                 tbl->obj = NULL;
8231                 /* remove the entry from the hash list and free memory. */
8232                 mlx5_hlist_remove(sh->flow_tbls, pos);
8233                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
8234                                 tbl_data->idx);
8235                 return 0;
8236         }
8237         return 1;
8238 }
8239
8240 /**
8241  * Register the flow matcher.
8242  *
8243  * @param[in, out] dev
8244  *   Pointer to rte_eth_dev structure.
8245  * @param[in, out] matcher
8246  *   Pointer to flow matcher.
8247  * @param[in, out] key
8248  *   Pointer to flow table key.
8249  * @parm[in, out] dev_flow
8250  *   Pointer to the dev_flow.
8251  * @param[out] error
8252  *   pointer to error structure.
8253  *
8254  * @return
8255  *   0 on success otherwise -errno and errno is set.
8256  */
8257 static int
8258 flow_dv_matcher_register(struct rte_eth_dev *dev,
8259                          struct mlx5_flow_dv_matcher *matcher,
8260                          union mlx5_flow_tbl_key *key,
8261                          struct mlx5_flow *dev_flow,
8262                          struct rte_flow_error *error)
8263 {
8264         struct mlx5_priv *priv = dev->data->dev_private;
8265         struct mlx5_dev_ctx_shared *sh = priv->sh;
8266         struct mlx5_flow_dv_matcher *cache_matcher;
8267         struct mlx5dv_flow_matcher_attr dv_attr = {
8268                 .type = IBV_FLOW_ATTR_NORMAL,
8269                 .match_mask = (void *)&matcher->mask,
8270         };
8271         struct mlx5_flow_tbl_resource *tbl;
8272         struct mlx5_flow_tbl_data_entry *tbl_data;
8273         int ret;
8274
8275         tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
8276                                        key->domain, error);
8277         if (!tbl)
8278                 return -rte_errno;      /* No need to refill the error info */
8279         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
8280         /* Lookup from cache. */
8281         LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
8282                 if (matcher->crc == cache_matcher->crc &&
8283                     matcher->priority == cache_matcher->priority &&
8284                     !memcmp((const void *)matcher->mask.buf,
8285                             (const void *)cache_matcher->mask.buf,
8286                             cache_matcher->mask.size)) {
8287                         DRV_LOG(DEBUG,
8288                                 "%s group %u priority %hd use %s "
8289                                 "matcher %p: refcnt %d++",
8290                                 key->domain ? "FDB" : "NIC", key->table_id,
8291                                 cache_matcher->priority,
8292                                 key->direction ? "tx" : "rx",
8293                                 (void *)cache_matcher,
8294                                 rte_atomic32_read(&cache_matcher->refcnt));
8295                         rte_atomic32_inc(&cache_matcher->refcnt);
8296                         dev_flow->handle->dvh.matcher = cache_matcher;
8297                         /* old matcher should not make the table ref++. */
8298                         flow_dv_tbl_resource_release(dev, tbl);
8299                         return 0;
8300                 }
8301         }
8302         /* Register new matcher. */
8303         cache_matcher = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache_matcher), 0,
8304                                     SOCKET_ID_ANY);
8305         if (!cache_matcher) {
8306                 flow_dv_tbl_resource_release(dev, tbl);
8307                 return rte_flow_error_set(error, ENOMEM,
8308                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8309                                           "cannot allocate matcher memory");
8310         }
8311         *cache_matcher = *matcher;
8312         dv_attr.match_criteria_enable =
8313                 flow_dv_matcher_enable(cache_matcher->mask.buf);
8314         dv_attr.priority = matcher->priority;
8315         if (key->direction)
8316                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
8317         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
8318                                                &cache_matcher->matcher_object);
8319         if (ret) {
8320                 mlx5_free(cache_matcher);
8321 #ifdef HAVE_MLX5DV_DR
8322                 flow_dv_tbl_resource_release(dev, tbl);
8323 #endif
8324                 return rte_flow_error_set(error, ENOMEM,
8325                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8326                                           NULL, "cannot create matcher");
8327         }
8328         /* Save the table information */
8329         cache_matcher->tbl = tbl;
8330         rte_atomic32_init(&cache_matcher->refcnt);
8331         /* only matcher ref++, table ref++ already done above in get API. */
8332         rte_atomic32_inc(&cache_matcher->refcnt);
8333         LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
8334         dev_flow->handle->dvh.matcher = cache_matcher;
8335         DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
8336                 key->domain ? "FDB" : "NIC", key->table_id,
8337                 cache_matcher->priority,
8338                 key->direction ? "tx" : "rx", (void *)cache_matcher,
8339                 rte_atomic32_read(&cache_matcher->refcnt));
8340         return 0;
8341 }
8342
8343 /**
8344  * Find existing tag resource or create and register a new one.
8345  *
8346  * @param dev[in, out]
8347  *   Pointer to rte_eth_dev structure.
8348  * @param[in, out] tag_be24
8349  *   Tag value in big endian then R-shift 8.
8350  * @parm[in, out] dev_flow
8351  *   Pointer to the dev_flow.
8352  * @param[out] error
8353  *   pointer to error structure.
8354  *
8355  * @return
8356  *   0 on success otherwise -errno and errno is set.
8357  */
8358 static int
8359 flow_dv_tag_resource_register
8360                         (struct rte_eth_dev *dev,
8361                          uint32_t tag_be24,
8362                          struct mlx5_flow *dev_flow,
8363                          struct rte_flow_error *error)
8364 {
8365         struct mlx5_priv *priv = dev->data->dev_private;
8366         struct mlx5_dev_ctx_shared *sh = priv->sh;
8367         struct mlx5_flow_dv_tag_resource *cache_resource;
8368         struct mlx5_hlist_entry *entry;
8369         int ret;
8370
8371         /* Lookup a matching resource from cache. */
8372         entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
8373         if (entry) {
8374                 cache_resource = container_of
8375                         (entry, struct mlx5_flow_dv_tag_resource, entry);
8376                 rte_atomic32_inc(&cache_resource->refcnt);
8377                 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
8378                 dev_flow->dv.tag_resource = cache_resource;
8379                 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
8380                         (void *)cache_resource,
8381                         rte_atomic32_read(&cache_resource->refcnt));
8382                 return 0;
8383         }
8384         /* Register new resource. */
8385         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG],
8386                                        &dev_flow->handle->dvh.rix_tag);
8387         if (!cache_resource)
8388                 return rte_flow_error_set(error, ENOMEM,
8389                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8390                                           "cannot allocate resource memory");
8391         cache_resource->entry.key = (uint64_t)tag_be24;
8392         ret = mlx5_flow_os_create_flow_action_tag(tag_be24,
8393                                                   &cache_resource->action);
8394         if (ret) {
8395                 mlx5_free(cache_resource);
8396                 return rte_flow_error_set(error, ENOMEM,
8397                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8398                                           NULL, "cannot create action");
8399         }
8400         rte_atomic32_init(&cache_resource->refcnt);
8401         rte_atomic32_inc(&cache_resource->refcnt);
8402         if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
8403                 mlx5_flow_os_destroy_flow_action(cache_resource->action);
8404                 mlx5_free(cache_resource);
8405                 return rte_flow_error_set(error, EEXIST,
8406                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8407                                           NULL, "cannot insert tag");
8408         }
8409         dev_flow->dv.tag_resource = cache_resource;
8410         DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
8411                 (void *)cache_resource,
8412                 rte_atomic32_read(&cache_resource->refcnt));
8413         return 0;
8414 }
8415
8416 /**
8417  * Release the tag.
8418  *
8419  * @param dev
8420  *   Pointer to Ethernet device.
8421  * @param tag_idx
8422  *   Tag index.
8423  *
8424  * @return
8425  *   1 while a reference on it exists, 0 when freed.
8426  */
8427 static int
8428 flow_dv_tag_release(struct rte_eth_dev *dev,
8429                     uint32_t tag_idx)
8430 {
8431         struct mlx5_priv *priv = dev->data->dev_private;
8432         struct mlx5_dev_ctx_shared *sh = priv->sh;
8433         struct mlx5_flow_dv_tag_resource *tag;
8434
8435         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8436         if (!tag)
8437                 return 0;
8438         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
8439                 dev->data->port_id, (void *)tag,
8440                 rte_atomic32_read(&tag->refcnt));
8441         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
8442                 claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
8443                 mlx5_hlist_remove(sh->tag_table, &tag->entry);
8444                 DRV_LOG(DEBUG, "port %u tag %p: removed",
8445                         dev->data->port_id, (void *)tag);
8446                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8447                 return 0;
8448         }
8449         return 1;
8450 }
8451
8452 /**
8453  * Translate port ID action to vport.
8454  *
8455  * @param[in] dev
8456  *   Pointer to rte_eth_dev structure.
8457  * @param[in] action
8458  *   Pointer to the port ID action.
8459  * @param[out] dst_port_id
8460  *   The target port ID.
8461  * @param[out] error
8462  *   Pointer to the error structure.
8463  *
8464  * @return
8465  *   0 on success, a negative errno value otherwise and rte_errno is set.
8466  */
8467 static int
8468 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
8469                                  const struct rte_flow_action *action,
8470                                  uint32_t *dst_port_id,
8471                                  struct rte_flow_error *error)
8472 {
8473         uint32_t port;
8474         struct mlx5_priv *priv;
8475         const struct rte_flow_action_port_id *conf =
8476                         (const struct rte_flow_action_port_id *)action->conf;
8477
8478         port = conf->original ? dev->data->port_id : conf->id;
8479         priv = mlx5_port_to_eswitch_info(port, false);
8480         if (!priv)
8481                 return rte_flow_error_set(error, -rte_errno,
8482                                           RTE_FLOW_ERROR_TYPE_ACTION,
8483                                           NULL,
8484                                           "No eswitch info was found for port");
8485 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
8486         /*
8487          * This parameter is transferred to
8488          * mlx5dv_dr_action_create_dest_ib_port().
8489          */
8490         *dst_port_id = priv->dev_port;
8491 #else
8492         /*
8493          * Legacy mode, no LAG configurations is supported.
8494          * This parameter is transferred to
8495          * mlx5dv_dr_action_create_dest_vport().
8496          */
8497         *dst_port_id = priv->vport_id;
8498 #endif
8499         return 0;
8500 }
8501
8502 /**
8503  * Create a counter with aging configuration.
8504  *
8505  * @param[in] dev
8506  *   Pointer to rte_eth_dev structure.
8507  * @param[out] count
8508  *   Pointer to the counter action configuration.
8509  * @param[in] age
8510  *   Pointer to the aging action configuration.
8511  *
8512  * @return
8513  *   Index to flow counter on success, 0 otherwise.
8514  */
8515 static uint32_t
8516 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
8517                                 struct mlx5_flow *dev_flow,
8518                                 const struct rte_flow_action_count *count,
8519                                 const struct rte_flow_action_age *age)
8520 {
8521         uint32_t counter;
8522         struct mlx5_age_param *age_param;
8523
8524         counter = flow_dv_counter_alloc(dev,
8525                                 count ? count->shared : 0,
8526                                 count ? count->id : 0,
8527                                 dev_flow->dv.group, !!age);
8528         if (!counter || age == NULL)
8529                 return counter;
8530         age_param  = flow_dv_counter_idx_get_age(dev, counter);
8531         age_param->context = age->context ? age->context :
8532                 (void *)(uintptr_t)(dev_flow->flow_idx);
8533         age_param->timeout = age->timeout;
8534         age_param->port_id = dev->data->port_id;
8535         __atomic_store_n(&age_param->sec_since_last_hit, 0, __ATOMIC_RELAXED);
8536         __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED);
8537         return counter;
8538 }
8539 /**
8540  * Add Tx queue matcher
8541  *
8542  * @param[in] dev
8543  *   Pointer to the dev struct.
8544  * @param[in, out] matcher
8545  *   Flow matcher.
8546  * @param[in, out] key
8547  *   Flow matcher value.
8548  * @param[in] item
8549  *   Flow pattern to translate.
8550  * @param[in] inner
8551  *   Item is inner pattern.
8552  */
8553 static void
8554 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
8555                                 void *matcher, void *key,
8556                                 const struct rte_flow_item *item)
8557 {
8558         const struct mlx5_rte_flow_item_tx_queue *queue_m;
8559         const struct mlx5_rte_flow_item_tx_queue *queue_v;
8560         void *misc_m =
8561                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8562         void *misc_v =
8563                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8564         struct mlx5_txq_ctrl *txq;
8565         uint32_t queue;
8566
8567
8568         queue_m = (const void *)item->mask;
8569         if (!queue_m)
8570                 return;
8571         queue_v = (const void *)item->spec;
8572         if (!queue_v)
8573                 return;
8574         txq = mlx5_txq_get(dev, queue_v->queue);
8575         if (!txq)
8576                 return;
8577         queue = txq->obj->sq->id;
8578         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
8579         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
8580                  queue & queue_m->queue);
8581         mlx5_txq_release(dev, queue_v->queue);
8582 }
8583
8584 /**
8585  * Set the hash fields according to the @p flow information.
8586  *
8587  * @param[in] dev_flow
8588  *   Pointer to the mlx5_flow.
8589  * @param[in] rss_desc
8590  *   Pointer to the mlx5_flow_rss_desc.
8591  */
8592 static void
8593 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
8594                        struct mlx5_flow_rss_desc *rss_desc)
8595 {
8596         uint64_t items = dev_flow->handle->layers;
8597         int rss_inner = 0;
8598         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
8599
8600         dev_flow->hash_fields = 0;
8601 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
8602         if (rss_desc->level >= 2) {
8603                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
8604                 rss_inner = 1;
8605         }
8606 #endif
8607         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
8608             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
8609                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
8610                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
8611                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
8612                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
8613                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
8614                         else
8615                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
8616                 }
8617         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
8618                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
8619                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
8620                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
8621                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
8622                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
8623                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
8624                         else
8625                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
8626                 }
8627         }
8628         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
8629             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
8630                 if (rss_types & ETH_RSS_UDP) {
8631                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
8632                                 dev_flow->hash_fields |=
8633                                                 IBV_RX_HASH_SRC_PORT_UDP;
8634                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
8635                                 dev_flow->hash_fields |=
8636                                                 IBV_RX_HASH_DST_PORT_UDP;
8637                         else
8638                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
8639                 }
8640         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
8641                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
8642                 if (rss_types & ETH_RSS_TCP) {
8643                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
8644                                 dev_flow->hash_fields |=
8645                                                 IBV_RX_HASH_SRC_PORT_TCP;
8646                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
8647                                 dev_flow->hash_fields |=
8648                                                 IBV_RX_HASH_DST_PORT_TCP;
8649                         else
8650                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
8651                 }
8652         }
8653 }
8654
8655 /**
8656  * Create an Rx Hash queue.
8657  *
8658  * @param dev
8659  *   Pointer to Ethernet device.
8660  * @param[in] dev_flow
8661  *   Pointer to the mlx5_flow.
8662  * @param[in] rss_desc
8663  *   Pointer to the mlx5_flow_rss_desc.
8664  * @param[out] hrxq_idx
8665  *   Hash Rx queue index.
8666  *
8667  * @return
8668  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
8669  */
8670 static struct mlx5_hrxq *
8671 flow_dv_handle_rx_queue(struct rte_eth_dev *dev,
8672                         struct mlx5_flow *dev_flow,
8673                         struct mlx5_flow_rss_desc *rss_desc,
8674                         uint32_t *hrxq_idx)
8675 {
8676         struct mlx5_priv *priv = dev->data->dev_private;
8677         struct mlx5_flow_handle *dh = dev_flow->handle;
8678         struct mlx5_hrxq *hrxq;
8679
8680         MLX5_ASSERT(rss_desc->queue_num);
8681         *hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
8682                                   MLX5_RSS_HASH_KEY_LEN,
8683                                   dev_flow->hash_fields,
8684                                   rss_desc->queue,
8685                                   rss_desc->queue_num);
8686         if (!*hrxq_idx) {
8687                 *hrxq_idx = mlx5_hrxq_new
8688                                 (dev, rss_desc->key,
8689                                  MLX5_RSS_HASH_KEY_LEN,
8690                                  dev_flow->hash_fields,
8691                                  rss_desc->queue,
8692                                  rss_desc->queue_num,
8693                                  !!(dh->layers &
8694                                  MLX5_FLOW_LAYER_TUNNEL));
8695                 if (!*hrxq_idx)
8696                         return NULL;
8697         }
8698         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
8699                               *hrxq_idx);
8700         return hrxq;
8701 }
8702
8703 /**
8704  * Find existing sample resource or create and register a new one.
8705  *
8706  * @param[in, out] dev
8707  *   Pointer to rte_eth_dev structure.
8708  * @param[in] attr
8709  *   Attributes of flow that includes this item.
8710  * @param[in] resource
8711  *   Pointer to sample resource.
8712  * @parm[in, out] dev_flow
8713  *   Pointer to the dev_flow.
8714  * @param[in, out] sample_dv_actions
8715  *   Pointer to sample actions list.
8716  * @param[out] error
8717  *   pointer to error structure.
8718  *
8719  * @return
8720  *   0 on success otherwise -errno and errno is set.
8721  */
8722 static int
8723 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
8724                          const struct rte_flow_attr *attr,
8725                          struct mlx5_flow_dv_sample_resource *resource,
8726                          struct mlx5_flow *dev_flow,
8727                          void **sample_dv_actions,
8728                          struct rte_flow_error *error)
8729 {
8730         struct mlx5_flow_dv_sample_resource *cache_resource;
8731         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
8732         struct mlx5_priv *priv = dev->data->dev_private;
8733         struct mlx5_dev_ctx_shared *sh = priv->sh;
8734         struct mlx5_flow_tbl_resource *tbl;
8735         uint32_t idx = 0;
8736         const uint32_t next_ft_step = 1;
8737         uint32_t next_ft_id = resource->ft_id + next_ft_step;
8738
8739         /* Lookup a matching resource from cache. */
8740         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_SAMPLE], sh->sample_action_list,
8741                       idx, cache_resource, next) {
8742                 if (resource->ratio == cache_resource->ratio &&
8743                     resource->ft_type == cache_resource->ft_type &&
8744                     resource->ft_id == cache_resource->ft_id &&
8745                     resource->set_action == cache_resource->set_action &&
8746                     !memcmp((void *)&resource->sample_act,
8747                             (void *)&cache_resource->sample_act,
8748                             sizeof(struct mlx5_flow_sub_actions_list))) {
8749                         DRV_LOG(DEBUG, "sample resource %p: refcnt %d++",
8750                                 (void *)cache_resource,
8751                                 __atomic_load_n(&cache_resource->refcnt,
8752                                                 __ATOMIC_RELAXED));
8753                         __atomic_fetch_add(&cache_resource->refcnt, 1,
8754                                            __ATOMIC_RELAXED);
8755                         dev_flow->handle->dvh.rix_sample = idx;
8756                         dev_flow->dv.sample_res = cache_resource;
8757                         return 0;
8758                 }
8759         }
8760         /* Register new sample resource. */
8761         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE],
8762                                        &dev_flow->handle->dvh.rix_sample);
8763         if (!cache_resource)
8764                 return rte_flow_error_set(error, ENOMEM,
8765                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8766                                           NULL,
8767                                           "cannot allocate resource memory");
8768         *cache_resource = *resource;
8769         /* Create normal path table level */
8770         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
8771                                         attr->egress, attr->transfer, error);
8772         if (!tbl) {
8773                 rte_flow_error_set(error, ENOMEM,
8774                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8775                                           NULL,
8776                                           "fail to create normal path table "
8777                                           "for sample");
8778                 goto error;
8779         }
8780         cache_resource->normal_path_tbl = tbl;
8781         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
8782                 cache_resource->default_miss =
8783                                 mlx5_glue->dr_create_flow_action_default_miss();
8784                 if (!cache_resource->default_miss) {
8785                         rte_flow_error_set(error, ENOMEM,
8786                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8787                                                 NULL,
8788                                                 "cannot create default miss "
8789                                                 "action");
8790                         goto error;
8791                 }
8792                 sample_dv_actions[resource->sample_act.actions_num++] =
8793                                                 cache_resource->default_miss;
8794         }
8795         /* Create a DR sample action */
8796         sampler_attr.sample_ratio = cache_resource->ratio;
8797         sampler_attr.default_next_table = tbl->obj;
8798         sampler_attr.num_sample_actions = resource->sample_act.actions_num;
8799         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
8800                                                         &sample_dv_actions[0];
8801         sampler_attr.action = cache_resource->set_action;
8802         cache_resource->verbs_action =
8803                 mlx5_glue->dr_create_flow_action_sampler(&sampler_attr);
8804         if (!cache_resource->verbs_action) {
8805                 rte_flow_error_set(error, ENOMEM,
8806                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8807                                         NULL, "cannot create sample action");
8808                 goto error;
8809         }
8810         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8811         ILIST_INSERT(sh->ipool[MLX5_IPOOL_SAMPLE], &sh->sample_action_list,
8812                      dev_flow->handle->dvh.rix_sample, cache_resource,
8813                      next);
8814         dev_flow->dv.sample_res = cache_resource;
8815         DRV_LOG(DEBUG, "new sample resource %p: refcnt %d++",
8816                 (void *)cache_resource,
8817                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8818         return 0;
8819 error:
8820         if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
8821                 if (cache_resource->default_miss)
8822                         claim_zero(mlx5_glue->destroy_flow_action
8823                                 (cache_resource->default_miss));
8824         } else {
8825                 if (cache_resource->sample_idx.rix_hrxq &&
8826                     !mlx5_hrxq_release(dev,
8827                                 cache_resource->sample_idx.rix_hrxq))
8828                         cache_resource->sample_idx.rix_hrxq = 0;
8829                 if (cache_resource->sample_idx.rix_tag &&
8830                     !flow_dv_tag_release(dev,
8831                                 cache_resource->sample_idx.rix_tag))
8832                         cache_resource->sample_idx.rix_tag = 0;
8833                 if (cache_resource->sample_idx.cnt) {
8834                         flow_dv_counter_release(dev,
8835                                 cache_resource->sample_idx.cnt);
8836                         cache_resource->sample_idx.cnt = 0;
8837                 }
8838         }
8839         if (cache_resource->normal_path_tbl)
8840                 flow_dv_tbl_resource_release(dev,
8841                                 cache_resource->normal_path_tbl);
8842         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE],
8843                                 dev_flow->handle->dvh.rix_sample);
8844         dev_flow->handle->dvh.rix_sample = 0;
8845         return -rte_errno;
8846 }
8847
8848 /**
8849  * Find existing destination array resource or create and register a new one.
8850  *
8851  * @param[in, out] dev
8852  *   Pointer to rte_eth_dev structure.
8853  * @param[in] attr
8854  *   Attributes of flow that includes this item.
8855  * @param[in] resource
8856  *   Pointer to destination array resource.
8857  * @parm[in, out] dev_flow
8858  *   Pointer to the dev_flow.
8859  * @param[out] error
8860  *   pointer to error structure.
8861  *
8862  * @return
8863  *   0 on success otherwise -errno and errno is set.
8864  */
8865 static int
8866 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
8867                          const struct rte_flow_attr *attr,
8868                          struct mlx5_flow_dv_dest_array_resource *resource,
8869                          struct mlx5_flow *dev_flow,
8870                          struct rte_flow_error *error)
8871 {
8872         struct mlx5_flow_dv_dest_array_resource *cache_resource;
8873         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
8874         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
8875         struct mlx5_priv *priv = dev->data->dev_private;
8876         struct mlx5_dev_ctx_shared *sh = priv->sh;
8877         struct mlx5_flow_sub_actions_list *sample_act;
8878         struct mlx5dv_dr_domain *domain;
8879         uint32_t idx = 0;
8880
8881         /* Lookup a matching resource from cache. */
8882         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8883                       sh->dest_array_list,
8884                       idx, cache_resource, next) {
8885                 if (resource->num_of_dest == cache_resource->num_of_dest &&
8886                     resource->ft_type == cache_resource->ft_type &&
8887                     !memcmp((void *)cache_resource->sample_act,
8888                             (void *)resource->sample_act,
8889                            (resource->num_of_dest *
8890                            sizeof(struct mlx5_flow_sub_actions_list)))) {
8891                         DRV_LOG(DEBUG, "dest array resource %p: refcnt %d++",
8892                                 (void *)cache_resource,
8893                                 __atomic_load_n(&cache_resource->refcnt,
8894                                                 __ATOMIC_RELAXED));
8895                         __atomic_fetch_add(&cache_resource->refcnt, 1,
8896                                            __ATOMIC_RELAXED);
8897                         dev_flow->handle->dvh.rix_dest_array = idx;
8898                         dev_flow->dv.dest_array_res = cache_resource;
8899                         return 0;
8900                 }
8901         }
8902         /* Register new destination array resource. */
8903         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8904                                        &dev_flow->handle->dvh.rix_dest_array);
8905         if (!cache_resource)
8906                 return rte_flow_error_set(error, ENOMEM,
8907                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8908                                           NULL,
8909                                           "cannot allocate resource memory");
8910         *cache_resource = *resource;
8911         if (attr->transfer)
8912                 domain = sh->fdb_domain;
8913         else if (attr->ingress)
8914                 domain = sh->rx_domain;
8915         else
8916                 domain = sh->tx_domain;
8917         for (idx = 0; idx < resource->num_of_dest; idx++) {
8918                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
8919                                  mlx5_malloc(MLX5_MEM_ZERO,
8920                                  sizeof(struct mlx5dv_dr_action_dest_attr),
8921                                  0, SOCKET_ID_ANY);
8922                 if (!dest_attr[idx]) {
8923                         rte_flow_error_set(error, ENOMEM,
8924                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8925                                            NULL,
8926                                            "cannot allocate resource memory");
8927                         goto error;
8928                 }
8929                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
8930                 sample_act = &resource->sample_act[idx];
8931                 if (sample_act->action_flags == MLX5_FLOW_ACTION_QUEUE) {
8932                         dest_attr[idx]->dest = sample_act->dr_queue_action;
8933                 } else if (sample_act->action_flags ==
8934                           (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP)) {
8935                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
8936                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
8937                         dest_attr[idx]->dest_reformat->reformat =
8938                                         sample_act->dr_encap_action;
8939                         dest_attr[idx]->dest_reformat->dest =
8940                                         sample_act->dr_port_id_action;
8941                 } else if (sample_act->action_flags ==
8942                            MLX5_FLOW_ACTION_PORT_ID) {
8943                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
8944                 }
8945         }
8946         /* create a dest array actioin */
8947         cache_resource->action = mlx5_glue->dr_create_flow_action_dest_array
8948                                                 (domain,
8949                                                  cache_resource->num_of_dest,
8950                                                  dest_attr);
8951         if (!cache_resource->action) {
8952                 rte_flow_error_set(error, ENOMEM,
8953                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8954                                    NULL,
8955                                    "cannot create destination array action");
8956                 goto error;
8957         }
8958         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8959         ILIST_INSERT(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8960                      &sh->dest_array_list,
8961                      dev_flow->handle->dvh.rix_dest_array, cache_resource,
8962                      next);
8963         dev_flow->dv.dest_array_res = cache_resource;
8964         DRV_LOG(DEBUG, "new destination array resource %p: refcnt %d++",
8965                 (void *)cache_resource,
8966                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8967         for (idx = 0; idx < resource->num_of_dest; idx++)
8968                 mlx5_free(dest_attr[idx]);
8969         return 0;
8970 error:
8971         for (idx = 0; idx < resource->num_of_dest; idx++) {
8972                 struct mlx5_flow_sub_actions_idx *act_res =
8973                                         &cache_resource->sample_idx[idx];
8974                 if (act_res->rix_hrxq &&
8975                     !mlx5_hrxq_release(dev,
8976                                 act_res->rix_hrxq))
8977                         act_res->rix_hrxq = 0;
8978                 if (act_res->rix_encap_decap &&
8979                         !flow_dv_encap_decap_resource_release(dev,
8980                                 act_res->rix_encap_decap))
8981                         act_res->rix_encap_decap = 0;
8982                 if (act_res->rix_port_id_action &&
8983                         !flow_dv_port_id_action_resource_release(dev,
8984                                 act_res->rix_port_id_action))
8985                         act_res->rix_port_id_action = 0;
8986                 if (dest_attr[idx])
8987                         mlx5_free(dest_attr[idx]);
8988         }
8989
8990         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8991                                 dev_flow->handle->dvh.rix_dest_array);
8992         dev_flow->handle->dvh.rix_dest_array = 0;
8993         return -rte_errno;
8994 }
8995
8996 /**
8997  * Convert Sample action to DV specification.
8998  *
8999  * @param[in] dev
9000  *   Pointer to rte_eth_dev structure.
9001  * @param[in] action
9002  *   Pointer to action structure.
9003  * @param[in, out] dev_flow
9004  *   Pointer to the mlx5_flow.
9005  * @param[in] attr
9006  *   Pointer to the flow attributes.
9007  * @param[in, out] num_of_dest
9008  *   Pointer to the num of destination.
9009  * @param[in, out] sample_actions
9010  *   Pointer to sample actions list.
9011  * @param[in, out] res
9012  *   Pointer to sample resource.
9013  * @param[out] error
9014  *   Pointer to the error structure.
9015  *
9016  * @return
9017  *   0 on success, a negative errno value otherwise and rte_errno is set.
9018  */
9019 static int
9020 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
9021                                 const struct rte_flow_action *action,
9022                                 struct mlx5_flow *dev_flow,
9023                                 const struct rte_flow_attr *attr,
9024                                 uint32_t *num_of_dest,
9025                                 void **sample_actions,
9026                                 struct mlx5_flow_dv_sample_resource *res,
9027                                 struct rte_flow_error *error)
9028 {
9029         struct mlx5_priv *priv = dev->data->dev_private;
9030         const struct rte_flow_action_sample *sample_action;
9031         const struct rte_flow_action *sub_actions;
9032         const struct rte_flow_action_queue *queue;
9033         struct mlx5_flow_sub_actions_list *sample_act;
9034         struct mlx5_flow_sub_actions_idx *sample_idx;
9035         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
9036                                               priv->rss_desc)
9037                                               [!!priv->flow_nested_idx];
9038         uint64_t action_flags = 0;
9039
9040         sample_act = &res->sample_act;
9041         sample_idx = &res->sample_idx;
9042         sample_action = (const struct rte_flow_action_sample *)action->conf;
9043         res->ratio = sample_action->ratio;
9044         sub_actions = sample_action->actions;
9045         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
9046                 int type = sub_actions->type;
9047                 uint32_t pre_rix = 0;
9048                 void *pre_r;
9049                 switch (type) {
9050                 case RTE_FLOW_ACTION_TYPE_QUEUE:
9051                 {
9052                         struct mlx5_hrxq *hrxq;
9053                         uint32_t hrxq_idx;
9054
9055                         queue = sub_actions->conf;
9056                         rss_desc->queue_num = 1;
9057                         rss_desc->queue[0] = queue->index;
9058                         hrxq = flow_dv_handle_rx_queue(dev, dev_flow,
9059                                         rss_desc, &hrxq_idx);
9060                         if (!hrxq)
9061                                 return rte_flow_error_set
9062                                         (error, rte_errno,
9063                                          RTE_FLOW_ERROR_TYPE_ACTION,
9064                                          NULL,
9065                                          "cannot create fate queue");
9066                         sample_act->dr_queue_action = hrxq->action;
9067                         sample_idx->rix_hrxq = hrxq_idx;
9068                         sample_actions[sample_act->actions_num++] =
9069                                                 hrxq->action;
9070                         (*num_of_dest)++;
9071                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
9072                         if (action_flags & MLX5_FLOW_ACTION_MARK)
9073                                 dev_flow->handle->rix_hrxq = hrxq_idx;
9074                         dev_flow->handle->fate_action =
9075                                         MLX5_FLOW_FATE_QUEUE;
9076                         break;
9077                 }
9078                 case RTE_FLOW_ACTION_TYPE_MARK:
9079                 {
9080                         uint32_t tag_be = mlx5_flow_mark_set
9081                                 (((const struct rte_flow_action_mark *)
9082                                 (sub_actions->conf))->id);
9083
9084                         dev_flow->handle->mark = 1;
9085                         pre_rix = dev_flow->handle->dvh.rix_tag;
9086                         /* Save the mark resource before sample */
9087                         pre_r = dev_flow->dv.tag_resource;
9088                         if (flow_dv_tag_resource_register(dev, tag_be,
9089                                                   dev_flow, error))
9090                                 return -rte_errno;
9091                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9092                         sample_act->dr_tag_action =
9093                                 dev_flow->dv.tag_resource->action;
9094                         sample_idx->rix_tag =
9095                                 dev_flow->handle->dvh.rix_tag;
9096                         sample_actions[sample_act->actions_num++] =
9097                                                 sample_act->dr_tag_action;
9098                         /* Recover the mark resource after sample */
9099                         dev_flow->dv.tag_resource = pre_r;
9100                         dev_flow->handle->dvh.rix_tag = pre_rix;
9101                         action_flags |= MLX5_FLOW_ACTION_MARK;
9102                         break;
9103                 }
9104                 case RTE_FLOW_ACTION_TYPE_COUNT:
9105                 {
9106                         uint32_t counter;
9107
9108                         counter = flow_dv_translate_create_counter(dev,
9109                                         dev_flow, sub_actions->conf, 0);
9110                         if (!counter)
9111                                 return rte_flow_error_set
9112                                                 (error, rte_errno,
9113                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9114                                                  NULL,
9115                                                  "cannot create counter"
9116                                                  " object.");
9117                         sample_idx->cnt = counter;
9118                         sample_act->dr_cnt_action =
9119                                   (flow_dv_counter_get_by_idx(dev,
9120                                   counter, NULL))->action;
9121                         sample_actions[sample_act->actions_num++] =
9122                                                 sample_act->dr_cnt_action;
9123                         action_flags |= MLX5_FLOW_ACTION_COUNT;
9124                         break;
9125                 }
9126                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
9127                 {
9128                         struct mlx5_flow_dv_port_id_action_resource
9129                                         port_id_resource;
9130                         uint32_t port_id = 0;
9131
9132                         memset(&port_id_resource, 0, sizeof(port_id_resource));
9133                         /* Save the port id resource before sample */
9134                         pre_rix = dev_flow->handle->rix_port_id_action;
9135                         pre_r = dev_flow->dv.port_id_action;
9136                         if (flow_dv_translate_action_port_id(dev, sub_actions,
9137                                                              &port_id, error))
9138                                 return -rte_errno;
9139                         port_id_resource.port_id = port_id;
9140                         if (flow_dv_port_id_action_resource_register
9141                             (dev, &port_id_resource, dev_flow, error))
9142                                 return -rte_errno;
9143                         sample_act->dr_port_id_action =
9144                                 dev_flow->dv.port_id_action->action;
9145                         sample_idx->rix_port_id_action =
9146                                 dev_flow->handle->rix_port_id_action;
9147                         sample_actions[sample_act->actions_num++] =
9148                                                 sample_act->dr_port_id_action;
9149                         /* Recover the port id resource after sample */
9150                         dev_flow->dv.port_id_action = pre_r;
9151                         dev_flow->handle->rix_port_id_action = pre_rix;
9152                         (*num_of_dest)++;
9153                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9154                         break;
9155                 }
9156                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
9157                         /* Save the encap resource before sample */
9158                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
9159                         pre_r = dev_flow->dv.encap_decap;
9160                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
9161                                                            dev_flow,
9162                                                            attr->transfer,
9163                                                            error))
9164                                 return -rte_errno;
9165                         sample_act->dr_encap_action =
9166                                 dev_flow->dv.encap_decap->action;
9167                         sample_idx->rix_encap_decap =
9168                                 dev_flow->handle->dvh.rix_encap_decap;
9169                         sample_actions[sample_act->actions_num++] =
9170                                                 sample_act->dr_encap_action;
9171                         /* Recover the encap resource after sample */
9172                         dev_flow->dv.encap_decap = pre_r;
9173                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
9174                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9175                         break;
9176                 default:
9177                         return rte_flow_error_set(error, EINVAL,
9178                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9179                                 NULL,
9180                                 "Not support for sampler action");
9181                 }
9182         }
9183         sample_act->action_flags = action_flags;
9184         res->ft_id = dev_flow->dv.group;
9185         if (attr->transfer) {
9186                 union {
9187                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
9188                         uint64_t set_action;
9189                 } action_ctx = { .set_action = 0 };
9190
9191                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
9192                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
9193                          MLX5_MODIFICATION_TYPE_SET);
9194                 MLX5_SET(set_action_in, action_ctx.action_in, field,
9195                          MLX5_MODI_META_REG_C_0);
9196                 MLX5_SET(set_action_in, action_ctx.action_in, data,
9197                          priv->vport_meta_tag);
9198                 res->set_action = action_ctx.set_action;
9199         } else if (attr->ingress) {
9200                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
9201         }
9202         return 0;
9203 }
9204
9205 /**
9206  * Convert Sample action to DV specification.
9207  *
9208  * @param[in] dev
9209  *   Pointer to rte_eth_dev structure.
9210  * @param[in, out] dev_flow
9211  *   Pointer to the mlx5_flow.
9212  * @param[in] attr
9213  *   Pointer to the flow attributes.
9214  * @param[in] num_of_dest
9215  *   The num of destination.
9216  * @param[in, out] res
9217  *   Pointer to sample resource.
9218  * @param[in, out] mdest_res
9219  *   Pointer to destination array resource.
9220  * @param[in] sample_actions
9221  *   Pointer to sample path actions list.
9222  * @param[in] action_flags
9223  *   Holds the actions detected until now.
9224  * @param[out] error
9225  *   Pointer to the error structure.
9226  *
9227  * @return
9228  *   0 on success, a negative errno value otherwise and rte_errno is set.
9229  */
9230 static int
9231 flow_dv_create_action_sample(struct rte_eth_dev *dev,
9232                              struct mlx5_flow *dev_flow,
9233                              const struct rte_flow_attr *attr,
9234                              uint32_t num_of_dest,
9235                              struct mlx5_flow_dv_sample_resource *res,
9236                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
9237                              void **sample_actions,
9238                              uint64_t action_flags,
9239                              struct rte_flow_error *error)
9240 {
9241         struct mlx5_priv *priv = dev->data->dev_private;
9242         /* update normal path action resource into last index of array */
9243         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
9244         struct mlx5_flow_sub_actions_list *sample_act =
9245                                         &mdest_res->sample_act[dest_index];
9246         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
9247                                               priv->rss_desc)
9248                                               [!!priv->flow_nested_idx];
9249         uint32_t normal_idx = 0;
9250         struct mlx5_hrxq *hrxq;
9251         uint32_t hrxq_idx;
9252
9253         if (num_of_dest > 1) {
9254                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
9255                         /* Handle QP action for mirroring */
9256                         hrxq = flow_dv_handle_rx_queue(dev, dev_flow,
9257                                                        rss_desc, &hrxq_idx);
9258                         if (!hrxq)
9259                                 return rte_flow_error_set
9260                                      (error, rte_errno,
9261                                       RTE_FLOW_ERROR_TYPE_ACTION,
9262                                       NULL,
9263                                       "cannot create rx queue");
9264                         normal_idx++;
9265                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
9266                         sample_act->dr_queue_action = hrxq->action;
9267                         if (action_flags & MLX5_FLOW_ACTION_MARK)
9268                                 dev_flow->handle->rix_hrxq = hrxq_idx;
9269                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9270                 }
9271                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
9272                         normal_idx++;
9273                         mdest_res->sample_idx[dest_index].rix_encap_decap =
9274                                 dev_flow->handle->dvh.rix_encap_decap;
9275                         sample_act->dr_encap_action =
9276                                 dev_flow->dv.encap_decap->action;
9277                 }
9278                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
9279                         normal_idx++;
9280                         mdest_res->sample_idx[dest_index].rix_port_id_action =
9281                                 dev_flow->handle->rix_port_id_action;
9282                         sample_act->dr_port_id_action =
9283                                 dev_flow->dv.port_id_action->action;
9284                 }
9285                 sample_act->actions_num = normal_idx;
9286                 /* update sample action resource into first index of array */
9287                 mdest_res->ft_type = res->ft_type;
9288                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
9289                                 sizeof(struct mlx5_flow_sub_actions_idx));
9290                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
9291                                 sizeof(struct mlx5_flow_sub_actions_list));
9292                 mdest_res->num_of_dest = num_of_dest;
9293                 if (flow_dv_dest_array_resource_register(dev, attr, mdest_res,
9294                                                          dev_flow, error))
9295                         return rte_flow_error_set(error, EINVAL,
9296                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9297                                                   NULL, "can't create sample "
9298                                                   "action");
9299         } else {
9300                 if (flow_dv_sample_resource_register(dev, attr, res, dev_flow,
9301                                                      sample_actions, error))
9302                         return rte_flow_error_set(error, EINVAL,
9303                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9304                                                   NULL,
9305                                                   "can't create sample action");
9306         }
9307         return 0;
9308 }
9309
9310 /**
9311  * Fill the flow with DV spec, lock free
9312  * (mutex should be acquired by caller).
9313  *
9314  * @param[in] dev
9315  *   Pointer to rte_eth_dev structure.
9316  * @param[in, out] dev_flow
9317  *   Pointer to the sub flow.
9318  * @param[in] attr
9319  *   Pointer to the flow attributes.
9320  * @param[in] items
9321  *   Pointer to the list of items.
9322  * @param[in] actions
9323  *   Pointer to the list of actions.
9324  * @param[out] error
9325  *   Pointer to the error structure.
9326  *
9327  * @return
9328  *   0 on success, a negative errno value otherwise and rte_errno is set.
9329  */
9330 static int
9331 __flow_dv_translate(struct rte_eth_dev *dev,
9332                     struct mlx5_flow *dev_flow,
9333                     const struct rte_flow_attr *attr,
9334                     const struct rte_flow_item items[],
9335                     const struct rte_flow_action actions[],
9336                     struct rte_flow_error *error)
9337 {
9338         struct mlx5_priv *priv = dev->data->dev_private;
9339         struct mlx5_dev_config *dev_conf = &priv->config;
9340         struct rte_flow *flow = dev_flow->flow;
9341         struct mlx5_flow_handle *handle = dev_flow->handle;
9342         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
9343                                               priv->rss_desc)
9344                                               [!!priv->flow_nested_idx];
9345         uint64_t item_flags = 0;
9346         uint64_t last_item = 0;
9347         uint64_t action_flags = 0;
9348         uint64_t priority = attr->priority;
9349         struct mlx5_flow_dv_matcher matcher = {
9350                 .mask = {
9351                         .size = sizeof(matcher.mask.buf) -
9352                                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
9353                 },
9354         };
9355         int actions_n = 0;
9356         bool actions_end = false;
9357         union {
9358                 struct mlx5_flow_dv_modify_hdr_resource res;
9359                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
9360                             sizeof(struct mlx5_modification_cmd) *
9361                             (MLX5_MAX_MODIFY_NUM + 1)];
9362         } mhdr_dummy;
9363         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
9364         const struct rte_flow_action_count *count = NULL;
9365         const struct rte_flow_action_age *age = NULL;
9366         union flow_dv_attr flow_attr = { .attr = 0 };
9367         uint32_t tag_be;
9368         union mlx5_flow_tbl_key tbl_key;
9369         uint32_t modify_action_position = UINT32_MAX;
9370         void *match_mask = matcher.mask.buf;
9371         void *match_value = dev_flow->dv.value.buf;
9372         uint8_t next_protocol = 0xff;
9373         struct rte_vlan_hdr vlan = { 0 };
9374         struct mlx5_flow_dv_dest_array_resource mdest_res;
9375         struct mlx5_flow_dv_sample_resource sample_res;
9376         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
9377         struct mlx5_flow_sub_actions_list *sample_act;
9378         uint32_t sample_act_pos = UINT32_MAX;
9379         uint32_t num_of_dest = 0;
9380         int tmp_actions_n = 0;
9381         uint32_t table;
9382         int ret = 0;
9383
9384         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
9385         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
9386         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
9387                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
9388         /* update normal path action resource into last index of array */
9389         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
9390         ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
9391                                        !!priv->fdb_def_rule, &table, error);
9392         if (ret)
9393                 return ret;
9394         dev_flow->dv.group = table;
9395         if (attr->transfer)
9396                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
9397         if (priority == MLX5_FLOW_PRIO_RSVD)
9398                 priority = dev_conf->flow_prio - 1;
9399         /* number of actions must be set to 0 in case of dirty stack. */
9400         mhdr_res->actions_num = 0;
9401         for (; !actions_end ; actions++) {
9402                 const struct rte_flow_action_queue *queue;
9403                 const struct rte_flow_action_rss *rss;
9404                 const struct rte_flow_action *action = actions;
9405                 const uint8_t *rss_key;
9406                 const struct rte_flow_action_meter *mtr;
9407                 struct mlx5_flow_tbl_resource *tbl;
9408                 uint32_t port_id = 0;
9409                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
9410                 int action_type = actions->type;
9411                 const struct rte_flow_action *found_action = NULL;
9412                 struct mlx5_flow_meter *fm = NULL;
9413                 uint32_t jump_group = 0;
9414
9415                 if (!mlx5_flow_os_action_supported(action_type))
9416                         return rte_flow_error_set(error, ENOTSUP,
9417                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9418                                                   actions,
9419                                                   "action not supported");
9420                 switch (action_type) {
9421                 case RTE_FLOW_ACTION_TYPE_VOID:
9422                         break;
9423                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
9424                         if (flow_dv_translate_action_port_id(dev, action,
9425                                                              &port_id, error))
9426                                 return -rte_errno;
9427                         port_id_resource.port_id = port_id;
9428                         MLX5_ASSERT(!handle->rix_port_id_action);
9429                         if (flow_dv_port_id_action_resource_register
9430                             (dev, &port_id_resource, dev_flow, error))
9431                                 return -rte_errno;
9432                         dev_flow->dv.actions[actions_n++] =
9433                                         dev_flow->dv.port_id_action->action;
9434                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9435                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
9436                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9437                         num_of_dest++;
9438                         break;
9439                 case RTE_FLOW_ACTION_TYPE_FLAG:
9440                         action_flags |= MLX5_FLOW_ACTION_FLAG;
9441                         dev_flow->handle->mark = 1;
9442                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
9443                                 struct rte_flow_action_mark mark = {
9444                                         .id = MLX5_FLOW_MARK_DEFAULT,
9445                                 };
9446
9447                                 if (flow_dv_convert_action_mark(dev, &mark,
9448                                                                 mhdr_res,
9449                                                                 error))
9450                                         return -rte_errno;
9451                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
9452                                 break;
9453                         }
9454                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
9455                         /*
9456                          * Only one FLAG or MARK is supported per device flow
9457                          * right now. So the pointer to the tag resource must be
9458                          * zero before the register process.
9459                          */
9460                         MLX5_ASSERT(!handle->dvh.rix_tag);
9461                         if (flow_dv_tag_resource_register(dev, tag_be,
9462                                                           dev_flow, error))
9463                                 return -rte_errno;
9464                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9465                         dev_flow->dv.actions[actions_n++] =
9466                                         dev_flow->dv.tag_resource->action;
9467                         break;
9468                 case RTE_FLOW_ACTION_TYPE_MARK:
9469                         action_flags |= MLX5_FLOW_ACTION_MARK;
9470                         dev_flow->handle->mark = 1;
9471                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
9472                                 const struct rte_flow_action_mark *mark =
9473                                         (const struct rte_flow_action_mark *)
9474                                                 actions->conf;
9475
9476                                 if (flow_dv_convert_action_mark(dev, mark,
9477                                                                 mhdr_res,
9478                                                                 error))
9479                                         return -rte_errno;
9480                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
9481                                 break;
9482                         }
9483                         /* Fall-through */
9484                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
9485                         /* Legacy (non-extensive) MARK action. */
9486                         tag_be = mlx5_flow_mark_set
9487                               (((const struct rte_flow_action_mark *)
9488                                (actions->conf))->id);
9489                         MLX5_ASSERT(!handle->dvh.rix_tag);
9490                         if (flow_dv_tag_resource_register(dev, tag_be,
9491                                                           dev_flow, error))
9492                                 return -rte_errno;
9493                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9494                         dev_flow->dv.actions[actions_n++] =
9495                                         dev_flow->dv.tag_resource->action;
9496                         break;
9497                 case RTE_FLOW_ACTION_TYPE_SET_META:
9498                         if (flow_dv_convert_action_set_meta
9499                                 (dev, mhdr_res, attr,
9500                                  (const struct rte_flow_action_set_meta *)
9501                                   actions->conf, error))
9502                                 return -rte_errno;
9503                         action_flags |= MLX5_FLOW_ACTION_SET_META;
9504                         break;
9505                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
9506                         if (flow_dv_convert_action_set_tag
9507                                 (dev, mhdr_res,
9508                                  (const struct rte_flow_action_set_tag *)
9509                                   actions->conf, error))
9510                                 return -rte_errno;
9511                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9512                         break;
9513                 case RTE_FLOW_ACTION_TYPE_DROP:
9514                         action_flags |= MLX5_FLOW_ACTION_DROP;
9515                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
9516                         break;
9517                 case RTE_FLOW_ACTION_TYPE_QUEUE:
9518                         queue = actions->conf;
9519                         rss_desc->queue_num = 1;
9520                         rss_desc->queue[0] = queue->index;
9521                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
9522                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9523                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
9524                         num_of_dest++;
9525                         break;
9526                 case RTE_FLOW_ACTION_TYPE_RSS:
9527                         rss = actions->conf;
9528                         memcpy(rss_desc->queue, rss->queue,
9529                                rss->queue_num * sizeof(uint16_t));
9530                         rss_desc->queue_num = rss->queue_num;
9531                         /* NULL RSS key indicates default RSS key. */
9532                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
9533                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
9534                         /*
9535                          * rss->level and rss.types should be set in advance
9536                          * when expanding items for RSS.
9537                          */
9538                         action_flags |= MLX5_FLOW_ACTION_RSS;
9539                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9540                         break;
9541                 case RTE_FLOW_ACTION_TYPE_AGE:
9542                 case RTE_FLOW_ACTION_TYPE_COUNT:
9543                         if (!dev_conf->devx) {
9544                                 return rte_flow_error_set
9545                                               (error, ENOTSUP,
9546                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9547                                                NULL,
9548                                                "count action not supported");
9549                         }
9550                         /* Save information first, will apply later. */
9551                         if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT)
9552                                 count = action->conf;
9553                         else
9554                                 age = action->conf;
9555                         action_flags |= MLX5_FLOW_ACTION_COUNT;
9556                         break;
9557                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
9558                         dev_flow->dv.actions[actions_n++] =
9559                                                 priv->sh->pop_vlan_action;
9560                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
9561                         break;
9562                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
9563                         if (!(action_flags &
9564                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
9565                                 flow_dev_get_vlan_info_from_items(items, &vlan);
9566                         vlan.eth_proto = rte_be_to_cpu_16
9567                              ((((const struct rte_flow_action_of_push_vlan *)
9568                                                    actions->conf)->ethertype));
9569                         found_action = mlx5_flow_find_action
9570                                         (actions + 1,
9571                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
9572                         if (found_action)
9573                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
9574                         found_action = mlx5_flow_find_action
9575                                         (actions + 1,
9576                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
9577                         if (found_action)
9578                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
9579                         if (flow_dv_create_action_push_vlan
9580                                             (dev, attr, &vlan, dev_flow, error))
9581                                 return -rte_errno;
9582                         dev_flow->dv.actions[actions_n++] =
9583                                         dev_flow->dv.push_vlan_res->action;
9584                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
9585                         break;
9586                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
9587                         /* of_vlan_push action handled this action */
9588                         MLX5_ASSERT(action_flags &
9589                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
9590                         break;
9591                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
9592                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
9593                                 break;
9594                         flow_dev_get_vlan_info_from_items(items, &vlan);
9595                         mlx5_update_vlan_vid_pcp(actions, &vlan);
9596                         /* If no VLAN push - this is a modify header action */
9597                         if (flow_dv_convert_action_modify_vlan_vid
9598                                                 (mhdr_res, actions, error))
9599                                 return -rte_errno;
9600                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
9601                         break;
9602                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
9603                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
9604                         if (flow_dv_create_action_l2_encap(dev, actions,
9605                                                            dev_flow,
9606                                                            attr->transfer,
9607                                                            error))
9608                                 return -rte_errno;
9609                         dev_flow->dv.actions[actions_n++] =
9610                                         dev_flow->dv.encap_decap->action;
9611                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9612                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
9613                                 sample_act->action_flags |=
9614                                                         MLX5_FLOW_ACTION_ENCAP;
9615                         break;
9616                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
9617                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
9618                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
9619                                                            attr->transfer,
9620                                                            error))
9621                                 return -rte_errno;
9622                         dev_flow->dv.actions[actions_n++] =
9623                                         dev_flow->dv.encap_decap->action;
9624                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9625                         break;
9626                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
9627                         /* Handle encap with preceding decap. */
9628                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
9629                                 if (flow_dv_create_action_raw_encap
9630                                         (dev, actions, dev_flow, attr, error))
9631                                         return -rte_errno;
9632                                 dev_flow->dv.actions[actions_n++] =
9633                                         dev_flow->dv.encap_decap->action;
9634                         } else {
9635                                 /* Handle encap without preceding decap. */
9636                                 if (flow_dv_create_action_l2_encap
9637                                     (dev, actions, dev_flow, attr->transfer,
9638                                      error))
9639                                         return -rte_errno;
9640                                 dev_flow->dv.actions[actions_n++] =
9641                                         dev_flow->dv.encap_decap->action;
9642                         }
9643                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9644                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
9645                                 sample_act->action_flags |=
9646                                                         MLX5_FLOW_ACTION_ENCAP;
9647                         break;
9648                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
9649                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
9650                                 ;
9651                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
9652                                 if (flow_dv_create_action_l2_decap
9653                                     (dev, dev_flow, attr->transfer, error))
9654                                         return -rte_errno;
9655                                 dev_flow->dv.actions[actions_n++] =
9656                                         dev_flow->dv.encap_decap->action;
9657                         }
9658                         /* If decap is followed by encap, handle it at encap. */
9659                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9660                         break;
9661                 case RTE_FLOW_ACTION_TYPE_JUMP:
9662                         jump_group = ((const struct rte_flow_action_jump *)
9663                                                         action->conf)->group;
9664                         if (dev_flow->external && jump_group <
9665                                         MLX5_MAX_TABLES_EXTERNAL)
9666                                 jump_group *= MLX5_FLOW_TABLE_FACTOR;
9667                         ret = mlx5_flow_group_to_table(attr, dev_flow->external,
9668                                                        jump_group,
9669                                                        !!priv->fdb_def_rule,
9670                                                        &table, error);
9671                         if (ret)
9672                                 return ret;
9673                         tbl = flow_dv_tbl_resource_get(dev, table,
9674                                                        attr->egress,
9675                                                        attr->transfer, error);
9676                         if (!tbl)
9677                                 return rte_flow_error_set
9678                                                 (error, errno,
9679                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9680                                                  NULL,
9681                                                  "cannot create jump action.");
9682                         if (flow_dv_jump_tbl_resource_register
9683                             (dev, tbl, dev_flow, error)) {
9684                                 flow_dv_tbl_resource_release(dev, tbl);
9685                                 return rte_flow_error_set
9686                                                 (error, errno,
9687                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9688                                                  NULL,
9689                                                  "cannot create jump action.");
9690                         }
9691                         dev_flow->dv.actions[actions_n++] =
9692                                         dev_flow->dv.jump->action;
9693                         action_flags |= MLX5_FLOW_ACTION_JUMP;
9694                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
9695                         break;
9696                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
9697                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
9698                         if (flow_dv_convert_action_modify_mac
9699                                         (mhdr_res, actions, error))
9700                                 return -rte_errno;
9701                         action_flags |= actions->type ==
9702                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
9703                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
9704                                         MLX5_FLOW_ACTION_SET_MAC_DST;
9705                         break;
9706                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
9707                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
9708                         if (flow_dv_convert_action_modify_ipv4
9709                                         (mhdr_res, actions, error))
9710                                 return -rte_errno;
9711                         action_flags |= actions->type ==
9712                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
9713                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
9714                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
9715                         break;
9716                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
9717                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
9718                         if (flow_dv_convert_action_modify_ipv6
9719                                         (mhdr_res, actions, error))
9720                                 return -rte_errno;
9721                         action_flags |= actions->type ==
9722                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
9723                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
9724                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
9725                         break;
9726                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
9727                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
9728                         if (flow_dv_convert_action_modify_tp
9729                                         (mhdr_res, actions, items,
9730                                          &flow_attr, dev_flow, !!(action_flags &
9731                                          MLX5_FLOW_ACTION_DECAP), error))
9732                                 return -rte_errno;
9733                         action_flags |= actions->type ==
9734                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
9735                                         MLX5_FLOW_ACTION_SET_TP_SRC :
9736                                         MLX5_FLOW_ACTION_SET_TP_DST;
9737                         break;
9738                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
9739                         if (flow_dv_convert_action_modify_dec_ttl
9740                                         (mhdr_res, items, &flow_attr, dev_flow,
9741                                          !!(action_flags &
9742                                          MLX5_FLOW_ACTION_DECAP), error))
9743                                 return -rte_errno;
9744                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
9745                         break;
9746                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
9747                         if (flow_dv_convert_action_modify_ttl
9748                                         (mhdr_res, actions, items, &flow_attr,
9749                                          dev_flow, !!(action_flags &
9750                                          MLX5_FLOW_ACTION_DECAP), error))
9751                                 return -rte_errno;
9752                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
9753                         break;
9754                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
9755                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
9756                         if (flow_dv_convert_action_modify_tcp_seq
9757                                         (mhdr_res, actions, error))
9758                                 return -rte_errno;
9759                         action_flags |= actions->type ==
9760                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
9761                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
9762                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
9763                         break;
9764
9765                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
9766                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
9767                         if (flow_dv_convert_action_modify_tcp_ack
9768                                         (mhdr_res, actions, error))
9769                                 return -rte_errno;
9770                         action_flags |= actions->type ==
9771                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
9772                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
9773                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
9774                         break;
9775                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
9776                         if (flow_dv_convert_action_set_reg
9777                                         (mhdr_res, actions, error))
9778                                 return -rte_errno;
9779                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9780                         break;
9781                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
9782                         if (flow_dv_convert_action_copy_mreg
9783                                         (dev, mhdr_res, actions, error))
9784                                 return -rte_errno;
9785                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9786                         break;
9787                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
9788                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
9789                         dev_flow->handle->fate_action =
9790                                         MLX5_FLOW_FATE_DEFAULT_MISS;
9791                         break;
9792                 case RTE_FLOW_ACTION_TYPE_METER:
9793                         mtr = actions->conf;
9794                         if (!flow->meter) {
9795                                 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
9796                                                             attr, error);
9797                                 if (!fm)
9798                                         return rte_flow_error_set(error,
9799                                                 rte_errno,
9800                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9801                                                 NULL,
9802                                                 "meter not found "
9803                                                 "or invalid parameters");
9804                                 flow->meter = fm->idx;
9805                         }
9806                         /* Set the meter action. */
9807                         if (!fm) {
9808                                 fm = mlx5_ipool_get(priv->sh->ipool
9809                                                 [MLX5_IPOOL_MTR], flow->meter);
9810                                 if (!fm)
9811                                         return rte_flow_error_set(error,
9812                                                 rte_errno,
9813                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9814                                                 NULL,
9815                                                 "meter not found "
9816                                                 "or invalid parameters");
9817                         }
9818                         dev_flow->dv.actions[actions_n++] =
9819                                 fm->mfts->meter_action;
9820                         action_flags |= MLX5_FLOW_ACTION_METER;
9821                         break;
9822                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
9823                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
9824                                                               actions, error))
9825                                 return -rte_errno;
9826                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
9827                         break;
9828                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
9829                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
9830                                                               actions, error))
9831                                 return -rte_errno;
9832                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
9833                         break;
9834                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
9835                         sample_act_pos = actions_n;
9836                         ret = flow_dv_translate_action_sample(dev,
9837                                                               actions,
9838                                                               dev_flow, attr,
9839                                                               &num_of_dest,
9840                                                               sample_actions,
9841                                                               &sample_res,
9842                                                               error);
9843                         if (ret < 0)
9844                                 return ret;
9845                         actions_n++;
9846                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
9847                         /* put encap action into group if work with port id */
9848                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
9849                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
9850                                 sample_act->action_flags |=
9851                                                         MLX5_FLOW_ACTION_ENCAP;
9852                         break;
9853                 case RTE_FLOW_ACTION_TYPE_END:
9854                         actions_end = true;
9855                         if (mhdr_res->actions_num) {
9856                                 /* create modify action if needed. */
9857                                 if (flow_dv_modify_hdr_resource_register
9858                                         (dev, mhdr_res, dev_flow, error))
9859                                         return -rte_errno;
9860                                 dev_flow->dv.actions[modify_action_position] =
9861                                         handle->dvh.modify_hdr->action;
9862                         }
9863                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
9864                                 flow->counter =
9865                                         flow_dv_translate_create_counter(dev,
9866                                                 dev_flow, count, age);
9867
9868                                 if (!flow->counter)
9869                                         return rte_flow_error_set
9870                                                 (error, rte_errno,
9871                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9872                                                 NULL,
9873                                                 "cannot create counter"
9874                                                 " object.");
9875                                 dev_flow->dv.actions[actions_n] =
9876                                           (flow_dv_counter_get_by_idx(dev,
9877                                           flow->counter, NULL))->action;
9878                                 actions_n++;
9879                         }
9880                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
9881                                 ret = flow_dv_create_action_sample(dev,
9882                                                           dev_flow, attr,
9883                                                           num_of_dest,
9884                                                           &sample_res,
9885                                                           &mdest_res,
9886                                                           sample_actions,
9887                                                           action_flags,
9888                                                           error);
9889                                 if (ret < 0)
9890                                         return rte_flow_error_set
9891                                                 (error, rte_errno,
9892                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9893                                                 NULL,
9894                                                 "cannot create sample action");
9895                                 if (num_of_dest > 1) {
9896                                         dev_flow->dv.actions[sample_act_pos] =
9897                                         dev_flow->dv.dest_array_res->action;
9898                                 } else {
9899                                         dev_flow->dv.actions[sample_act_pos] =
9900                                         dev_flow->dv.sample_res->verbs_action;
9901                                 }
9902                         }
9903                         break;
9904                 default:
9905                         break;
9906                 }
9907                 if (mhdr_res->actions_num &&
9908                     modify_action_position == UINT32_MAX)
9909                         modify_action_position = actions_n++;
9910         }
9911         /*
9912          * For multiple destination (sample action with ratio=1), the encap
9913          * action and port id action will be combined into group action.
9914          * So need remove the original these actions in the flow and only
9915          * use the sample action instead of.
9916          */
9917         if (num_of_dest > 1 && sample_act->dr_port_id_action) {
9918                 int i;
9919                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
9920
9921                 for (i = 0; i < actions_n; i++) {
9922                         if ((sample_act->dr_encap_action &&
9923                                 sample_act->dr_encap_action ==
9924                                 dev_flow->dv.actions[i]) ||
9925                                 (sample_act->dr_port_id_action &&
9926                                 sample_act->dr_port_id_action ==
9927                                 dev_flow->dv.actions[i]))
9928                                 continue;
9929                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
9930                 }
9931                 memcpy((void *)dev_flow->dv.actions,
9932                                 (void *)temp_actions,
9933                                 tmp_actions_n * sizeof(void *));
9934                 actions_n = tmp_actions_n;
9935         }
9936         dev_flow->dv.actions_n = actions_n;
9937         dev_flow->act_flags = action_flags;
9938         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
9939                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
9940                 int item_type = items->type;
9941
9942                 if (!mlx5_flow_os_item_supported(item_type))
9943                         return rte_flow_error_set(error, ENOTSUP,
9944                                                   RTE_FLOW_ERROR_TYPE_ITEM,
9945                                                   NULL, "item not supported");
9946                 switch (item_type) {
9947                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
9948                         flow_dv_translate_item_port_id(dev, match_mask,
9949                                                        match_value, items);
9950                         last_item = MLX5_FLOW_ITEM_PORT_ID;
9951                         break;
9952                 case RTE_FLOW_ITEM_TYPE_ETH:
9953                         flow_dv_translate_item_eth(match_mask, match_value,
9954                                                    items, tunnel,
9955                                                    dev_flow->dv.group);
9956                         matcher.priority = action_flags &
9957                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
9958                                         !dev_flow->external ?
9959                                         MLX5_PRIORITY_MAP_L3 :
9960                                         MLX5_PRIORITY_MAP_L2;
9961                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
9962                                              MLX5_FLOW_LAYER_OUTER_L2;
9963                         break;
9964                 case RTE_FLOW_ITEM_TYPE_VLAN:
9965                         flow_dv_translate_item_vlan(dev_flow,
9966                                                     match_mask, match_value,
9967                                                     items, tunnel,
9968                                                     dev_flow->dv.group);
9969                         matcher.priority = MLX5_PRIORITY_MAP_L2;
9970                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
9971                                               MLX5_FLOW_LAYER_INNER_VLAN) :
9972                                              (MLX5_FLOW_LAYER_OUTER_L2 |
9973                                               MLX5_FLOW_LAYER_OUTER_VLAN);
9974                         break;
9975                 case RTE_FLOW_ITEM_TYPE_IPV4:
9976                         mlx5_flow_tunnel_ip_check(items, next_protocol,
9977                                                   &item_flags, &tunnel);
9978                         flow_dv_translate_item_ipv4(match_mask, match_value,
9979                                                     items, item_flags, tunnel,
9980                                                     dev_flow->dv.group);
9981                         matcher.priority = MLX5_PRIORITY_MAP_L3;
9982                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
9983                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
9984                         if (items->mask != NULL &&
9985                             ((const struct rte_flow_item_ipv4 *)
9986                              items->mask)->hdr.next_proto_id) {
9987                                 next_protocol =
9988                                         ((const struct rte_flow_item_ipv4 *)
9989                                          (items->spec))->hdr.next_proto_id;
9990                                 next_protocol &=
9991                                         ((const struct rte_flow_item_ipv4 *)
9992                                          (items->mask))->hdr.next_proto_id;
9993                         } else {
9994                                 /* Reset for inner layer. */
9995                                 next_protocol = 0xff;
9996                         }
9997                         break;
9998                 case RTE_FLOW_ITEM_TYPE_IPV6:
9999                         mlx5_flow_tunnel_ip_check(items, next_protocol,
10000                                                   &item_flags, &tunnel);
10001                         flow_dv_translate_item_ipv6(match_mask, match_value,
10002                                                     items, item_flags, tunnel,
10003                                                     dev_flow->dv.group);
10004                         matcher.priority = MLX5_PRIORITY_MAP_L3;
10005                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
10006                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
10007                         if (items->mask != NULL &&
10008                             ((const struct rte_flow_item_ipv6 *)
10009                              items->mask)->hdr.proto) {
10010                                 next_protocol =
10011                                         ((const struct rte_flow_item_ipv6 *)
10012                                          items->spec)->hdr.proto;
10013                                 next_protocol &=
10014                                         ((const struct rte_flow_item_ipv6 *)
10015                                          items->mask)->hdr.proto;
10016                         } else {
10017                                 /* Reset for inner layer. */
10018                                 next_protocol = 0xff;
10019                         }
10020                         break;
10021                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
10022                         flow_dv_translate_item_ipv6_frag_ext(match_mask,
10023                                                              match_value,
10024                                                              items, tunnel);
10025                         last_item = tunnel ?
10026                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
10027                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
10028                         if (items->mask != NULL &&
10029                             ((const struct rte_flow_item_ipv6_frag_ext *)
10030                              items->mask)->hdr.next_header) {
10031                                 next_protocol =
10032                                 ((const struct rte_flow_item_ipv6_frag_ext *)
10033                                  items->spec)->hdr.next_header;
10034                                 next_protocol &=
10035                                 ((const struct rte_flow_item_ipv6_frag_ext *)
10036                                  items->mask)->hdr.next_header;
10037                         } else {
10038                                 /* Reset for inner layer. */
10039                                 next_protocol = 0xff;
10040                         }
10041                         break;
10042                 case RTE_FLOW_ITEM_TYPE_TCP:
10043                         flow_dv_translate_item_tcp(match_mask, match_value,
10044                                                    items, tunnel);
10045                         matcher.priority = MLX5_PRIORITY_MAP_L4;
10046                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
10047                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
10048                         break;
10049                 case RTE_FLOW_ITEM_TYPE_UDP:
10050                         flow_dv_translate_item_udp(match_mask, match_value,
10051                                                    items, tunnel);
10052                         matcher.priority = MLX5_PRIORITY_MAP_L4;
10053                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
10054                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
10055                         break;
10056                 case RTE_FLOW_ITEM_TYPE_GRE:
10057                         flow_dv_translate_item_gre(match_mask, match_value,
10058                                                    items, tunnel);
10059                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10060                         last_item = MLX5_FLOW_LAYER_GRE;
10061                         break;
10062                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
10063                         flow_dv_translate_item_gre_key(match_mask,
10064                                                        match_value, items);
10065                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
10066                         break;
10067                 case RTE_FLOW_ITEM_TYPE_NVGRE:
10068                         flow_dv_translate_item_nvgre(match_mask, match_value,
10069                                                      items, tunnel);
10070                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10071                         last_item = MLX5_FLOW_LAYER_GRE;
10072                         break;
10073                 case RTE_FLOW_ITEM_TYPE_VXLAN:
10074                         flow_dv_translate_item_vxlan(match_mask, match_value,
10075                                                      items, tunnel);
10076                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10077                         last_item = MLX5_FLOW_LAYER_VXLAN;
10078                         break;
10079                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
10080                         flow_dv_translate_item_vxlan_gpe(match_mask,
10081                                                          match_value, items,
10082                                                          tunnel);
10083                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10084                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
10085                         break;
10086                 case RTE_FLOW_ITEM_TYPE_GENEVE:
10087                         flow_dv_translate_item_geneve(match_mask, match_value,
10088                                                       items, tunnel);
10089                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10090                         last_item = MLX5_FLOW_LAYER_GENEVE;
10091                         break;
10092                 case RTE_FLOW_ITEM_TYPE_MPLS:
10093                         flow_dv_translate_item_mpls(match_mask, match_value,
10094                                                     items, last_item, tunnel);
10095                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10096                         last_item = MLX5_FLOW_LAYER_MPLS;
10097                         break;
10098                 case RTE_FLOW_ITEM_TYPE_MARK:
10099                         flow_dv_translate_item_mark(dev, match_mask,
10100                                                     match_value, items);
10101                         last_item = MLX5_FLOW_ITEM_MARK;
10102                         break;
10103                 case RTE_FLOW_ITEM_TYPE_META:
10104                         flow_dv_translate_item_meta(dev, match_mask,
10105                                                     match_value, attr, items);
10106                         last_item = MLX5_FLOW_ITEM_METADATA;
10107                         break;
10108                 case RTE_FLOW_ITEM_TYPE_ICMP:
10109                         flow_dv_translate_item_icmp(match_mask, match_value,
10110                                                     items, tunnel);
10111                         last_item = MLX5_FLOW_LAYER_ICMP;
10112                         break;
10113                 case RTE_FLOW_ITEM_TYPE_ICMP6:
10114                         flow_dv_translate_item_icmp6(match_mask, match_value,
10115                                                       items, tunnel);
10116                         last_item = MLX5_FLOW_LAYER_ICMP6;
10117                         break;
10118                 case RTE_FLOW_ITEM_TYPE_TAG:
10119                         flow_dv_translate_item_tag(dev, match_mask,
10120                                                    match_value, items);
10121                         last_item = MLX5_FLOW_ITEM_TAG;
10122                         break;
10123                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
10124                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
10125                                                         match_value, items);
10126                         last_item = MLX5_FLOW_ITEM_TAG;
10127                         break;
10128                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
10129                         flow_dv_translate_item_tx_queue(dev, match_mask,
10130                                                         match_value,
10131                                                         items);
10132                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
10133                         break;
10134                 case RTE_FLOW_ITEM_TYPE_GTP:
10135                         flow_dv_translate_item_gtp(match_mask, match_value,
10136                                                    items, tunnel);
10137                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10138                         last_item = MLX5_FLOW_LAYER_GTP;
10139                         break;
10140                 case RTE_FLOW_ITEM_TYPE_ECPRI:
10141                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
10142                                 /* Create it only the first time to be used. */
10143                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
10144                                 if (ret)
10145                                         return rte_flow_error_set
10146                                                 (error, -ret,
10147                                                 RTE_FLOW_ERROR_TYPE_ITEM,
10148                                                 NULL,
10149                                                 "cannot create eCPRI parser");
10150                         }
10151                         /* Adjust the length matcher and device flow value. */
10152                         matcher.mask.size = MLX5_ST_SZ_BYTES(fte_match_param);
10153                         dev_flow->dv.value.size =
10154                                         MLX5_ST_SZ_BYTES(fte_match_param);
10155                         flow_dv_translate_item_ecpri(dev, match_mask,
10156                                                      match_value, items);
10157                         /* No other protocol should follow eCPRI layer. */
10158                         last_item = MLX5_FLOW_LAYER_ECPRI;
10159                         break;
10160                 default:
10161                         break;
10162                 }
10163                 item_flags |= last_item;
10164         }
10165         /*
10166          * When E-Switch mode is enabled, we have two cases where we need to
10167          * set the source port manually.
10168          * The first one, is in case of Nic steering rule, and the second is
10169          * E-Switch rule where no port_id item was found. In both cases
10170          * the source port is set according the current port in use.
10171          */
10172         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
10173             (priv->representor || priv->master)) {
10174                 if (flow_dv_translate_item_port_id(dev, match_mask,
10175                                                    match_value, NULL))
10176                         return -rte_errno;
10177         }
10178 #ifdef RTE_LIBRTE_MLX5_DEBUG
10179         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
10180                                               dev_flow->dv.value.buf));
10181 #endif
10182         /*
10183          * Layers may be already initialized from prefix flow if this dev_flow
10184          * is the suffix flow.
10185          */
10186         handle->layers |= item_flags;
10187         if (action_flags & MLX5_FLOW_ACTION_RSS)
10188                 flow_dv_hashfields_set(dev_flow, rss_desc);
10189         /* Register matcher. */
10190         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
10191                                     matcher.mask.size);
10192         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
10193                                                      matcher.priority);
10194         /* reserved field no needs to be set to 0 here. */
10195         tbl_key.domain = attr->transfer;
10196         tbl_key.direction = attr->egress;
10197         tbl_key.table_id = dev_flow->dv.group;
10198         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
10199                 return -rte_errno;
10200         return 0;
10201 }
10202
10203 /**
10204  * Apply the flow to the NIC, lock free,
10205  * (mutex should be acquired by caller).
10206  *
10207  * @param[in] dev
10208  *   Pointer to the Ethernet device structure.
10209  * @param[in, out] flow
10210  *   Pointer to flow structure.
10211  * @param[out] error
10212  *   Pointer to error structure.
10213  *
10214  * @return
10215  *   0 on success, a negative errno value otherwise and rte_errno is set.
10216  */
10217 static int
10218 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
10219                 struct rte_flow_error *error)
10220 {
10221         struct mlx5_flow_dv_workspace *dv;
10222         struct mlx5_flow_handle *dh;
10223         struct mlx5_flow_handle_dv *dv_h;
10224         struct mlx5_flow *dev_flow;
10225         struct mlx5_priv *priv = dev->data->dev_private;
10226         uint32_t handle_idx;
10227         int n;
10228         int err;
10229         int idx;
10230
10231         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
10232                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
10233                 dv = &dev_flow->dv;
10234                 dh = dev_flow->handle;
10235                 dv_h = &dh->dvh;
10236                 n = dv->actions_n;
10237                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
10238                         if (dv->transfer) {
10239                                 dv->actions[n++] = priv->sh->esw_drop_action;
10240                         } else {
10241                                 struct mlx5_hrxq *drop_hrxq;
10242                                 drop_hrxq = mlx5_drop_action_create(dev);
10243                                 if (!drop_hrxq) {
10244                                         rte_flow_error_set
10245                                                 (error, errno,
10246                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10247                                                  NULL,
10248                                                  "cannot get drop hash queue");
10249                                         goto error;
10250                                 }
10251                                 /*
10252                                  * Drop queues will be released by the specify
10253                                  * mlx5_drop_action_destroy() function. Assign
10254                                  * the special index to hrxq to mark the queue
10255                                  * has been allocated.
10256                                  */
10257                                 dh->rix_hrxq = UINT32_MAX;
10258                                 dv->actions[n++] = drop_hrxq->action;
10259                         }
10260                 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
10261                            !dv_h->rix_sample && !dv_h->rix_dest_array) {
10262                         struct mlx5_hrxq *hrxq;
10263                         uint32_t hrxq_idx;
10264                         struct mlx5_flow_rss_desc *rss_desc =
10265                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
10266                                 [!!priv->flow_nested_idx];
10267
10268                         MLX5_ASSERT(rss_desc->queue_num);
10269                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
10270                                                  MLX5_RSS_HASH_KEY_LEN,
10271                                                  dev_flow->hash_fields,
10272                                                  rss_desc->queue,
10273                                                  rss_desc->queue_num);
10274                         if (!hrxq_idx) {
10275                                 hrxq_idx = mlx5_hrxq_new
10276                                                 (dev, rss_desc->key,
10277                                                  MLX5_RSS_HASH_KEY_LEN,
10278                                                  dev_flow->hash_fields,
10279                                                  rss_desc->queue,
10280                                                  rss_desc->queue_num,
10281                                                  !!(dh->layers &
10282                                                  MLX5_FLOW_LAYER_TUNNEL));
10283                         }
10284                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
10285                                               hrxq_idx);
10286                         if (!hrxq) {
10287                                 rte_flow_error_set
10288                                         (error, rte_errno,
10289                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10290                                          "cannot get hash queue");
10291                                 goto error;
10292                         }
10293                         dh->rix_hrxq = hrxq_idx;
10294                         dv->actions[n++] = hrxq->action;
10295                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
10296                         if (flow_dv_default_miss_resource_register
10297                                         (dev, error)) {
10298                                 rte_flow_error_set
10299                                         (error, rte_errno,
10300                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10301                                          "cannot create default miss resource");
10302                                 goto error_default_miss;
10303                         }
10304                         dh->rix_default_fate =  MLX5_FLOW_FATE_DEFAULT_MISS;
10305                         dv->actions[n++] = priv->sh->default_miss.action;
10306                 }
10307                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
10308                                                (void *)&dv->value, n,
10309                                                dv->actions, &dh->drv_flow);
10310                 if (err) {
10311                         rte_flow_error_set(error, errno,
10312                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10313                                            NULL,
10314                                            "hardware refuses to create flow");
10315                         goto error;
10316                 }
10317                 if (priv->vmwa_context &&
10318                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
10319                         /*
10320                          * The rule contains the VLAN pattern.
10321                          * For VF we are going to create VLAN
10322                          * interface to make hypervisor set correct
10323                          * e-Switch vport context.
10324                          */
10325                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
10326                 }
10327         }
10328         return 0;
10329 error:
10330         if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
10331                 flow_dv_default_miss_resource_release(dev);
10332 error_default_miss:
10333         err = rte_errno; /* Save rte_errno before cleanup. */
10334         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
10335                        handle_idx, dh, next) {
10336                 /* hrxq is union, don't clear it if the flag is not set. */
10337                 if (dh->rix_hrxq) {
10338                         if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
10339                                 mlx5_drop_action_destroy(dev);
10340                                 dh->rix_hrxq = 0;
10341                         } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
10342                                 mlx5_hrxq_release(dev, dh->rix_hrxq);
10343                                 dh->rix_hrxq = 0;
10344                         }
10345                 }
10346                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
10347                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
10348         }
10349         rte_errno = err; /* Restore rte_errno. */
10350         return -rte_errno;
10351 }
10352
10353 /**
10354  * Release the flow matcher.
10355  *
10356  * @param dev
10357  *   Pointer to Ethernet device.
10358  * @param handle
10359  *   Pointer to mlx5_flow_handle.
10360  *
10361  * @return
10362  *   1 while a reference on it exists, 0 when freed.
10363  */
10364 static int
10365 flow_dv_matcher_release(struct rte_eth_dev *dev,
10366                         struct mlx5_flow_handle *handle)
10367 {
10368         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
10369
10370         MLX5_ASSERT(matcher->matcher_object);
10371         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
10372                 dev->data->port_id, (void *)matcher,
10373                 rte_atomic32_read(&matcher->refcnt));
10374         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
10375                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10376                            (matcher->matcher_object));
10377                 LIST_REMOVE(matcher, next);
10378                 /* table ref-- in release interface. */
10379                 flow_dv_tbl_resource_release(dev, matcher->tbl);
10380                 mlx5_free(matcher);
10381                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
10382                         dev->data->port_id, (void *)matcher);
10383                 return 0;
10384         }
10385         return 1;
10386 }
10387
10388 /**
10389  * Release an encap/decap resource.
10390  *
10391  * @param dev
10392  *   Pointer to Ethernet device.
10393  * @param encap_decap_idx
10394  *   Index of encap decap resource.
10395  *
10396  * @return
10397  *   1 while a reference on it exists, 0 when freed.
10398  */
10399 static int
10400 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
10401                                      uint32_t encap_decap_idx)
10402 {
10403         struct mlx5_priv *priv = dev->data->dev_private;
10404         uint32_t idx = encap_decap_idx;
10405         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
10406
10407         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
10408                          idx);
10409         if (!cache_resource)
10410                 return 0;
10411         MLX5_ASSERT(cache_resource->action);
10412         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
10413                 (void *)cache_resource,
10414                 rte_atomic32_read(&cache_resource->refcnt));
10415         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10416                 claim_zero(mlx5_flow_os_destroy_flow_action
10417                                                 (cache_resource->action));
10418                 mlx5_hlist_remove(priv->sh->encaps_decaps,
10419                                   &cache_resource->entry);
10420                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
10421                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
10422                         (void *)cache_resource);
10423                 return 0;
10424         }
10425         return 1;
10426 }
10427
10428 /**
10429  * Release an jump to table action resource.
10430  *
10431  * @param dev
10432  *   Pointer to Ethernet device.
10433  * @param handle
10434  *   Pointer to mlx5_flow_handle.
10435  *
10436  * @return
10437  *   1 while a reference on it exists, 0 when freed.
10438  */
10439 static int
10440 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
10441                                   struct mlx5_flow_handle *handle)
10442 {
10443         struct mlx5_priv *priv = dev->data->dev_private;
10444         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
10445         struct mlx5_flow_tbl_data_entry *tbl_data;
10446
10447         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
10448                              handle->rix_jump);
10449         if (!tbl_data)
10450                 return 0;
10451         cache_resource = &tbl_data->jump;
10452         MLX5_ASSERT(cache_resource->action);
10453         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
10454                 (void *)cache_resource,
10455                 rte_atomic32_read(&cache_resource->refcnt));
10456         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10457                 claim_zero(mlx5_flow_os_destroy_flow_action
10458                                                 (cache_resource->action));
10459                 /* jump action memory free is inside the table release. */
10460                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
10461                 DRV_LOG(DEBUG, "jump table resource %p: removed",
10462                         (void *)cache_resource);
10463                 return 0;
10464         }
10465         return 1;
10466 }
10467
10468 /**
10469  * Release a default miss resource.
10470  *
10471  * @param dev
10472  *   Pointer to Ethernet device.
10473  * @return
10474  *   1 while a reference on it exists, 0 when freed.
10475  */
10476 static int
10477 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
10478 {
10479         struct mlx5_priv *priv = dev->data->dev_private;
10480         struct mlx5_dev_ctx_shared *sh = priv->sh;
10481         struct mlx5_flow_default_miss_resource *cache_resource =
10482                         &sh->default_miss;
10483
10484         MLX5_ASSERT(cache_resource->action);
10485         DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--",
10486                         (void *)cache_resource->action,
10487                         rte_atomic32_read(&cache_resource->refcnt));
10488         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10489                 claim_zero(mlx5_glue->destroy_flow_action
10490                                 (cache_resource->action));
10491                 DRV_LOG(DEBUG, "default miss resource %p: removed",
10492                                 (void *)cache_resource->action);
10493                 return 0;
10494         }
10495         return 1;
10496 }
10497
10498 /**
10499  * Release a modify-header resource.
10500  *
10501  * @param dev
10502  *   Pointer to Ethernet device.
10503  * @param handle
10504  *   Pointer to mlx5_flow_handle.
10505  *
10506  * @return
10507  *   1 while a reference on it exists, 0 when freed.
10508  */
10509 static int
10510 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
10511                                     struct mlx5_flow_handle *handle)
10512 {
10513         struct mlx5_priv *priv = dev->data->dev_private;
10514         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
10515                                                         handle->dvh.modify_hdr;
10516
10517         MLX5_ASSERT(cache_resource->action);
10518         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
10519                 (void *)cache_resource,
10520                 rte_atomic32_read(&cache_resource->refcnt));
10521         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10522                 claim_zero(mlx5_flow_os_destroy_flow_action
10523                                                 (cache_resource->action));
10524                 mlx5_hlist_remove(priv->sh->modify_cmds,
10525                                   &cache_resource->entry);
10526                 mlx5_free(cache_resource);
10527                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
10528                         (void *)cache_resource);
10529                 return 0;
10530         }
10531         return 1;
10532 }
10533
10534 /**
10535  * Release port ID action resource.
10536  *
10537  * @param dev
10538  *   Pointer to Ethernet device.
10539  * @param handle
10540  *   Pointer to mlx5_flow_handle.
10541  *
10542  * @return
10543  *   1 while a reference on it exists, 0 when freed.
10544  */
10545 static int
10546 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
10547                                         uint32_t port_id)
10548 {
10549         struct mlx5_priv *priv = dev->data->dev_private;
10550         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
10551         uint32_t idx = port_id;
10552
10553         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
10554                                         idx);
10555         if (!cache_resource)
10556                 return 0;
10557         MLX5_ASSERT(cache_resource->action);
10558         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
10559                 (void *)cache_resource,
10560                 rte_atomic32_read(&cache_resource->refcnt));
10561         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10562                 claim_zero(mlx5_flow_os_destroy_flow_action
10563                                                 (cache_resource->action));
10564                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
10565                              &priv->sh->port_id_action_list, idx,
10566                              cache_resource, next);
10567                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx);
10568                 DRV_LOG(DEBUG, "port id action resource %p: removed",
10569                         (void *)cache_resource);
10570                 return 0;
10571         }
10572         return 1;
10573 }
10574
10575 /**
10576  * Release push vlan action resource.
10577  *
10578  * @param dev
10579  *   Pointer to Ethernet device.
10580  * @param handle
10581  *   Pointer to mlx5_flow_handle.
10582  *
10583  * @return
10584  *   1 while a reference on it exists, 0 when freed.
10585  */
10586 static int
10587 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
10588                                           struct mlx5_flow_handle *handle)
10589 {
10590         struct mlx5_priv *priv = dev->data->dev_private;
10591         uint32_t idx = handle->dvh.rix_push_vlan;
10592         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
10593
10594         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
10595                                         idx);
10596         if (!cache_resource)
10597                 return 0;
10598         MLX5_ASSERT(cache_resource->action);
10599         DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
10600                 (void *)cache_resource,
10601                 rte_atomic32_read(&cache_resource->refcnt));
10602         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10603                 claim_zero(mlx5_flow_os_destroy_flow_action
10604                                                 (cache_resource->action));
10605                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
10606                              &priv->sh->push_vlan_action_list, idx,
10607                              cache_resource, next);
10608                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
10609                 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
10610                         (void *)cache_resource);
10611                 return 0;
10612         }
10613         return 1;
10614 }
10615
10616 /**
10617  * Release the fate resource.
10618  *
10619  * @param dev
10620  *   Pointer to Ethernet device.
10621  * @param handle
10622  *   Pointer to mlx5_flow_handle.
10623  */
10624 static void
10625 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
10626                                struct mlx5_flow_handle *handle)
10627 {
10628         if (!handle->rix_fate)
10629                 return;
10630         switch (handle->fate_action) {
10631         case MLX5_FLOW_FATE_DROP:
10632                 mlx5_drop_action_destroy(dev);
10633                 break;
10634         case MLX5_FLOW_FATE_QUEUE:
10635                 mlx5_hrxq_release(dev, handle->rix_hrxq);
10636                 break;
10637         case MLX5_FLOW_FATE_JUMP:
10638                 flow_dv_jump_tbl_resource_release(dev, handle);
10639                 break;
10640         case MLX5_FLOW_FATE_PORT_ID:
10641                 flow_dv_port_id_action_resource_release(dev,
10642                                 handle->rix_port_id_action);
10643                 break;
10644         case MLX5_FLOW_FATE_DEFAULT_MISS:
10645                 flow_dv_default_miss_resource_release(dev);
10646                 break;
10647         default:
10648                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
10649                 break;
10650         }
10651         handle->rix_fate = 0;
10652 }
10653
10654 /**
10655  * Release an sample resource.
10656  *
10657  * @param dev
10658  *   Pointer to Ethernet device.
10659  * @param handle
10660  *   Pointer to mlx5_flow_handle.
10661  *
10662  * @return
10663  *   1 while a reference on it exists, 0 when freed.
10664  */
10665 static int
10666 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
10667                                      struct mlx5_flow_handle *handle)
10668 {
10669         struct mlx5_priv *priv = dev->data->dev_private;
10670         uint32_t idx = handle->dvh.rix_sample;
10671         struct mlx5_flow_dv_sample_resource *cache_resource;
10672
10673         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
10674                          idx);
10675         if (!cache_resource)
10676                 return 0;
10677         MLX5_ASSERT(cache_resource->verbs_action);
10678         DRV_LOG(DEBUG, "sample resource %p: refcnt %d--",
10679                 (void *)cache_resource,
10680                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10681         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10682                                __ATOMIC_RELAXED) == 0) {
10683                 if (cache_resource->verbs_action)
10684                         claim_zero(mlx5_glue->destroy_flow_action
10685                                         (cache_resource->verbs_action));
10686                 if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
10687                         if (cache_resource->default_miss)
10688                                 claim_zero(mlx5_glue->destroy_flow_action
10689                                   (cache_resource->default_miss));
10690                 }
10691                 if (cache_resource->normal_path_tbl)
10692                         flow_dv_tbl_resource_release(dev,
10693                                 cache_resource->normal_path_tbl);
10694         }
10695         if (cache_resource->sample_idx.rix_hrxq &&
10696                 !mlx5_hrxq_release(dev,
10697                         cache_resource->sample_idx.rix_hrxq))
10698                 cache_resource->sample_idx.rix_hrxq = 0;
10699         if (cache_resource->sample_idx.rix_tag &&
10700                 !flow_dv_tag_release(dev,
10701                         cache_resource->sample_idx.rix_tag))
10702                 cache_resource->sample_idx.rix_tag = 0;
10703         if (cache_resource->sample_idx.cnt) {
10704                 flow_dv_counter_release(dev,
10705                         cache_resource->sample_idx.cnt);
10706                 cache_resource->sample_idx.cnt = 0;
10707         }
10708         if (!__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)) {
10709                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
10710                              &priv->sh->sample_action_list, idx,
10711                              cache_resource, next);
10712                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], idx);
10713                 DRV_LOG(DEBUG, "sample resource %p: removed",
10714                         (void *)cache_resource);
10715                 return 0;
10716         }
10717         return 1;
10718 }
10719
10720 /**
10721  * Release an destination array resource.
10722  *
10723  * @param dev
10724  *   Pointer to Ethernet device.
10725  * @param handle
10726  *   Pointer to mlx5_flow_handle.
10727  *
10728  * @return
10729  *   1 while a reference on it exists, 0 when freed.
10730  */
10731 static int
10732 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
10733                                      struct mlx5_flow_handle *handle)
10734 {
10735         struct mlx5_priv *priv = dev->data->dev_private;
10736         struct mlx5_flow_dv_dest_array_resource *cache_resource;
10737         struct mlx5_flow_sub_actions_idx *mdest_act_res;
10738         uint32_t idx = handle->dvh.rix_dest_array;
10739         uint32_t i = 0;
10740
10741         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10742                          idx);
10743         if (!cache_resource)
10744                 return 0;
10745         MLX5_ASSERT(cache_resource->action);
10746         DRV_LOG(DEBUG, "destination array resource %p: refcnt %d--",
10747                 (void *)cache_resource,
10748                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10749         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10750                                __ATOMIC_RELAXED) == 0) {
10751                 if (cache_resource->action)
10752                         claim_zero(mlx5_glue->destroy_flow_action
10753                                                 (cache_resource->action));
10754                 for (; i < cache_resource->num_of_dest; i++) {
10755                         mdest_act_res = &cache_resource->sample_idx[i];
10756                         if (mdest_act_res->rix_hrxq) {
10757                                 mlx5_hrxq_release(dev,
10758                                         mdest_act_res->rix_hrxq);
10759                                 mdest_act_res->rix_hrxq = 0;
10760                         }
10761                         if (mdest_act_res->rix_encap_decap) {
10762                                 flow_dv_encap_decap_resource_release(dev,
10763                                         mdest_act_res->rix_encap_decap);
10764                                 mdest_act_res->rix_encap_decap = 0;
10765                         }
10766                         if (mdest_act_res->rix_port_id_action) {
10767                                 flow_dv_port_id_action_resource_release(dev,
10768                                         mdest_act_res->rix_port_id_action);
10769                                 mdest_act_res->rix_port_id_action = 0;
10770                         }
10771                         if (mdest_act_res->rix_tag) {
10772                                 flow_dv_tag_release(dev,
10773                                         mdest_act_res->rix_tag);
10774                                 mdest_act_res->rix_tag = 0;
10775                         }
10776                 }
10777                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10778                              &priv->sh->dest_array_list, idx,
10779                              cache_resource, next);
10780                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], idx);
10781                 DRV_LOG(DEBUG, "destination array resource %p: removed",
10782                         (void *)cache_resource);
10783                 return 0;
10784         }
10785         return 1;
10786 }
10787
10788 /**
10789  * Remove the flow from the NIC but keeps it in memory.
10790  * Lock free, (mutex should be acquired by caller).
10791  *
10792  * @param[in] dev
10793  *   Pointer to Ethernet device.
10794  * @param[in, out] flow
10795  *   Pointer to flow structure.
10796  */
10797 static void
10798 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
10799 {
10800         struct mlx5_flow_handle *dh;
10801         uint32_t handle_idx;
10802         struct mlx5_priv *priv = dev->data->dev_private;
10803
10804         if (!flow)
10805                 return;
10806         handle_idx = flow->dev_handles;
10807         while (handle_idx) {
10808                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10809                                     handle_idx);
10810                 if (!dh)
10811                         return;
10812                 if (dh->drv_flow) {
10813                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
10814                         dh->drv_flow = NULL;
10815                 }
10816                 if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
10817                     dh->fate_action == MLX5_FLOW_FATE_QUEUE ||
10818                     dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
10819                         flow_dv_fate_resource_release(dev, dh);
10820                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
10821                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
10822                 handle_idx = dh->next.next;
10823         }
10824 }
10825
10826 /**
10827  * Remove the flow from the NIC and the memory.
10828  * Lock free, (mutex should be acquired by caller).
10829  *
10830  * @param[in] dev
10831  *   Pointer to the Ethernet device structure.
10832  * @param[in, out] flow
10833  *   Pointer to flow structure.
10834  */
10835 static void
10836 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
10837 {
10838         struct mlx5_flow_handle *dev_handle;
10839         struct mlx5_priv *priv = dev->data->dev_private;
10840
10841         if (!flow)
10842                 return;
10843         __flow_dv_remove(dev, flow);
10844         if (flow->counter) {
10845                 flow_dv_counter_release(dev, flow->counter);
10846                 flow->counter = 0;
10847         }
10848         if (flow->meter) {
10849                 struct mlx5_flow_meter *fm;
10850
10851                 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
10852                                     flow->meter);
10853                 if (fm)
10854                         mlx5_flow_meter_detach(fm);
10855                 flow->meter = 0;
10856         }
10857         while (flow->dev_handles) {
10858                 uint32_t tmp_idx = flow->dev_handles;
10859
10860                 dev_handle = mlx5_ipool_get(priv->sh->ipool
10861                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
10862                 if (!dev_handle)
10863                         return;
10864                 flow->dev_handles = dev_handle->next.next;
10865                 if (dev_handle->dvh.matcher)
10866                         flow_dv_matcher_release(dev, dev_handle);
10867                 if (dev_handle->dvh.rix_sample)
10868                         flow_dv_sample_resource_release(dev, dev_handle);
10869                 if (dev_handle->dvh.rix_dest_array)
10870                         flow_dv_dest_array_resource_release(dev, dev_handle);
10871                 if (dev_handle->dvh.rix_encap_decap)
10872                         flow_dv_encap_decap_resource_release(dev,
10873                                 dev_handle->dvh.rix_encap_decap);
10874                 if (dev_handle->dvh.modify_hdr)
10875                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
10876                 if (dev_handle->dvh.rix_push_vlan)
10877                         flow_dv_push_vlan_action_resource_release(dev,
10878                                                                   dev_handle);
10879                 if (dev_handle->dvh.rix_tag)
10880                         flow_dv_tag_release(dev,
10881                                             dev_handle->dvh.rix_tag);
10882                 flow_dv_fate_resource_release(dev, dev_handle);
10883                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10884                            tmp_idx);
10885         }
10886 }
10887
10888 /**
10889  * Query a dv flow  rule for its statistics via devx.
10890  *
10891  * @param[in] dev
10892  *   Pointer to Ethernet device.
10893  * @param[in] flow
10894  *   Pointer to the sub flow.
10895  * @param[out] data
10896  *   data retrieved by the query.
10897  * @param[out] error
10898  *   Perform verbose error reporting if not NULL.
10899  *
10900  * @return
10901  *   0 on success, a negative errno value otherwise and rte_errno is set.
10902  */
10903 static int
10904 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
10905                     void *data, struct rte_flow_error *error)
10906 {
10907         struct mlx5_priv *priv = dev->data->dev_private;
10908         struct rte_flow_query_count *qc = data;
10909
10910         if (!priv->config.devx)
10911                 return rte_flow_error_set(error, ENOTSUP,
10912                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10913                                           NULL,
10914                                           "counters are not supported");
10915         if (flow->counter) {
10916                 uint64_t pkts, bytes;
10917                 struct mlx5_flow_counter *cnt;
10918
10919                 cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
10920                                                  NULL);
10921                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
10922                                                &bytes);
10923
10924                 if (err)
10925                         return rte_flow_error_set(error, -err,
10926                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10927                                         NULL, "cannot read counters");
10928                 qc->hits_set = 1;
10929                 qc->bytes_set = 1;
10930                 qc->hits = pkts - cnt->hits;
10931                 qc->bytes = bytes - cnt->bytes;
10932                 if (qc->reset) {
10933                         cnt->hits = pkts;
10934                         cnt->bytes = bytes;
10935                 }
10936                 return 0;
10937         }
10938         return rte_flow_error_set(error, EINVAL,
10939                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10940                                   NULL,
10941                                   "counters are not available");
10942 }
10943
10944 /**
10945  * Query a flow rule AGE action for aging information.
10946  *
10947  * @param[in] dev
10948  *   Pointer to Ethernet device.
10949  * @param[in] flow
10950  *   Pointer to the sub flow.
10951  * @param[out] data
10952  *   data retrieved by the query.
10953  * @param[out] error
10954  *   Perform verbose error reporting if not NULL.
10955  *
10956  * @return
10957  *   0 on success, a negative errno value otherwise and rte_errno is set.
10958  */
10959 static int
10960 flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow,
10961                   void *data, struct rte_flow_error *error)
10962 {
10963         struct rte_flow_query_age *resp = data;
10964
10965         if (flow->counter) {
10966                 struct mlx5_age_param *age_param =
10967                                 flow_dv_counter_idx_get_age(dev, flow->counter);
10968
10969                 if (!age_param || !age_param->timeout)
10970                         return rte_flow_error_set
10971                                         (error, EINVAL,
10972                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10973                                          NULL, "cannot read age data");
10974                 resp->aged = __atomic_load_n(&age_param->state,
10975                                              __ATOMIC_RELAXED) ==
10976                                                         AGE_TMOUT ? 1 : 0;
10977                 resp->sec_since_last_hit_valid = !resp->aged;
10978                 if (resp->sec_since_last_hit_valid)
10979                         resp->sec_since_last_hit =
10980                                 __atomic_load_n(&age_param->sec_since_last_hit,
10981                                                 __ATOMIC_RELAXED);
10982                 return 0;
10983         }
10984         return rte_flow_error_set(error, EINVAL,
10985                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10986                                   NULL,
10987                                   "age data not available");
10988 }
10989
10990 /**
10991  * Query a flow.
10992  *
10993  * @see rte_flow_query()
10994  * @see rte_flow_ops
10995  */
10996 static int
10997 flow_dv_query(struct rte_eth_dev *dev,
10998               struct rte_flow *flow __rte_unused,
10999               const struct rte_flow_action *actions __rte_unused,
11000               void *data __rte_unused,
11001               struct rte_flow_error *error __rte_unused)
11002 {
11003         int ret = -EINVAL;
11004
11005         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
11006                 switch (actions->type) {
11007                 case RTE_FLOW_ACTION_TYPE_VOID:
11008                         break;
11009                 case RTE_FLOW_ACTION_TYPE_COUNT:
11010                         ret = flow_dv_query_count(dev, flow, data, error);
11011                         break;
11012                 case RTE_FLOW_ACTION_TYPE_AGE:
11013                         ret = flow_dv_query_age(dev, flow, data, error);
11014                         break;
11015                 default:
11016                         return rte_flow_error_set(error, ENOTSUP,
11017                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11018                                                   actions,
11019                                                   "action not supported");
11020                 }
11021         }
11022         return ret;
11023 }
11024
11025 /**
11026  * Destroy the meter table set.
11027  * Lock free, (mutex should be acquired by caller).
11028  *
11029  * @param[in] dev
11030  *   Pointer to Ethernet device.
11031  * @param[in] tbl
11032  *   Pointer to the meter table set.
11033  *
11034  * @return
11035  *   Always 0.
11036  */
11037 static int
11038 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
11039                         struct mlx5_meter_domains_infos *tbl)
11040 {
11041         struct mlx5_priv *priv = dev->data->dev_private;
11042         struct mlx5_meter_domains_infos *mtd =
11043                                 (struct mlx5_meter_domains_infos *)tbl;
11044
11045         if (!mtd || !priv->config.dv_flow_en)
11046                 return 0;
11047         if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
11048                 claim_zero(mlx5_flow_os_destroy_flow
11049                            (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
11050         if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
11051                 claim_zero(mlx5_flow_os_destroy_flow
11052                            (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
11053         if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
11054                 claim_zero(mlx5_flow_os_destroy_flow
11055                            (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
11056         if (mtd->egress.color_matcher)
11057                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11058                            (mtd->egress.color_matcher));
11059         if (mtd->egress.any_matcher)
11060                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11061                            (mtd->egress.any_matcher));
11062         if (mtd->egress.tbl)
11063                 flow_dv_tbl_resource_release(dev, mtd->egress.tbl);
11064         if (mtd->egress.sfx_tbl)
11065                 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl);
11066         if (mtd->ingress.color_matcher)
11067                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11068                            (mtd->ingress.color_matcher));
11069         if (mtd->ingress.any_matcher)
11070                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11071                            (mtd->ingress.any_matcher));
11072         if (mtd->ingress.tbl)
11073                 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl);
11074         if (mtd->ingress.sfx_tbl)
11075                 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl);
11076         if (mtd->transfer.color_matcher)
11077                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11078                            (mtd->transfer.color_matcher));
11079         if (mtd->transfer.any_matcher)
11080                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11081                            (mtd->transfer.any_matcher));
11082         if (mtd->transfer.tbl)
11083                 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl);
11084         if (mtd->transfer.sfx_tbl)
11085                 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl);
11086         if (mtd->drop_actn)
11087                 claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn));
11088         mlx5_free(mtd);
11089         return 0;
11090 }
11091
11092 /* Number of meter flow actions, count and jump or count and drop. */
11093 #define METER_ACTIONS 2
11094
11095 /**
11096  * Create specify domain meter table and suffix table.
11097  *
11098  * @param[in] dev
11099  *   Pointer to Ethernet device.
11100  * @param[in,out] mtb
11101  *   Pointer to DV meter table set.
11102  * @param[in] egress
11103  *   Table attribute.
11104  * @param[in] transfer
11105  *   Table attribute.
11106  * @param[in] color_reg_c_idx
11107  *   Reg C index for color match.
11108  *
11109  * @return
11110  *   0 on success, -1 otherwise and rte_errno is set.
11111  */
11112 static int
11113 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
11114                            struct mlx5_meter_domains_infos *mtb,
11115                            uint8_t egress, uint8_t transfer,
11116                            uint32_t color_reg_c_idx)
11117 {
11118         struct mlx5_priv *priv = dev->data->dev_private;
11119         struct mlx5_dev_ctx_shared *sh = priv->sh;
11120         struct mlx5_flow_dv_match_params mask = {
11121                 .size = sizeof(mask.buf),
11122         };
11123         struct mlx5_flow_dv_match_params value = {
11124                 .size = sizeof(value.buf),
11125         };
11126         struct mlx5dv_flow_matcher_attr dv_attr = {
11127                 .type = IBV_FLOW_ATTR_NORMAL,
11128                 .priority = 0,
11129                 .match_criteria_enable = 0,
11130                 .match_mask = (void *)&mask,
11131         };
11132         void *actions[METER_ACTIONS];
11133         struct mlx5_meter_domain_info *dtb;
11134         struct rte_flow_error error;
11135         int i = 0;
11136         int ret;
11137
11138         if (transfer)
11139                 dtb = &mtb->transfer;
11140         else if (egress)
11141                 dtb = &mtb->egress;
11142         else
11143                 dtb = &mtb->ingress;
11144         /* Create the meter table with METER level. */
11145         dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
11146                                             egress, transfer, &error);
11147         if (!dtb->tbl) {
11148                 DRV_LOG(ERR, "Failed to create meter policer table.");
11149                 return -1;
11150         }
11151         /* Create the meter suffix table with SUFFIX level. */
11152         dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
11153                                             MLX5_FLOW_TABLE_LEVEL_SUFFIX,
11154                                             egress, transfer, &error);
11155         if (!dtb->sfx_tbl) {
11156                 DRV_LOG(ERR, "Failed to create meter suffix table.");
11157                 return -1;
11158         }
11159         /* Create matchers, Any and Color. */
11160         dv_attr.priority = 3;
11161         dv_attr.match_criteria_enable = 0;
11162         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
11163                                                &dtb->any_matcher);
11164         if (ret) {
11165                 DRV_LOG(ERR, "Failed to create meter"
11166                              " policer default matcher.");
11167                 goto error_exit;
11168         }
11169         dv_attr.priority = 0;
11170         dv_attr.match_criteria_enable =
11171                                 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
11172         flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
11173                                rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
11174         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
11175                                                &dtb->color_matcher);
11176         if (ret) {
11177                 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
11178                 goto error_exit;
11179         }
11180         if (mtb->count_actns[RTE_MTR_DROPPED])
11181                 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
11182         actions[i++] = mtb->drop_actn;
11183         /* Default rule: lowest priority, match any, actions: drop. */
11184         ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i,
11185                                        actions,
11186                                        &dtb->policer_rules[RTE_MTR_DROPPED]);
11187         if (ret) {
11188                 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
11189                 goto error_exit;
11190         }
11191         return 0;
11192 error_exit:
11193         return -1;
11194 }
11195
11196 /**
11197  * Create the needed meter and suffix tables.
11198  * Lock free, (mutex should be acquired by caller).
11199  *
11200  * @param[in] dev
11201  *   Pointer to Ethernet device.
11202  * @param[in] fm
11203  *   Pointer to the flow meter.
11204  *
11205  * @return
11206  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
11207  */
11208 static struct mlx5_meter_domains_infos *
11209 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
11210                        const struct mlx5_flow_meter *fm)
11211 {
11212         struct mlx5_priv *priv = dev->data->dev_private;
11213         struct mlx5_meter_domains_infos *mtb;
11214         int ret;
11215         int i;
11216
11217         if (!priv->mtr_en) {
11218                 rte_errno = ENOTSUP;
11219                 return NULL;
11220         }
11221         mtb = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*mtb), 0, SOCKET_ID_ANY);
11222         if (!mtb) {
11223                 DRV_LOG(ERR, "Failed to allocate memory for meter.");
11224                 return NULL;
11225         }
11226         /* Create meter count actions */
11227         for (i = 0; i <= RTE_MTR_DROPPED; i++) {
11228                 struct mlx5_flow_counter *cnt;
11229                 if (!fm->policer_stats.cnt[i])
11230                         continue;
11231                 cnt = flow_dv_counter_get_by_idx(dev,
11232                       fm->policer_stats.cnt[i], NULL);
11233                 mtb->count_actns[i] = cnt->action;
11234         }
11235         /* Create drop action. */
11236         ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn);
11237         if (ret) {
11238                 DRV_LOG(ERR, "Failed to create drop action.");
11239                 goto error_exit;
11240         }
11241         /* Egress meter table. */
11242         ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
11243         if (ret) {
11244                 DRV_LOG(ERR, "Failed to prepare egress meter table.");
11245                 goto error_exit;
11246         }
11247         /* Ingress meter table. */
11248         ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
11249         if (ret) {
11250                 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
11251                 goto error_exit;
11252         }
11253         /* FDB meter table. */
11254         if (priv->config.dv_esw_en) {
11255                 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
11256                                                  priv->mtr_color_reg);
11257                 if (ret) {
11258                         DRV_LOG(ERR, "Failed to prepare fdb meter table.");
11259                         goto error_exit;
11260                 }
11261         }
11262         return mtb;
11263 error_exit:
11264         flow_dv_destroy_mtr_tbl(dev, mtb);
11265         return NULL;
11266 }
11267
11268 /**
11269  * Destroy domain policer rule.
11270  *
11271  * @param[in] dt
11272  *   Pointer to domain table.
11273  */
11274 static void
11275 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
11276 {
11277         int i;
11278
11279         for (i = 0; i < RTE_MTR_DROPPED; i++) {
11280                 if (dt->policer_rules[i]) {
11281                         claim_zero(mlx5_flow_os_destroy_flow
11282                                    (dt->policer_rules[i]));
11283                         dt->policer_rules[i] = NULL;
11284                 }
11285         }
11286         if (dt->jump_actn) {
11287                 claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn));
11288                 dt->jump_actn = NULL;
11289         }
11290 }
11291
11292 /**
11293  * Destroy policer rules.
11294  *
11295  * @param[in] dev
11296  *   Pointer to Ethernet device.
11297  * @param[in] fm
11298  *   Pointer to flow meter structure.
11299  * @param[in] attr
11300  *   Pointer to flow attributes.
11301  *
11302  * @return
11303  *   Always 0.
11304  */
11305 static int
11306 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
11307                               const struct mlx5_flow_meter *fm,
11308                               const struct rte_flow_attr *attr)
11309 {
11310         struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
11311
11312         if (!mtb)
11313                 return 0;
11314         if (attr->egress)
11315                 flow_dv_destroy_domain_policer_rule(&mtb->egress);
11316         if (attr->ingress)
11317                 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
11318         if (attr->transfer)
11319                 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
11320         return 0;
11321 }
11322
11323 /**
11324  * Create specify domain meter policer rule.
11325  *
11326  * @param[in] fm
11327  *   Pointer to flow meter structure.
11328  * @param[in] mtb
11329  *   Pointer to DV meter table set.
11330  * @param[in] mtr_reg_c
11331  *   Color match REG_C.
11332  *
11333  * @return
11334  *   0 on success, -1 otherwise.
11335  */
11336 static int
11337 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
11338                                     struct mlx5_meter_domain_info *dtb,
11339                                     uint8_t mtr_reg_c)
11340 {
11341         struct mlx5_flow_dv_match_params matcher = {
11342                 .size = sizeof(matcher.buf),
11343         };
11344         struct mlx5_flow_dv_match_params value = {
11345                 .size = sizeof(value.buf),
11346         };
11347         struct mlx5_meter_domains_infos *mtb = fm->mfts;
11348         void *actions[METER_ACTIONS];
11349         int i;
11350         int ret = 0;
11351
11352         /* Create jump action. */
11353         if (!dtb->jump_actn)
11354                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
11355                                 (dtb->sfx_tbl->obj, &dtb->jump_actn);
11356         if (ret) {
11357                 DRV_LOG(ERR, "Failed to create policer jump action.");
11358                 goto error;
11359         }
11360         for (i = 0; i < RTE_MTR_DROPPED; i++) {
11361                 int j = 0;
11362
11363                 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
11364                                        rte_col_2_mlx5_col(i), UINT8_MAX);
11365                 if (mtb->count_actns[i])
11366                         actions[j++] = mtb->count_actns[i];
11367                 if (fm->action[i] == MTR_POLICER_ACTION_DROP)
11368                         actions[j++] = mtb->drop_actn;
11369                 else
11370                         actions[j++] = dtb->jump_actn;
11371                 ret = mlx5_flow_os_create_flow(dtb->color_matcher,
11372                                                (void *)&value, j, actions,
11373                                                &dtb->policer_rules[i]);
11374                 if (ret) {
11375                         DRV_LOG(ERR, "Failed to create policer rule.");
11376                         goto error;
11377                 }
11378         }
11379         return 0;
11380 error:
11381         rte_errno = errno;
11382         return -1;
11383 }
11384
11385 /**
11386  * Create policer rules.
11387  *
11388  * @param[in] dev
11389  *   Pointer to Ethernet device.
11390  * @param[in] fm
11391  *   Pointer to flow meter structure.
11392  * @param[in] attr
11393  *   Pointer to flow attributes.
11394  *
11395  * @return
11396  *   0 on success, -1 otherwise.
11397  */
11398 static int
11399 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
11400                              struct mlx5_flow_meter *fm,
11401                              const struct rte_flow_attr *attr)
11402 {
11403         struct mlx5_priv *priv = dev->data->dev_private;
11404         struct mlx5_meter_domains_infos *mtb = fm->mfts;
11405         int ret;
11406
11407         if (attr->egress) {
11408                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
11409                                                 priv->mtr_color_reg);
11410                 if (ret) {
11411                         DRV_LOG(ERR, "Failed to create egress policer.");
11412                         goto error;
11413                 }
11414         }
11415         if (attr->ingress) {
11416                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
11417                                                 priv->mtr_color_reg);
11418                 if (ret) {
11419                         DRV_LOG(ERR, "Failed to create ingress policer.");
11420                         goto error;
11421                 }
11422         }
11423         if (attr->transfer) {
11424                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
11425                                                 priv->mtr_color_reg);
11426                 if (ret) {
11427                         DRV_LOG(ERR, "Failed to create transfer policer.");
11428                         goto error;
11429                 }
11430         }
11431         return 0;
11432 error:
11433         flow_dv_destroy_policer_rules(dev, fm, attr);
11434         return -1;
11435 }
11436
11437 /**
11438  * Query a devx counter.
11439  *
11440  * @param[in] dev
11441  *   Pointer to the Ethernet device structure.
11442  * @param[in] cnt
11443  *   Index to the flow counter.
11444  * @param[in] clear
11445  *   Set to clear the counter statistics.
11446  * @param[out] pkts
11447  *   The statistics value of packets.
11448  * @param[out] bytes
11449  *   The statistics value of bytes.
11450  *
11451  * @return
11452  *   0 on success, otherwise return -1.
11453  */
11454 static int
11455 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
11456                       uint64_t *pkts, uint64_t *bytes)
11457 {
11458         struct mlx5_priv *priv = dev->data->dev_private;
11459         struct mlx5_flow_counter *cnt;
11460         uint64_t inn_pkts, inn_bytes;
11461         int ret;
11462
11463         if (!priv->config.devx)
11464                 return -1;
11465
11466         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
11467         if (ret)
11468                 return -1;
11469         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
11470         *pkts = inn_pkts - cnt->hits;
11471         *bytes = inn_bytes - cnt->bytes;
11472         if (clear) {
11473                 cnt->hits = inn_pkts;
11474                 cnt->bytes = inn_bytes;
11475         }
11476         return 0;
11477 }
11478
11479 /**
11480  * Get aged-out flows.
11481  *
11482  * @param[in] dev
11483  *   Pointer to the Ethernet device structure.
11484  * @param[in] context
11485  *   The address of an array of pointers to the aged-out flows contexts.
11486  * @param[in] nb_contexts
11487  *   The length of context array pointers.
11488  * @param[out] error
11489  *   Perform verbose error reporting if not NULL. Initialized in case of
11490  *   error only.
11491  *
11492  * @return
11493  *   how many contexts get in success, otherwise negative errno value.
11494  *   if nb_contexts is 0, return the amount of all aged contexts.
11495  *   if nb_contexts is not 0 , return the amount of aged flows reported
11496  *   in the context array.
11497  * @note: only stub for now
11498  */
11499 static int
11500 flow_get_aged_flows(struct rte_eth_dev *dev,
11501                     void **context,
11502                     uint32_t nb_contexts,
11503                     struct rte_flow_error *error)
11504 {
11505         struct mlx5_priv *priv = dev->data->dev_private;
11506         struct mlx5_age_info *age_info;
11507         struct mlx5_age_param *age_param;
11508         struct mlx5_flow_counter *counter;
11509         int nb_flows = 0;
11510
11511         if (nb_contexts && !context)
11512                 return rte_flow_error_set(error, EINVAL,
11513                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11514                                           NULL,
11515                                           "Should assign at least one flow or"
11516                                           " context to get if nb_contexts != 0");
11517         age_info = GET_PORT_AGE_INFO(priv);
11518         rte_spinlock_lock(&age_info->aged_sl);
11519         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
11520                 nb_flows++;
11521                 if (nb_contexts) {
11522                         age_param = MLX5_CNT_TO_AGE(counter);
11523                         context[nb_flows - 1] = age_param->context;
11524                         if (!(--nb_contexts))
11525                                 break;
11526                 }
11527         }
11528         rte_spinlock_unlock(&age_info->aged_sl);
11529         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
11530         return nb_flows;
11531 }
11532
11533 /*
11534  * Mutex-protected thunk to lock-free  __flow_dv_translate().
11535  */
11536 static int
11537 flow_dv_translate(struct rte_eth_dev *dev,
11538                   struct mlx5_flow *dev_flow,
11539                   const struct rte_flow_attr *attr,
11540                   const struct rte_flow_item items[],
11541                   const struct rte_flow_action actions[],
11542                   struct rte_flow_error *error)
11543 {
11544         int ret;
11545
11546         flow_dv_shared_lock(dev);
11547         ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
11548         flow_dv_shared_unlock(dev);
11549         return ret;
11550 }
11551
11552 /*
11553  * Mutex-protected thunk to lock-free  __flow_dv_apply().
11554  */
11555 static int
11556 flow_dv_apply(struct rte_eth_dev *dev,
11557               struct rte_flow *flow,
11558               struct rte_flow_error *error)
11559 {
11560         int ret;
11561
11562         flow_dv_shared_lock(dev);
11563         ret = __flow_dv_apply(dev, flow, error);
11564         flow_dv_shared_unlock(dev);
11565         return ret;
11566 }
11567
11568 /*
11569  * Mutex-protected thunk to lock-free __flow_dv_remove().
11570  */
11571 static void
11572 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
11573 {
11574         flow_dv_shared_lock(dev);
11575         __flow_dv_remove(dev, flow);
11576         flow_dv_shared_unlock(dev);
11577 }
11578
11579 /*
11580  * Mutex-protected thunk to lock-free __flow_dv_destroy().
11581  */
11582 static void
11583 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
11584 {
11585         flow_dv_shared_lock(dev);
11586         __flow_dv_destroy(dev, flow);
11587         flow_dv_shared_unlock(dev);
11588 }
11589
11590 /*
11591  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
11592  */
11593 static uint32_t
11594 flow_dv_counter_allocate(struct rte_eth_dev *dev)
11595 {
11596         uint32_t cnt;
11597
11598         flow_dv_shared_lock(dev);
11599         cnt = flow_dv_counter_alloc(dev, 0, 0, 1, 0);
11600         flow_dv_shared_unlock(dev);
11601         return cnt;
11602 }
11603
11604 /*
11605  * Mutex-protected thunk to lock-free flow_dv_counter_release().
11606  */
11607 static void
11608 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
11609 {
11610         flow_dv_shared_lock(dev);
11611         flow_dv_counter_release(dev, cnt);
11612         flow_dv_shared_unlock(dev);
11613 }
11614
11615 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
11616         .validate = flow_dv_validate,
11617         .prepare = flow_dv_prepare,
11618         .translate = flow_dv_translate,
11619         .apply = flow_dv_apply,
11620         .remove = flow_dv_remove,
11621         .destroy = flow_dv_destroy,
11622         .query = flow_dv_query,
11623         .create_mtr_tbls = flow_dv_create_mtr_tbl,
11624         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
11625         .create_policer_rules = flow_dv_create_policer_rules,
11626         .destroy_policer_rules = flow_dv_destroy_policer_rules,
11627         .counter_alloc = flow_dv_counter_allocate,
11628         .counter_free = flow_dv_counter_free,
11629         .counter_query = flow_dv_counter_query,
11630         .get_aged_flows = flow_get_aged_flows,
11631 };
11632
11633 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
11634