drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <rte_ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_ip.h>
  19 #include <rte_gre.h>
  20 #include <rte_vxlan.h>
  21 #include <rte_gtp.h>
  22 #include <rte_eal_paging.h>
  23 #include <rte_mpls.h>
  24
  25 #include <mlx5_glue.h>
  26 #include <mlx5_devx_cmds.h>
  27 #include <mlx5_prm.h>
  28 #include <mlx5_malloc.h>
  29
  30 #include "mlx5_defs.h"
  31 #include "mlx5.h"
  32 #include "mlx5_common_os.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_flow_os.h"
  35 #include "mlx5_rxtx.h"
  36
  37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  38
  39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  41 #endif
  42
  43 #ifndef HAVE_MLX5DV_DR_ESWITCH
  44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  46 #endif
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR
  50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  51 #endif
  52
  53 /* VLAN header definitions */
  54 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  55 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  56 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  57 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  58 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  59
  60 union flow_dv_attr {
  61         struct {
  62                 uint32_t valid:1;
  63                 uint32_t ipv4:1;
  64                 uint32_t ipv6:1;
  65                 uint32_t tcp:1;
  66                 uint32_t udp:1;
  67                 uint32_t reserved:27;
  68         };
  69         uint32_t attr;
  70 };
  71
  72 static int
  73 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
  74                              struct mlx5_flow_tbl_resource *tbl);
  75
  76 static int
  77 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev);
  78
  79 static int
  80 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  81                                       uint32_t encap_decap_idx);
  82
  83 static int
  84 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  85                                         uint32_t port_id);
  86
  87 /**
  88  * Initialize flow attributes structure according to flow items' types.
  89  *
  90  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  91  * mode. For tunnel mode, the items to be modified are the outermost ones.
  92  *
  93  * @param[in] item
  94  *   Pointer to item specification.
  95  * @param[out] attr
  96  *   Pointer to flow attributes structure.
  97  * @param[in] dev_flow
  98  *   Pointer to the sub flow.
  99  * @param[in] tunnel_decap
 100  *   Whether action is after tunnel decapsulation.
 101  */
 102 static void
 103 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 104                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 105 {
 106         uint64_t layers = dev_flow->handle->layers;
 107
 108         /*
 109          * If layers is already initialized, it means this dev_flow is the
 110          * suffix flow, the layers flags is set by the prefix flow. Need to
 111          * use the layer flags from prefix flow as the suffix flow may not
 112          * have the user defined items as the flow is split.
 113          */
 114         if (layers) {
 115                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 116                         attr->ipv4 = 1;
 117                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 118                         attr->ipv6 = 1;
 119                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 120                         attr->tcp = 1;
 121                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 122                         attr->udp = 1;
 123                 attr->valid = 1;
 124                 return;
 125         }
 126         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 127                 uint8_t next_protocol = 0xff;
 128                 switch (item->type) {
 129                 case RTE_FLOW_ITEM_TYPE_GRE:
 130                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 131                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 132                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 133                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 134                 case RTE_FLOW_ITEM_TYPE_MPLS:
 135                         if (tunnel_decap)
 136                                 attr->attr = 0;
 137                         break;
 138                 case RTE_FLOW_ITEM_TYPE_IPV4:
 139                         if (!attr->ipv6)
 140                                 attr->ipv4 = 1;
 141                         if (item->mask != NULL &&
 142                             ((const struct rte_flow_item_ipv4 *)
 143                             item->mask)->hdr.next_proto_id)
 144                                 next_protocol =
 145                                     ((const struct rte_flow_item_ipv4 *)
 146                                       (item->spec))->hdr.next_proto_id &
 147                                     ((const struct rte_flow_item_ipv4 *)
 148                                       (item->mask))->hdr.next_proto_id;
 149                         if ((next_protocol == IPPROTO_IPIP ||
 150                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 151                                 attr->attr = 0;
 152                         break;
 153                 case RTE_FLOW_ITEM_TYPE_IPV6:
 154                         if (!attr->ipv4)
 155                                 attr->ipv6 = 1;
 156                         if (item->mask != NULL &&
 157                             ((const struct rte_flow_item_ipv6 *)
 158                             item->mask)->hdr.proto)
 159                                 next_protocol =
 160                                     ((const struct rte_flow_item_ipv6 *)
 161                                       (item->spec))->hdr.proto &
 162                                     ((const struct rte_flow_item_ipv6 *)
 163                                       (item->mask))->hdr.proto;
 164                         if ((next_protocol == IPPROTO_IPIP ||
 165                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 166                                 attr->attr = 0;
 167                         break;
 168                 case RTE_FLOW_ITEM_TYPE_UDP:
 169                         if (!attr->tcp)
 170                                 attr->udp = 1;
 171                         break;
 172                 case RTE_FLOW_ITEM_TYPE_TCP:
 173                         if (!attr->udp)
 174                                 attr->tcp = 1;
 175                         break;
 176                 default:
 177                         break;
 178                 }
 179         }
 180         attr->valid = 1;
 181 }
 182
 183 /**
 184  * Convert rte_mtr_color to mlx5 color.
 185  *
 186  * @param[in] rcol
 187  *   rte_mtr_color.
 188  *
 189  * @return
 190  *   mlx5 color.
 191  */
 192 static int
 193 rte_col_2_mlx5_col(enum rte_color rcol)
 194 {
 195         switch (rcol) {
 196         case RTE_COLOR_GREEN:
 197                 return MLX5_FLOW_COLOR_GREEN;
 198         case RTE_COLOR_YELLOW:
 199                 return MLX5_FLOW_COLOR_YELLOW;
 200         case RTE_COLOR_RED:
 201                 return MLX5_FLOW_COLOR_RED;
 202         default:
 203                 break;
 204         }
 205         return MLX5_FLOW_COLOR_UNDEFINED;
 206 }
 207
 208 struct field_modify_info {
 209         uint32_t size; /* Size of field in protocol header, in bytes. */
 210         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 211         enum mlx5_modification_field id;
 212 };
 213
 214 struct field_modify_info modify_eth[] = {
 215         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 216         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 217         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 218         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 219         {0, 0, 0},
 220 };
 221
 222 struct field_modify_info modify_vlan_out_first_vid[] = {
 223         /* Size in bits !!! */
 224         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 225         {0, 0, 0},
 226 };
 227
 228 struct field_modify_info modify_ipv4[] = {
 229         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 230         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 231         {4, 12, MLX5_MODI_OUT_SIPV4},
 232         {4, 16, MLX5_MODI_OUT_DIPV4},
 233         {0, 0, 0},
 234 };
 235
 236 struct field_modify_info modify_ipv6[] = {
 237         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 238         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 239         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 240         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 241         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 242         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 243         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 244         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 245         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 246         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 247         {0, 0, 0},
 248 };
 249
 250 struct field_modify_info modify_udp[] = {
 251         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 252         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 253         {0, 0, 0},
 254 };
 255
 256 struct field_modify_info modify_tcp[] = {
 257         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 258         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 259         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 260         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 261         {0, 0, 0},
 262 };
 263
 264 static void
 265 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 266                           uint8_t next_protocol, uint64_t *item_flags,
 267                           int *tunnel)
 268 {
 269         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 270                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 271         if (next_protocol == IPPROTO_IPIP) {
 272                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 273                 *tunnel = 1;
 274         }
 275         if (next_protocol == IPPROTO_IPV6) {
 276                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 277                 *tunnel = 1;
 278         }
 279 }
 280
 281 /**
 282  * Acquire the synchronizing object to protect multithreaded access
 283  * to shared dv context. Lock occurs only if context is actually
 284  * shared, i.e. we have multiport IB device and representors are
 285  * created.
 286  *
 287  * @param[in] dev
 288  *   Pointer to the rte_eth_dev structure.
 289  */
 290 static void
 291 flow_dv_shared_lock(struct rte_eth_dev *dev)
 292 {
 293         struct mlx5_priv *priv = dev->data->dev_private;
 294         struct mlx5_dev_ctx_shared *sh = priv->sh;
 295
 296         if (sh->dv_refcnt > 1) {
 297                 int ret;
 298
 299                 ret = pthread_mutex_lock(&sh->dv_mutex);
 300                 MLX5_ASSERT(!ret);
 301                 (void)ret;
 302         }
 303 }
 304
 305 static void
 306 flow_dv_shared_unlock(struct rte_eth_dev *dev)
 307 {
 308         struct mlx5_priv *priv = dev->data->dev_private;
 309         struct mlx5_dev_ctx_shared *sh = priv->sh;
 310
 311         if (sh->dv_refcnt > 1) {
 312                 int ret;
 313
 314                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 315                 MLX5_ASSERT(!ret);
 316                 (void)ret;
 317         }
 318 }
 319
 320 /* Update VLAN's VID/PCP based on input rte_flow_action.
 321  *
 322  * @param[in] action
 323  *   Pointer to struct rte_flow_action.
 324  * @param[out] vlan
 325  *   Pointer to struct rte_vlan_hdr.
 326  */
 327 static void
 328 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 329                          struct rte_vlan_hdr *vlan)
 330 {
 331         uint16_t vlan_tci;
 332         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 333                 vlan_tci =
 334                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 335                                                action->conf)->vlan_pcp;
 336                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 337                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 338                 vlan->vlan_tci |= vlan_tci;
 339         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 340                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 341                 vlan->vlan_tci |= rte_be_to_cpu_16
 342                     (((const struct rte_flow_action_of_set_vlan_vid *)
 343                                              action->conf)->vlan_vid);
 344         }
 345 }
 346
 347 /**
 348  * Fetch 1, 2, 3 or 4 byte field from the byte array
 349  * and return as unsigned integer in host-endian format.
 350  *
 351  * @param[in] data
 352  *   Pointer to data array.
 353  * @param[in] size
 354  *   Size of field to extract.
 355  *
 356  * @return
 357  *   converted field in host endian format.
 358  */
 359 static inline uint32_t
 360 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 361 {
 362         uint32_t ret;
 363
 364         switch (size) {
 365         case 1:
 366                 ret = *data;
 367                 break;
 368         case 2:
 369                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 370                 break;
 371         case 3:
 372                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 373                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 374                 break;
 375         case 4:
 376                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 377                 break;
 378         default:
 379                 MLX5_ASSERT(false);
 380                 ret = 0;
 381                 break;
 382         }
 383         return ret;
 384 }
 385
 386 /**
 387  * Convert modify-header action to DV specification.
 388  *
 389  * Data length of each action is determined by provided field description
 390  * and the item mask. Data bit offset and width of each action is determined
 391  * by provided item mask.
 392  *
 393  * @param[in] item
 394  *   Pointer to item specification.
 395  * @param[in] field
 396  *   Pointer to field modification information.
 397  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 398  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 399  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 400  * @param[in] dcopy
 401  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 402  *   Negative offset value sets the same offset as source offset.
 403  *   size field is ignored, value is taken from source field.
 404  * @param[in,out] resource
 405  *   Pointer to the modify-header resource.
 406  * @param[in] type
 407  *   Type of modification.
 408  * @param[out] error
 409  *   Pointer to the error structure.
 410  *
 411  * @return
 412  *   0 on success, a negative errno value otherwise and rte_errno is set.
 413  */
 414 static int
 415 flow_dv_convert_modify_action(struct rte_flow_item *item,
 416                               struct field_modify_info *field,
 417                               struct field_modify_info *dcopy,
 418                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 419                               uint32_t type, struct rte_flow_error *error)
 420 {
 421         uint32_t i = resource->actions_num;
 422         struct mlx5_modification_cmd *actions = resource->actions;
 423
 424         /*
 425          * The item and mask are provided in big-endian format.
 426          * The fields should be presented as in big-endian format either.
 427          * Mask must be always present, it defines the actual field width.
 428          */
 429         MLX5_ASSERT(item->mask);
 430         MLX5_ASSERT(field->size);
 431         do {
 432                 unsigned int size_b;
 433                 unsigned int off_b;
 434                 uint32_t mask;
 435                 uint32_t data;
 436
 437                 if (i >= MLX5_MAX_MODIFY_NUM)
 438                         return rte_flow_error_set(error, EINVAL,
 439                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 440                                  "too many items to modify");
 441                 /* Fetch variable byte size mask from the array. */
 442                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 443                                            field->offset, field->size);
 444                 if (!mask) {
 445                         ++field;
 446                         continue;
 447                 }
 448                 /* Deduce actual data width in bits from mask value. */
 449                 off_b = rte_bsf32(mask);
 450                 size_b = sizeof(uint32_t) * CHAR_BIT -
 451                          off_b - __builtin_clz(mask);
 452                 MLX5_ASSERT(size_b);
 453                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 454                 actions[i] = (struct mlx5_modification_cmd) {
 455                         .action_type = type,
 456                         .field = field->id,
 457                         .offset = off_b,
 458                         .length = size_b,
 459                 };
 460                 /* Convert entire record to expected big-endian format. */
 461                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 462                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 463                         MLX5_ASSERT(dcopy);
 464                         actions[i].dst_field = dcopy->id;
 465                         actions[i].dst_offset =
 466                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 467                         /* Convert entire record to big-endian format. */
 468                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 469                 } else {
 470                         MLX5_ASSERT(item->spec);
 471                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 472                                                    field->offset, field->size);
 473                         /* Shift out the trailing masked bits from data. */
 474                         data = (data & mask) >> off_b;
 475                         actions[i].data1 = rte_cpu_to_be_32(data);
 476                 }
 477                 ++i;
 478                 ++field;
 479         } while (field->size);
 480         if (resource->actions_num == i)
 481                 return rte_flow_error_set(error, EINVAL,
 482                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 483                                           "invalid modification flow item");
 484         resource->actions_num = i;
 485         return 0;
 486 }
 487
 488 /**
 489  * Convert modify-header set IPv4 address action to DV specification.
 490  *
 491  * @param[in,out] resource
 492  *   Pointer to the modify-header resource.
 493  * @param[in] action
 494  *   Pointer to action specification.
 495  * @param[out] error
 496  *   Pointer to the error structure.
 497  *
 498  * @return
 499  *   0 on success, a negative errno value otherwise and rte_errno is set.
 500  */
 501 static int
 502 flow_dv_convert_action_modify_ipv4
 503                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 504                          const struct rte_flow_action *action,
 505                          struct rte_flow_error *error)
 506 {
 507         const struct rte_flow_action_set_ipv4 *conf =
 508                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 509         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 510         struct rte_flow_item_ipv4 ipv4;
 511         struct rte_flow_item_ipv4 ipv4_mask;
 512
 513         memset(&ipv4, 0, sizeof(ipv4));
 514         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 515         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 516                 ipv4.hdr.src_addr = conf->ipv4_addr;
 517                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 518         } else {
 519                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 520                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 521         }
 522         item.spec = &ipv4;
 523         item.mask = &ipv4_mask;
 524         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 525                                              MLX5_MODIFICATION_TYPE_SET, error);
 526 }
 527
 528 /**
 529  * Convert modify-header set IPv6 address action to DV specification.
 530  *
 531  * @param[in,out] resource
 532  *   Pointer to the modify-header resource.
 533  * @param[in] action
 534  *   Pointer to action specification.
 535  * @param[out] error
 536  *   Pointer to the error structure.
 537  *
 538  * @return
 539  *   0 on success, a negative errno value otherwise and rte_errno is set.
 540  */
 541 static int
 542 flow_dv_convert_action_modify_ipv6
 543                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 544                          const struct rte_flow_action *action,
 545                          struct rte_flow_error *error)
 546 {
 547         const struct rte_flow_action_set_ipv6 *conf =
 548                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 549         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 550         struct rte_flow_item_ipv6 ipv6;
 551         struct rte_flow_item_ipv6 ipv6_mask;
 552
 553         memset(&ipv6, 0, sizeof(ipv6));
 554         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 555         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 556                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 557                        sizeof(ipv6.hdr.src_addr));
 558                 memcpy(&ipv6_mask.hdr.src_addr,
 559                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 560                        sizeof(ipv6.hdr.src_addr));
 561         } else {
 562                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 563                        sizeof(ipv6.hdr.dst_addr));
 564                 memcpy(&ipv6_mask.hdr.dst_addr,
 565                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 566                        sizeof(ipv6.hdr.dst_addr));
 567         }
 568         item.spec = &ipv6;
 569         item.mask = &ipv6_mask;
 570         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 571                                              MLX5_MODIFICATION_TYPE_SET, error);
 572 }
 573
 574 /**
 575  * Convert modify-header set MAC address action to DV specification.
 576  *
 577  * @param[in,out] resource
 578  *   Pointer to the modify-header resource.
 579  * @param[in] action
 580  *   Pointer to action specification.
 581  * @param[out] error
 582  *   Pointer to the error structure.
 583  *
 584  * @return
 585  *   0 on success, a negative errno value otherwise and rte_errno is set.
 586  */
 587 static int
 588 flow_dv_convert_action_modify_mac
 589                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 590                          const struct rte_flow_action *action,
 591                          struct rte_flow_error *error)
 592 {
 593         const struct rte_flow_action_set_mac *conf =
 594                 (const struct rte_flow_action_set_mac *)(action->conf);
 595         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 596         struct rte_flow_item_eth eth;
 597         struct rte_flow_item_eth eth_mask;
 598
 599         memset(&eth, 0, sizeof(eth));
 600         memset(&eth_mask, 0, sizeof(eth_mask));
 601         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 602                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 603                        sizeof(eth.src.addr_bytes));
 604                 memcpy(&eth_mask.src.addr_bytes,
 605                        &rte_flow_item_eth_mask.src.addr_bytes,
 606                        sizeof(eth_mask.src.addr_bytes));
 607         } else {
 608                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 609                        sizeof(eth.dst.addr_bytes));
 610                 memcpy(&eth_mask.dst.addr_bytes,
 611                        &rte_flow_item_eth_mask.dst.addr_bytes,
 612                        sizeof(eth_mask.dst.addr_bytes));
 613         }
 614         item.spec = &eth;
 615         item.mask = &eth_mask;
 616         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 617                                              MLX5_MODIFICATION_TYPE_SET, error);
 618 }
 619
 620 /**
 621  * Convert modify-header set VLAN VID action to DV specification.
 622  *
 623  * @param[in,out] resource
 624  *   Pointer to the modify-header resource.
 625  * @param[in] action
 626  *   Pointer to action specification.
 627  * @param[out] error
 628  *   Pointer to the error structure.
 629  *
 630  * @return
 631  *   0 on success, a negative errno value otherwise and rte_errno is set.
 632  */
 633 static int
 634 flow_dv_convert_action_modify_vlan_vid
 635                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 636                          const struct rte_flow_action *action,
 637                          struct rte_flow_error *error)
 638 {
 639         const struct rte_flow_action_of_set_vlan_vid *conf =
 640                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 641         int i = resource->actions_num;
 642         struct mlx5_modification_cmd *actions = resource->actions;
 643         struct field_modify_info *field = modify_vlan_out_first_vid;
 644
 645         if (i >= MLX5_MAX_MODIFY_NUM)
 646                 return rte_flow_error_set(error, EINVAL,
 647                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 648                          "too many items to modify");
 649         actions[i] = (struct mlx5_modification_cmd) {
 650                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 651                 .field = field->id,
 652                 .length = field->size,
 653                 .offset = field->offset,
 654         };
 655         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 656         actions[i].data1 = conf->vlan_vid;
 657         actions[i].data1 = actions[i].data1 << 16;
 658         resource->actions_num = ++i;
 659         return 0;
 660 }
 661
 662 /**
 663  * Convert modify-header set TP action to DV specification.
 664  *
 665  * @param[in,out] resource
 666  *   Pointer to the modify-header resource.
 667  * @param[in] action
 668  *   Pointer to action specification.
 669  * @param[in] items
 670  *   Pointer to rte_flow_item objects list.
 671  * @param[in] attr
 672  *   Pointer to flow attributes structure.
 673  * @param[in] dev_flow
 674  *   Pointer to the sub flow.
 675  * @param[in] tunnel_decap
 676  *   Whether action is after tunnel decapsulation.
 677  * @param[out] error
 678  *   Pointer to the error structure.
 679  *
 680  * @return
 681  *   0 on success, a negative errno value otherwise and rte_errno is set.
 682  */
 683 static int
 684 flow_dv_convert_action_modify_tp
 685                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 686                          const struct rte_flow_action *action,
 687                          const struct rte_flow_item *items,
 688                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 689                          bool tunnel_decap, struct rte_flow_error *error)
 690 {
 691         const struct rte_flow_action_set_tp *conf =
 692                 (const struct rte_flow_action_set_tp *)(action->conf);
 693         struct rte_flow_item item;
 694         struct rte_flow_item_udp udp;
 695         struct rte_flow_item_udp udp_mask;
 696         struct rte_flow_item_tcp tcp;
 697         struct rte_flow_item_tcp tcp_mask;
 698         struct field_modify_info *field;
 699
 700         if (!attr->valid)
 701                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 702         if (attr->udp) {
 703                 memset(&udp, 0, sizeof(udp));
 704                 memset(&udp_mask, 0, sizeof(udp_mask));
 705                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 706                         udp.hdr.src_port = conf->port;
 707                         udp_mask.hdr.src_port =
 708                                         rte_flow_item_udp_mask.hdr.src_port;
 709                 } else {
 710                         udp.hdr.dst_port = conf->port;
 711                         udp_mask.hdr.dst_port =
 712                                         rte_flow_item_udp_mask.hdr.dst_port;
 713                 }
 714                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 715                 item.spec = &udp;
 716                 item.mask = &udp_mask;
 717                 field = modify_udp;
 718         } else {
 719                 MLX5_ASSERT(attr->tcp);
 720                 memset(&tcp, 0, sizeof(tcp));
 721                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 722                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 723                         tcp.hdr.src_port = conf->port;
 724                         tcp_mask.hdr.src_port =
 725                                         rte_flow_item_tcp_mask.hdr.src_port;
 726                 } else {
 727                         tcp.hdr.dst_port = conf->port;
 728                         tcp_mask.hdr.dst_port =
 729                                         rte_flow_item_tcp_mask.hdr.dst_port;
 730                 }
 731                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 732                 item.spec = &tcp;
 733                 item.mask = &tcp_mask;
 734                 field = modify_tcp;
 735         }
 736         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 737                                              MLX5_MODIFICATION_TYPE_SET, error);
 738 }
 739
 740 /**
 741  * Convert modify-header set TTL action to DV specification.
 742  *
 743  * @param[in,out] resource
 744  *   Pointer to the modify-header resource.
 745  * @param[in] action
 746  *   Pointer to action specification.
 747  * @param[in] items
 748  *   Pointer to rte_flow_item objects list.
 749  * @param[in] attr
 750  *   Pointer to flow attributes structure.
 751  * @param[in] dev_flow
 752  *   Pointer to the sub flow.
 753  * @param[in] tunnel_decap
 754  *   Whether action is after tunnel decapsulation.
 755  * @param[out] error
 756  *   Pointer to the error structure.
 757  *
 758  * @return
 759  *   0 on success, a negative errno value otherwise and rte_errno is set.
 760  */
 761 static int
 762 flow_dv_convert_action_modify_ttl
 763                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 764                          const struct rte_flow_action *action,
 765                          const struct rte_flow_item *items,
 766                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 767                          bool tunnel_decap, struct rte_flow_error *error)
 768 {
 769         const struct rte_flow_action_set_ttl *conf =
 770                 (const struct rte_flow_action_set_ttl *)(action->conf);
 771         struct rte_flow_item item;
 772         struct rte_flow_item_ipv4 ipv4;
 773         struct rte_flow_item_ipv4 ipv4_mask;
 774         struct rte_flow_item_ipv6 ipv6;
 775         struct rte_flow_item_ipv6 ipv6_mask;
 776         struct field_modify_info *field;
 777
 778         if (!attr->valid)
 779                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 780         if (attr->ipv4) {
 781                 memset(&ipv4, 0, sizeof(ipv4));
 782                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 783                 ipv4.hdr.time_to_live = conf->ttl_value;
 784                 ipv4_mask.hdr.time_to_live = 0xFF;
 785                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 786                 item.spec = &ipv4;
 787                 item.mask = &ipv4_mask;
 788                 field = modify_ipv4;
 789         } else {
 790                 MLX5_ASSERT(attr->ipv6);
 791                 memset(&ipv6, 0, sizeof(ipv6));
 792                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 793                 ipv6.hdr.hop_limits = conf->ttl_value;
 794                 ipv6_mask.hdr.hop_limits = 0xFF;
 795                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 796                 item.spec = &ipv6;
 797                 item.mask = &ipv6_mask;
 798                 field = modify_ipv6;
 799         }
 800         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 801                                              MLX5_MODIFICATION_TYPE_SET, error);
 802 }
 803
 804 /**
 805  * Convert modify-header decrement TTL action to DV specification.
 806  *
 807  * @param[in,out] resource
 808  *   Pointer to the modify-header resource.
 809  * @param[in] action
 810  *   Pointer to action specification.
 811  * @param[in] items
 812  *   Pointer to rte_flow_item objects list.
 813  * @param[in] attr
 814  *   Pointer to flow attributes structure.
 815  * @param[in] dev_flow
 816  *   Pointer to the sub flow.
 817  * @param[in] tunnel_decap
 818  *   Whether action is after tunnel decapsulation.
 819  * @param[out] error
 820  *   Pointer to the error structure.
 821  *
 822  * @return
 823  *   0 on success, a negative errno value otherwise and rte_errno is set.
 824  */
 825 static int
 826 flow_dv_convert_action_modify_dec_ttl
 827                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 828                          const struct rte_flow_item *items,
 829                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 830                          bool tunnel_decap, struct rte_flow_error *error)
 831 {
 832         struct rte_flow_item item;
 833         struct rte_flow_item_ipv4 ipv4;
 834         struct rte_flow_item_ipv4 ipv4_mask;
 835         struct rte_flow_item_ipv6 ipv6;
 836         struct rte_flow_item_ipv6 ipv6_mask;
 837         struct field_modify_info *field;
 838
 839         if (!attr->valid)
 840                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 841         if (attr->ipv4) {
 842                 memset(&ipv4, 0, sizeof(ipv4));
 843                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 844                 ipv4.hdr.time_to_live = 0xFF;
 845                 ipv4_mask.hdr.time_to_live = 0xFF;
 846                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 847                 item.spec = &ipv4;
 848                 item.mask = &ipv4_mask;
 849                 field = modify_ipv4;
 850         } else {
 851                 MLX5_ASSERT(attr->ipv6);
 852                 memset(&ipv6, 0, sizeof(ipv6));
 853                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 854                 ipv6.hdr.hop_limits = 0xFF;
 855                 ipv6_mask.hdr.hop_limits = 0xFF;
 856                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 857                 item.spec = &ipv6;
 858                 item.mask = &ipv6_mask;
 859                 field = modify_ipv6;
 860         }
 861         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 862                                              MLX5_MODIFICATION_TYPE_ADD, error);
 863 }
 864
 865 /**
 866  * Convert modify-header increment/decrement TCP Sequence number
 867  * to DV specification.
 868  *
 869  * @param[in,out] resource
 870  *   Pointer to the modify-header resource.
 871  * @param[in] action
 872  *   Pointer to action specification.
 873  * @param[out] error
 874  *   Pointer to the error structure.
 875  *
 876  * @return
 877  *   0 on success, a negative errno value otherwise and rte_errno is set.
 878  */
 879 static int
 880 flow_dv_convert_action_modify_tcp_seq
 881                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 882                          const struct rte_flow_action *action,
 883                          struct rte_flow_error *error)
 884 {
 885         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 886         uint64_t value = rte_be_to_cpu_32(*conf);
 887         struct rte_flow_item item;
 888         struct rte_flow_item_tcp tcp;
 889         struct rte_flow_item_tcp tcp_mask;
 890
 891         memset(&tcp, 0, sizeof(tcp));
 892         memset(&tcp_mask, 0, sizeof(tcp_mask));
 893         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 894                 /*
 895                  * The HW has no decrement operation, only increment operation.
 896                  * To simulate decrement X from Y using increment operation
 897                  * we need to add UINT32_MAX X times to Y.
 898                  * Each adding of UINT32_MAX decrements Y by 1.
 899                  */
 900                 value *= UINT32_MAX;
 901         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 902         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 903         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 904         item.spec = &tcp;
 905         item.mask = &tcp_mask;
 906         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 907                                              MLX5_MODIFICATION_TYPE_ADD, error);
 908 }
 909
 910 /**
 911  * Convert modify-header increment/decrement TCP Acknowledgment number
 912  * to DV specification.
 913  *
 914  * @param[in,out] resource
 915  *   Pointer to the modify-header resource.
 916  * @param[in] action
 917  *   Pointer to action specification.
 918  * @param[out] error
 919  *   Pointer to the error structure.
 920  *
 921  * @return
 922  *   0 on success, a negative errno value otherwise and rte_errno is set.
 923  */
 924 static int
 925 flow_dv_convert_action_modify_tcp_ack
 926                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 927                          const struct rte_flow_action *action,
 928                          struct rte_flow_error *error)
 929 {
 930         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 931         uint64_t value = rte_be_to_cpu_32(*conf);
 932         struct rte_flow_item item;
 933         struct rte_flow_item_tcp tcp;
 934         struct rte_flow_item_tcp tcp_mask;
 935
 936         memset(&tcp, 0, sizeof(tcp));
 937         memset(&tcp_mask, 0, sizeof(tcp_mask));
 938         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 939                 /*
 940                  * The HW has no decrement operation, only increment operation.
 941                  * To simulate decrement X from Y using increment operation
 942                  * we need to add UINT32_MAX X times to Y.
 943                  * Each adding of UINT32_MAX decrements Y by 1.
 944                  */
 945                 value *= UINT32_MAX;
 946         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 947         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 948         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 949         item.spec = &tcp;
 950         item.mask = &tcp_mask;
 951         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 952                                              MLX5_MODIFICATION_TYPE_ADD, error);
 953 }
 954
 955 static enum mlx5_modification_field reg_to_field[] = {
 956         [REG_NON] = MLX5_MODI_OUT_NONE,
 957         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 958         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 959         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 960         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 961         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 962         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 963         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 964         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 965         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 966         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 967 };
 968
 969 /**
 970  * Convert register set to DV specification.
 971  *
 972  * @param[in,out] resource
 973  *   Pointer to the modify-header resource.
 974  * @param[in] action
 975  *   Pointer to action specification.
 976  * @param[out] error
 977  *   Pointer to the error structure.
 978  *
 979  * @return
 980  *   0 on success, a negative errno value otherwise and rte_errno is set.
 981  */
 982 static int
 983 flow_dv_convert_action_set_reg
 984                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 985                          const struct rte_flow_action *action,
 986                          struct rte_flow_error *error)
 987 {
 988         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 989         struct mlx5_modification_cmd *actions = resource->actions;
 990         uint32_t i = resource->actions_num;
 991
 992         if (i >= MLX5_MAX_MODIFY_NUM)
 993                 return rte_flow_error_set(error, EINVAL,
 994                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 995                                           "too many items to modify");
 996         MLX5_ASSERT(conf->id != REG_NON);
 997         MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field));
 998         actions[i] = (struct mlx5_modification_cmd) {
 999                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1000                 .field = reg_to_field[conf->id],
1001         };
1002         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1003         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1004         ++i;
1005         resource->actions_num = i;
1006         return 0;
1007 }
1008
1009 /**
1010  * Convert SET_TAG action to DV specification.
1011  *
1012  * @param[in] dev
1013  *   Pointer to the rte_eth_dev structure.
1014  * @param[in,out] resource
1015  *   Pointer to the modify-header resource.
1016  * @param[in] conf
1017  *   Pointer to action specification.
1018  * @param[out] error
1019  *   Pointer to the error structure.
1020  *
1021  * @return
1022  *   0 on success, a negative errno value otherwise and rte_errno is set.
1023  */
1024 static int
1025 flow_dv_convert_action_set_tag
1026                         (struct rte_eth_dev *dev,
1027                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1028                          const struct rte_flow_action_set_tag *conf,
1029                          struct rte_flow_error *error)
1030 {
1031         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1032         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1033         struct rte_flow_item item = {
1034                 .spec = &data,
1035                 .mask = &mask,
1036         };
1037         struct field_modify_info reg_c_x[] = {
1038                 [1] = {0, 0, 0},
1039         };
1040         enum mlx5_modification_field reg_type;
1041         int ret;
1042
1043         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1044         if (ret < 0)
1045                 return ret;
1046         MLX5_ASSERT(ret != REG_NON);
1047         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1048         reg_type = reg_to_field[ret];
1049         MLX5_ASSERT(reg_type > 0);
1050         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1051         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1052                                              MLX5_MODIFICATION_TYPE_SET, error);
1053 }
1054
1055 /**
1056  * Convert internal COPY_REG action to DV specification.
1057  *
1058  * @param[in] dev
1059  *   Pointer to the rte_eth_dev structure.
1060  * @param[in,out] res
1061  *   Pointer to the modify-header resource.
1062  * @param[in] action
1063  *   Pointer to action specification.
1064  * @param[out] error
1065  *   Pointer to the error structure.
1066  *
1067  * @return
1068  *   0 on success, a negative errno value otherwise and rte_errno is set.
1069  */
1070 static int
1071 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1072                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1073                                  const struct rte_flow_action *action,
1074                                  struct rte_flow_error *error)
1075 {
1076         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1077         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1078         struct rte_flow_item item = {
1079                 .spec = NULL,
1080                 .mask = &mask,
1081         };
1082         struct field_modify_info reg_src[] = {
1083                 {4, 0, reg_to_field[conf->src]},
1084                 {0, 0, 0},
1085         };
1086         struct field_modify_info reg_dst = {
1087                 .offset = 0,
1088                 .id = reg_to_field[conf->dst],
1089         };
1090         /* Adjust reg_c[0] usage according to reported mask. */
1091         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1092                 struct mlx5_priv *priv = dev->data->dev_private;
1093                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1094
1095                 MLX5_ASSERT(reg_c0);
1096                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1097                 if (conf->dst == REG_C_0) {
1098                         /* Copy to reg_c[0], within mask only. */
1099                         reg_dst.offset = rte_bsf32(reg_c0);
1100                         /*
1101                          * Mask is ignoring the enianness, because
1102                          * there is no conversion in datapath.
1103                          */
1104 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1105                         /* Copy from destination lower bits to reg_c[0]. */
1106                         mask = reg_c0 >> reg_dst.offset;
1107 #else
1108                         /* Copy from destination upper bits to reg_c[0]. */
1109                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1110                                           rte_fls_u32(reg_c0));
1111 #endif
1112                 } else {
1113                         mask = rte_cpu_to_be_32(reg_c0);
1114 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1115                         /* Copy from reg_c[0] to destination lower bits. */
1116                         reg_dst.offset = 0;
1117 #else
1118                         /* Copy from reg_c[0] to destination upper bits. */
1119                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1120                                          (rte_fls_u32(reg_c0) -
1121                                           rte_bsf32(reg_c0));
1122 #endif
1123                 }
1124         }
1125         return flow_dv_convert_modify_action(&item,
1126                                              reg_src, &reg_dst, res,
1127                                              MLX5_MODIFICATION_TYPE_COPY,
1128                                              error);
1129 }
1130
1131 /**
1132  * Convert MARK action to DV specification. This routine is used
1133  * in extensive metadata only and requires metadata register to be
1134  * handled. In legacy mode hardware tag resource is engaged.
1135  *
1136  * @param[in] dev
1137  *   Pointer to the rte_eth_dev structure.
1138  * @param[in] conf
1139  *   Pointer to MARK action specification.
1140  * @param[in,out] resource
1141  *   Pointer to the modify-header resource.
1142  * @param[out] error
1143  *   Pointer to the error structure.
1144  *
1145  * @return
1146  *   0 on success, a negative errno value otherwise and rte_errno is set.
1147  */
1148 static int
1149 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1150                             const struct rte_flow_action_mark *conf,
1151                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1152                             struct rte_flow_error *error)
1153 {
1154         struct mlx5_priv *priv = dev->data->dev_private;
1155         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1156                                            priv->sh->dv_mark_mask);
1157         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1158         struct rte_flow_item item = {
1159                 .spec = &data,
1160                 .mask = &mask,
1161         };
1162         struct field_modify_info reg_c_x[] = {
1163                 [1] = {0, 0, 0},
1164         };
1165         int reg;
1166
1167         if (!mask)
1168                 return rte_flow_error_set(error, EINVAL,
1169                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1170                                           NULL, "zero mark action mask");
1171         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1172         if (reg < 0)
1173                 return reg;
1174         MLX5_ASSERT(reg > 0);
1175         if (reg == REG_C_0) {
1176                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1177                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1178
1179                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1180                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1181                 mask = rte_cpu_to_be_32(mask << shl_c0);
1182         }
1183         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1184         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1185                                              MLX5_MODIFICATION_TYPE_SET, error);
1186 }
1187
1188 /**
1189  * Get metadata register index for specified steering domain.
1190  *
1191  * @param[in] dev
1192  *   Pointer to the rte_eth_dev structure.
1193  * @param[in] attr
1194  *   Attributes of flow to determine steering domain.
1195  * @param[out] error
1196  *   Pointer to the error structure.
1197  *
1198  * @return
1199  *   positive index on success, a negative errno value otherwise
1200  *   and rte_errno is set.
1201  */
1202 static enum modify_reg
1203 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1204                          const struct rte_flow_attr *attr,
1205                          struct rte_flow_error *error)
1206 {
1207         int reg =
1208                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1209                                           MLX5_METADATA_FDB :
1210                                             attr->egress ?
1211                                             MLX5_METADATA_TX :
1212                                             MLX5_METADATA_RX, 0, error);
1213         if (reg < 0)
1214                 return rte_flow_error_set(error,
1215                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1216                                           NULL, "unavailable "
1217                                           "metadata register");
1218         return reg;
1219 }
1220
1221 /**
1222  * Convert SET_META action to DV specification.
1223  *
1224  * @param[in] dev
1225  *   Pointer to the rte_eth_dev structure.
1226  * @param[in,out] resource
1227  *   Pointer to the modify-header resource.
1228  * @param[in] attr
1229  *   Attributes of flow that includes this item.
1230  * @param[in] conf
1231  *   Pointer to action specification.
1232  * @param[out] error
1233  *   Pointer to the error structure.
1234  *
1235  * @return
1236  *   0 on success, a negative errno value otherwise and rte_errno is set.
1237  */
1238 static int
1239 flow_dv_convert_action_set_meta
1240                         (struct rte_eth_dev *dev,
1241                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1242                          const struct rte_flow_attr *attr,
1243                          const struct rte_flow_action_set_meta *conf,
1244                          struct rte_flow_error *error)
1245 {
1246         uint32_t data = conf->data;
1247         uint32_t mask = conf->mask;
1248         struct rte_flow_item item = {
1249                 .spec = &data,
1250                 .mask = &mask,
1251         };
1252         struct field_modify_info reg_c_x[] = {
1253                 [1] = {0, 0, 0},
1254         };
1255         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1256
1257         if (reg < 0)
1258                 return reg;
1259         /*
1260          * In datapath code there is no endianness
1261          * coversions for perfromance reasons, all
1262          * pattern conversions are done in rte_flow.
1263          */
1264         if (reg == REG_C_0) {
1265                 struct mlx5_priv *priv = dev->data->dev_private;
1266                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1267                 uint32_t shl_c0;
1268
1269                 MLX5_ASSERT(msk_c0);
1270 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1271                 shl_c0 = rte_bsf32(msk_c0);
1272 #else
1273                 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1274 #endif
1275                 mask <<= shl_c0;
1276                 data <<= shl_c0;
1277                 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1278         }
1279         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1280         /* The routine expects parameters in memory as big-endian ones. */
1281         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1282                                              MLX5_MODIFICATION_TYPE_SET, error);
1283 }
1284
1285 /**
1286  * Convert modify-header set IPv4 DSCP action to DV specification.
1287  *
1288  * @param[in,out] resource
1289  *   Pointer to the modify-header resource.
1290  * @param[in] action
1291  *   Pointer to action specification.
1292  * @param[out] error
1293  *   Pointer to the error structure.
1294  *
1295  * @return
1296  *   0 on success, a negative errno value otherwise and rte_errno is set.
1297  */
1298 static int
1299 flow_dv_convert_action_modify_ipv4_dscp
1300                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1301                          const struct rte_flow_action *action,
1302                          struct rte_flow_error *error)
1303 {
1304         const struct rte_flow_action_set_dscp *conf =
1305                 (const struct rte_flow_action_set_dscp *)(action->conf);
1306         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1307         struct rte_flow_item_ipv4 ipv4;
1308         struct rte_flow_item_ipv4 ipv4_mask;
1309
1310         memset(&ipv4, 0, sizeof(ipv4));
1311         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1312         ipv4.hdr.type_of_service = conf->dscp;
1313         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1314         item.spec = &ipv4;
1315         item.mask = &ipv4_mask;
1316         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1317                                              MLX5_MODIFICATION_TYPE_SET, error);
1318 }
1319
1320 /**
1321  * Convert modify-header set IPv6 DSCP action to DV specification.
1322  *
1323  * @param[in,out] resource
1324  *   Pointer to the modify-header resource.
1325  * @param[in] action
1326  *   Pointer to action specification.
1327  * @param[out] error
1328  *   Pointer to the error structure.
1329  *
1330  * @return
1331  *   0 on success, a negative errno value otherwise and rte_errno is set.
1332  */
1333 static int
1334 flow_dv_convert_action_modify_ipv6_dscp
1335                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1336                          const struct rte_flow_action *action,
1337                          struct rte_flow_error *error)
1338 {
1339         const struct rte_flow_action_set_dscp *conf =
1340                 (const struct rte_flow_action_set_dscp *)(action->conf);
1341         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1342         struct rte_flow_item_ipv6 ipv6;
1343         struct rte_flow_item_ipv6 ipv6_mask;
1344
1345         memset(&ipv6, 0, sizeof(ipv6));
1346         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1347         /*
1348          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1349          * rdma-core only accept the DSCP bits byte aligned start from
1350          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1351          * bits in IPv6 case as rdma-core requires byte aligned value.
1352          */
1353         ipv6.hdr.vtc_flow = conf->dscp;
1354         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1355         item.spec = &ipv6;
1356         item.mask = &ipv6_mask;
1357         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1358                                              MLX5_MODIFICATION_TYPE_SET, error);
1359 }
1360
1361 /**
1362  * Validate MARK item.
1363  *
1364  * @param[in] dev
1365  *   Pointer to the rte_eth_dev structure.
1366  * @param[in] item
1367  *   Item specification.
1368  * @param[in] attr
1369  *   Attributes of flow that includes this item.
1370  * @param[out] error
1371  *   Pointer to error structure.
1372  *
1373  * @return
1374  *   0 on success, a negative errno value otherwise and rte_errno is set.
1375  */
1376 static int
1377 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1378                            const struct rte_flow_item *item,
1379                            const struct rte_flow_attr *attr __rte_unused,
1380                            struct rte_flow_error *error)
1381 {
1382         struct mlx5_priv *priv = dev->data->dev_private;
1383         struct mlx5_dev_config *config = &priv->config;
1384         const struct rte_flow_item_mark *spec = item->spec;
1385         const struct rte_flow_item_mark *mask = item->mask;
1386         const struct rte_flow_item_mark nic_mask = {
1387                 .id = priv->sh->dv_mark_mask,
1388         };
1389         int ret;
1390
1391         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1392                 return rte_flow_error_set(error, ENOTSUP,
1393                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1394                                           "extended metadata feature"
1395                                           " isn't enabled");
1396         if (!mlx5_flow_ext_mreg_supported(dev))
1397                 return rte_flow_error_set(error, ENOTSUP,
1398                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1399                                           "extended metadata register"
1400                                           " isn't supported");
1401         if (!nic_mask.id)
1402                 return rte_flow_error_set(error, ENOTSUP,
1403                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1404                                           "extended metadata register"
1405                                           " isn't available");
1406         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1407         if (ret < 0)
1408                 return ret;
1409         if (!spec)
1410                 return rte_flow_error_set(error, EINVAL,
1411                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1412                                           item->spec,
1413                                           "data cannot be empty");
1414         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1415                 return rte_flow_error_set(error, EINVAL,
1416                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1417                                           &spec->id,
1418                                           "mark id exceeds the limit");
1419         if (!mask)
1420                 mask = &nic_mask;
1421         if (!mask->id)
1422                 return rte_flow_error_set(error, EINVAL,
1423                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1424                                         "mask cannot be zero");
1425
1426         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1427                                         (const uint8_t *)&nic_mask,
1428                                         sizeof(struct rte_flow_item_mark),
1429                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1430         if (ret < 0)
1431                 return ret;
1432         return 0;
1433 }
1434
1435 /**
1436  * Validate META item.
1437  *
1438  * @param[in] dev
1439  *   Pointer to the rte_eth_dev structure.
1440  * @param[in] item
1441  *   Item specification.
1442  * @param[in] attr
1443  *   Attributes of flow that includes this item.
1444  * @param[out] error
1445  *   Pointer to error structure.
1446  *
1447  * @return
1448  *   0 on success, a negative errno value otherwise and rte_errno is set.
1449  */
1450 static int
1451 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1452                            const struct rte_flow_item *item,
1453                            const struct rte_flow_attr *attr,
1454                            struct rte_flow_error *error)
1455 {
1456         struct mlx5_priv *priv = dev->data->dev_private;
1457         struct mlx5_dev_config *config = &priv->config;
1458         const struct rte_flow_item_meta *spec = item->spec;
1459         const struct rte_flow_item_meta *mask = item->mask;
1460         struct rte_flow_item_meta nic_mask = {
1461                 .data = UINT32_MAX
1462         };
1463         int reg;
1464         int ret;
1465
1466         if (!spec)
1467                 return rte_flow_error_set(error, EINVAL,
1468                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1469                                           item->spec,
1470                                           "data cannot be empty");
1471         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1472                 if (!mlx5_flow_ext_mreg_supported(dev))
1473                         return rte_flow_error_set(error, ENOTSUP,
1474                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1475                                           "extended metadata register"
1476                                           " isn't supported");
1477                 reg = flow_dv_get_metadata_reg(dev, attr, error);
1478                 if (reg < 0)
1479                         return reg;
1480                 if (reg == REG_B)
1481                         return rte_flow_error_set(error, ENOTSUP,
1482                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1483                                           "match on reg_b "
1484                                           "isn't supported");
1485                 if (reg != REG_A)
1486                         nic_mask.data = priv->sh->dv_meta_mask;
1487         } else if (attr->transfer) {
1488                 return rte_flow_error_set(error, ENOTSUP,
1489                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1490                                         "extended metadata feature "
1491                                         "should be enabled when "
1492                                         "meta item is requested "
1493                                         "with e-switch mode ");
1494         }
1495         if (!mask)
1496                 mask = &rte_flow_item_meta_mask;
1497         if (!mask->data)
1498                 return rte_flow_error_set(error, EINVAL,
1499                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1500                                         "mask cannot be zero");
1501
1502         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1503                                         (const uint8_t *)&nic_mask,
1504                                         sizeof(struct rte_flow_item_meta),
1505                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1506         return ret;
1507 }
1508
1509 /**
1510  * Validate TAG item.
1511  *
1512  * @param[in] dev
1513  *   Pointer to the rte_eth_dev structure.
1514  * @param[in] item
1515  *   Item specification.
1516  * @param[in] attr
1517  *   Attributes of flow that includes this item.
1518  * @param[out] error
1519  *   Pointer to error structure.
1520  *
1521  * @return
1522  *   0 on success, a negative errno value otherwise and rte_errno is set.
1523  */
1524 static int
1525 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1526                           const struct rte_flow_item *item,
1527                           const struct rte_flow_attr *attr __rte_unused,
1528                           struct rte_flow_error *error)
1529 {
1530         const struct rte_flow_item_tag *spec = item->spec;
1531         const struct rte_flow_item_tag *mask = item->mask;
1532         const struct rte_flow_item_tag nic_mask = {
1533                 .data = RTE_BE32(UINT32_MAX),
1534                 .index = 0xff,
1535         };
1536         int ret;
1537
1538         if (!mlx5_flow_ext_mreg_supported(dev))
1539                 return rte_flow_error_set(error, ENOTSUP,
1540                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1541                                           "extensive metadata register"
1542                                           " isn't supported");
1543         if (!spec)
1544                 return rte_flow_error_set(error, EINVAL,
1545                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1546                                           item->spec,
1547                                           "data cannot be empty");
1548         if (!mask)
1549                 mask = &rte_flow_item_tag_mask;
1550         if (!mask->data)
1551                 return rte_flow_error_set(error, EINVAL,
1552                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1553                                         "mask cannot be zero");
1554
1555         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1556                                         (const uint8_t *)&nic_mask,
1557                                         sizeof(struct rte_flow_item_tag),
1558                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1559         if (ret < 0)
1560                 return ret;
1561         if (mask->index != 0xff)
1562                 return rte_flow_error_set(error, EINVAL,
1563                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1564                                           "partial mask for tag index"
1565                                           " is not supported");
1566         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1567         if (ret < 0)
1568                 return ret;
1569         MLX5_ASSERT(ret != REG_NON);
1570         return 0;
1571 }
1572
1573 /**
1574  * Validate vport item.
1575  *
1576  * @param[in] dev
1577  *   Pointer to the rte_eth_dev structure.
1578  * @param[in] item
1579  *   Item specification.
1580  * @param[in] attr
1581  *   Attributes of flow that includes this item.
1582  * @param[in] item_flags
1583  *   Bit-fields that holds the items detected until now.
1584  * @param[out] error
1585  *   Pointer to error structure.
1586  *
1587  * @return
1588  *   0 on success, a negative errno value otherwise and rte_errno is set.
1589  */
1590 static int
1591 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1592                               const struct rte_flow_item *item,
1593                               const struct rte_flow_attr *attr,
1594                               uint64_t item_flags,
1595                               struct rte_flow_error *error)
1596 {
1597         const struct rte_flow_item_port_id *spec = item->spec;
1598         const struct rte_flow_item_port_id *mask = item->mask;
1599         const struct rte_flow_item_port_id switch_mask = {
1600                         .id = 0xffffffff,
1601         };
1602         struct mlx5_priv *esw_priv;
1603         struct mlx5_priv *dev_priv;
1604         int ret;
1605
1606         if (!attr->transfer)
1607                 return rte_flow_error_set(error, EINVAL,
1608                                           RTE_FLOW_ERROR_TYPE_ITEM,
1609                                           NULL,
1610                                           "match on port id is valid only"
1611                                           " when transfer flag is enabled");
1612         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1613                 return rte_flow_error_set(error, ENOTSUP,
1614                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1615                                           "multiple source ports are not"
1616                                           " supported");
1617         if (!mask)
1618                 mask = &switch_mask;
1619         if (mask->id != 0xffffffff)
1620                 return rte_flow_error_set(error, ENOTSUP,
1621                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1622                                            mask,
1623                                            "no support for partial mask on"
1624                                            " \"id\" field");
1625         ret = mlx5_flow_item_acceptable
1626                                 (item, (const uint8_t *)mask,
1627                                  (const uint8_t *)&rte_flow_item_port_id_mask,
1628                                  sizeof(struct rte_flow_item_port_id),
1629                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1630         if (ret)
1631                 return ret;
1632         if (!spec)
1633                 return 0;
1634         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1635         if (!esw_priv)
1636                 return rte_flow_error_set(error, rte_errno,
1637                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1638                                           "failed to obtain E-Switch info for"
1639                                           " port");
1640         dev_priv = mlx5_dev_to_eswitch_info(dev);
1641         if (!dev_priv)
1642                 return rte_flow_error_set(error, rte_errno,
1643                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1644                                           NULL,
1645                                           "failed to obtain E-Switch info");
1646         if (esw_priv->domain_id != dev_priv->domain_id)
1647                 return rte_flow_error_set(error, EINVAL,
1648                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1649                                           "cannot match on a port from a"
1650                                           " different E-Switch");
1651         return 0;
1652 }
1653
1654 /**
1655  * Validate VLAN item.
1656  *
1657  * @param[in] item
1658  *   Item specification.
1659  * @param[in] item_flags
1660  *   Bit-fields that holds the items detected until now.
1661  * @param[in] dev
1662  *   Ethernet device flow is being created on.
1663  * @param[out] error
1664  *   Pointer to error structure.
1665  *
1666  * @return
1667  *   0 on success, a negative errno value otherwise and rte_errno is set.
1668  */
1669 static int
1670 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
1671                            uint64_t item_flags,
1672                            struct rte_eth_dev *dev,
1673                            struct rte_flow_error *error)
1674 {
1675         const struct rte_flow_item_vlan *mask = item->mask;
1676         const struct rte_flow_item_vlan nic_mask = {
1677                 .tci = RTE_BE16(UINT16_MAX),
1678                 .inner_type = RTE_BE16(UINT16_MAX),
1679         };
1680         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1681         int ret;
1682         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1683                                         MLX5_FLOW_LAYER_INNER_L4) :
1684                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1685                                         MLX5_FLOW_LAYER_OUTER_L4);
1686         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1687                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1688
1689         if (item_flags & vlanm)
1690                 return rte_flow_error_set(error, EINVAL,
1691                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1692                                           "multiple VLAN layers not supported");
1693         else if ((item_flags & l34m) != 0)
1694                 return rte_flow_error_set(error, EINVAL,
1695                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1696                                           "VLAN cannot follow L3/L4 layer");
1697         if (!mask)
1698                 mask = &rte_flow_item_vlan_mask;
1699         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1700                                         (const uint8_t *)&nic_mask,
1701                                         sizeof(struct rte_flow_item_vlan),
1702                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1703         if (ret)
1704                 return ret;
1705         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1706                 struct mlx5_priv *priv = dev->data->dev_private;
1707
1708                 if (priv->vmwa_context) {
1709                         /*
1710                          * Non-NULL context means we have a virtual machine
1711                          * and SR-IOV enabled, we have to create VLAN interface
1712                          * to make hypervisor to setup E-Switch vport
1713                          * context correctly. We avoid creating the multiple
1714                          * VLAN interfaces, so we cannot support VLAN tag mask.
1715                          */
1716                         return rte_flow_error_set(error, EINVAL,
1717                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1718                                                   item,
1719                                                   "VLAN tag mask is not"
1720                                                   " supported in virtual"
1721                                                   " environment");
1722                 }
1723         }
1724         return 0;
1725 }
1726
1727 /*
1728  * GTP flags are contained in 1 byte of the format:
1729  * -------------------------------------------
1730  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
1731  * |-----------------------------------------|
1732  * | value | Version | PT | Res | E | S | PN |
1733  * -------------------------------------------
1734  *
1735  * Matching is supported only for GTP flags E, S, PN.
1736  */
1737 #define MLX5_GTP_FLAGS_MASK     0x07
1738
1739 /**
1740  * Validate GTP item.
1741  *
1742  * @param[in] dev
1743  *   Pointer to the rte_eth_dev structure.
1744  * @param[in] item
1745  *   Item specification.
1746  * @param[in] item_flags
1747  *   Bit-fields that holds the items detected until now.
1748  * @param[out] error
1749  *   Pointer to error structure.
1750  *
1751  * @return
1752  *   0 on success, a negative errno value otherwise and rte_errno is set.
1753  */
1754 static int
1755 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
1756                           const struct rte_flow_item *item,
1757                           uint64_t item_flags,
1758                           struct rte_flow_error *error)
1759 {
1760         struct mlx5_priv *priv = dev->data->dev_private;
1761         const struct rte_flow_item_gtp *spec = item->spec;
1762         const struct rte_flow_item_gtp *mask = item->mask;
1763         const struct rte_flow_item_gtp nic_mask = {
1764                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
1765                 .msg_type = 0xff,
1766                 .teid = RTE_BE32(0xffffffff),
1767         };
1768
1769         if (!priv->config.hca_attr.tunnel_stateless_gtp)
1770                 return rte_flow_error_set(error, ENOTSUP,
1771                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1772                                           "GTP support is not enabled");
1773         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1774                 return rte_flow_error_set(error, ENOTSUP,
1775                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1776                                           "multiple tunnel layers not"
1777                                           " supported");
1778         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1779                 return rte_flow_error_set(error, EINVAL,
1780                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1781                                           "no outer UDP layer found");
1782         if (!mask)
1783                 mask = &rte_flow_item_gtp_mask;
1784         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
1785                 return rte_flow_error_set(error, ENOTSUP,
1786                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1787                                           "Match is supported for GTP"
1788                                           " flags only");
1789         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1790                                          (const uint8_t *)&nic_mask,
1791                                          sizeof(struct rte_flow_item_gtp),
1792                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1793 }
1794
1795 /**
1796  * Validate IPV4 item.
1797  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
1798  * add specific validation of fragment_offset field,
1799  *
1800  * @param[in] item
1801  *   Item specification.
1802  * @param[in] item_flags
1803  *   Bit-fields that holds the items detected until now.
1804  * @param[out] error
1805  *   Pointer to error structure.
1806  *
1807  * @return
1808  *   0 on success, a negative errno value otherwise and rte_errno is set.
1809  */
1810 static int
1811 flow_dv_validate_item_ipv4(const struct rte_flow_item *item,
1812                            uint64_t item_flags,
1813                            uint64_t last_item,
1814                            uint16_t ether_type,
1815                            struct rte_flow_error *error)
1816 {
1817         int ret;
1818         const struct rte_flow_item_ipv4 *spec = item->spec;
1819         const struct rte_flow_item_ipv4 *last = item->last;
1820         const struct rte_flow_item_ipv4 *mask = item->mask;
1821         rte_be16_t fragment_offset_spec = 0;
1822         rte_be16_t fragment_offset_last = 0;
1823         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
1824                 .hdr = {
1825                         .src_addr = RTE_BE32(0xffffffff),
1826                         .dst_addr = RTE_BE32(0xffffffff),
1827                         .type_of_service = 0xff,
1828                         .fragment_offset = RTE_BE16(0xffff),
1829                         .next_proto_id = 0xff,
1830                         .time_to_live = 0xff,
1831                 },
1832         };
1833
1834         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
1835                                            ether_type, &nic_ipv4_mask,
1836                                            MLX5_ITEM_RANGE_ACCEPTED, error);
1837         if (ret < 0)
1838                 return ret;
1839         if (spec && mask)
1840                 fragment_offset_spec = spec->hdr.fragment_offset &
1841                                        mask->hdr.fragment_offset;
1842         if (!fragment_offset_spec)
1843                 return 0;
1844         /*
1845          * spec and mask are valid, enforce using full mask to make sure the
1846          * complete value is used correctly.
1847          */
1848         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1849                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1850                 return rte_flow_error_set(error, EINVAL,
1851                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1852                                           item, "must use full mask for"
1853                                           " fragment_offset");
1854         /*
1855          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
1856          * indicating this is 1st fragment of fragmented packet.
1857          * This is not yet supported in MLX5, return appropriate error message.
1858          */
1859         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
1860                 return rte_flow_error_set(error, ENOTSUP,
1861                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1862                                           "match on first fragment not "
1863                                           "supported");
1864         if (fragment_offset_spec && !last)
1865                 return rte_flow_error_set(error, ENOTSUP,
1866                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1867                                           "specified value not supported");
1868         /* spec and last are valid, validate the specified range. */
1869         fragment_offset_last = last->hdr.fragment_offset &
1870                                mask->hdr.fragment_offset;
1871         /*
1872          * Match on fragment_offset spec 0x2001 and last 0x3fff
1873          * means MF is 1 and frag-offset is > 0.
1874          * This packet is fragment 2nd and onward, excluding last.
1875          * This is not yet supported in MLX5, return appropriate
1876          * error message.
1877          */
1878         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
1879             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1880                 return rte_flow_error_set(error, ENOTSUP,
1881                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1882                                           last, "match on following "
1883                                           "fragments not supported");
1884         /*
1885          * Match on fragment_offset spec 0x0001 and last 0x1fff
1886          * means MF is 0 and frag-offset is > 0.
1887          * This packet is last fragment of fragmented packet.
1888          * This is not yet supported in MLX5, return appropriate
1889          * error message.
1890          */
1891         if (fragment_offset_spec == RTE_BE16(1) &&
1892             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
1893                 return rte_flow_error_set(error, ENOTSUP,
1894                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1895                                           last, "match on last "
1896                                           "fragment not supported");
1897         /*
1898          * Match on fragment_offset spec 0x0001 and last 0x3fff
1899          * means MF and/or frag-offset is not 0.
1900          * This is a fragmented packet.
1901          * Other range values are invalid and rejected.
1902          */
1903         if (!(fragment_offset_spec == RTE_BE16(1) &&
1904               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
1905                 return rte_flow_error_set(error, ENOTSUP,
1906                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
1907                                           "specified range not supported");
1908         return 0;
1909 }
1910
1911 /**
1912  * Validate IPV6 fragment extension item.
1913  *
1914  * @param[in] item
1915  *   Item specification.
1916  * @param[in] item_flags
1917  *   Bit-fields that holds the items detected until now.
1918  * @param[out] error
1919  *   Pointer to error structure.
1920  *
1921  * @return
1922  *   0 on success, a negative errno value otherwise and rte_errno is set.
1923  */
1924 static int
1925 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
1926                                     uint64_t item_flags,
1927                                     struct rte_flow_error *error)
1928 {
1929         const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
1930         const struct rte_flow_item_ipv6_frag_ext *last = item->last;
1931         const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
1932         rte_be16_t frag_data_spec = 0;
1933         rte_be16_t frag_data_last = 0;
1934         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1935         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1936                                       MLX5_FLOW_LAYER_OUTER_L4;
1937         int ret = 0;
1938         struct rte_flow_item_ipv6_frag_ext nic_mask = {
1939                 .hdr = {
1940                         .next_header = 0xff,
1941                         .frag_data = RTE_BE16(0xffff),
1942                 },
1943         };
1944
1945         if (item_flags & l4m)
1946                 return rte_flow_error_set(error, EINVAL,
1947                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1948                                           "ipv6 fragment extension item cannot "
1949                                           "follow L4 item.");
1950         if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
1951             (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
1952                 return rte_flow_error_set(error, EINVAL,
1953                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1954                                           "ipv6 fragment extension item must "
1955                                           "follow ipv6 item");
1956         if (spec && mask)
1957                 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
1958         if (!frag_data_spec)
1959                 return 0;
1960         /*
1961          * spec and mask are valid, enforce using full mask to make sure the
1962          * complete value is used correctly.
1963          */
1964         if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
1965                                 RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
1966                 return rte_flow_error_set(error, EINVAL,
1967                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1968                                           item, "must use full mask for"
1969                                           " frag_data");
1970         /*
1971          * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
1972          * This is 1st fragment of fragmented packet.
1973          */
1974         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
1975                 return rte_flow_error_set(error, ENOTSUP,
1976                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1977                                           "match on first fragment not "
1978                                           "supported");
1979         if (frag_data_spec && !last)
1980                 return rte_flow_error_set(error, EINVAL,
1981                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1982                                           "specified value not supported");
1983         ret = mlx5_flow_item_acceptable
1984                                 (item, (const uint8_t *)mask,
1985                                  (const uint8_t *)&nic_mask,
1986                                  sizeof(struct rte_flow_item_ipv6_frag_ext),
1987                                  MLX5_ITEM_RANGE_ACCEPTED, error);
1988         if (ret)
1989                 return ret;
1990         /* spec and last are valid, validate the specified range. */
1991         frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
1992         /*
1993          * Match on frag_data spec 0x0009 and last 0xfff9
1994          * means M is 1 and frag-offset is > 0.
1995          * This packet is fragment 2nd and onward, excluding last.
1996          * This is not yet supported in MLX5, return appropriate
1997          * error message.
1998          */
1999         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
2000                                        RTE_IPV6_EHDR_MF_MASK) &&
2001             frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2002                 return rte_flow_error_set(error, ENOTSUP,
2003                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2004                                           last, "match on following "
2005                                           "fragments not supported");
2006         /*
2007          * Match on frag_data spec 0x0008 and last 0xfff8
2008          * means M is 0 and frag-offset is > 0.
2009          * This packet is last fragment of fragmented packet.
2010          * This is not yet supported in MLX5, return appropriate
2011          * error message.
2012          */
2013         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
2014             frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
2015                 return rte_flow_error_set(error, ENOTSUP,
2016                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2017                                           last, "match on last "
2018                                           "fragment not supported");
2019         /* Other range values are invalid and rejected. */
2020         return rte_flow_error_set(error, EINVAL,
2021                                   RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2022                                   "specified range not supported");
2023 }
2024
2025 /**
2026  * Validate the pop VLAN action.
2027  *
2028  * @param[in] dev
2029  *   Pointer to the rte_eth_dev structure.
2030  * @param[in] action_flags
2031  *   Holds the actions detected until now.
2032  * @param[in] action
2033  *   Pointer to the pop vlan action.
2034  * @param[in] item_flags
2035  *   The items found in this flow rule.
2036  * @param[in] attr
2037  *   Pointer to flow attributes.
2038  * @param[out] error
2039  *   Pointer to error structure.
2040  *
2041  * @return
2042  *   0 on success, a negative errno value otherwise and rte_errno is set.
2043  */
2044 static int
2045 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
2046                                  uint64_t action_flags,
2047                                  const struct rte_flow_action *action,
2048                                  uint64_t item_flags,
2049                                  const struct rte_flow_attr *attr,
2050                                  struct rte_flow_error *error)
2051 {
2052         const struct mlx5_priv *priv = dev->data->dev_private;
2053
2054         (void)action;
2055         (void)attr;
2056         if (!priv->sh->pop_vlan_action)
2057                 return rte_flow_error_set(error, ENOTSUP,
2058                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2059                                           NULL,
2060                                           "pop vlan action is not supported");
2061         if (attr->egress)
2062                 return rte_flow_error_set(error, ENOTSUP,
2063                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2064                                           NULL,
2065                                           "pop vlan action not supported for "
2066                                           "egress");
2067         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
2068                 return rte_flow_error_set(error, ENOTSUP,
2069                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2070                                           "no support for multiple VLAN "
2071                                           "actions");
2072         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
2073         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
2074             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
2075                 return rte_flow_error_set(error, ENOTSUP,
2076                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2077                                           NULL,
2078                                           "cannot pop vlan after decap without "
2079                                           "match on inner vlan in the flow");
2080         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
2081         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
2082             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2083                 return rte_flow_error_set(error, ENOTSUP,
2084                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2085                                           NULL,
2086                                           "cannot pop vlan without a "
2087                                           "match on (outer) vlan in the flow");
2088         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2089                 return rte_flow_error_set(error, EINVAL,
2090                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2091                                           "wrong action order, port_id should "
2092                                           "be after pop VLAN action");
2093         if (!attr->transfer && priv->representor)
2094                 return rte_flow_error_set(error, ENOTSUP,
2095                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2096                                           "pop vlan action for VF representor "
2097                                           "not supported on NIC table");
2098         return 0;
2099 }
2100
2101 /**
2102  * Get VLAN default info from vlan match info.
2103  *
2104  * @param[in] items
2105  *   the list of item specifications.
2106  * @param[out] vlan
2107  *   pointer VLAN info to fill to.
2108  *
2109  * @return
2110  *   0 on success, a negative errno value otherwise and rte_errno is set.
2111  */
2112 static void
2113 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2114                                   struct rte_vlan_hdr *vlan)
2115 {
2116         const struct rte_flow_item_vlan nic_mask = {
2117                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2118                                 MLX5DV_FLOW_VLAN_VID_MASK),
2119                 .inner_type = RTE_BE16(0xffff),
2120         };
2121
2122         if (items == NULL)
2123                 return;
2124         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2125                 int type = items->type;
2126
2127                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2128                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2129                         break;
2130         }
2131         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2132                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2133                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2134
2135                 /* If VLAN item in pattern doesn't contain data, return here. */
2136                 if (!vlan_v)
2137                         return;
2138                 if (!vlan_m)
2139                         vlan_m = &nic_mask;
2140                 /* Only full match values are accepted */
2141                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2142                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2143                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2144                         vlan->vlan_tci |=
2145                                 rte_be_to_cpu_16(vlan_v->tci &
2146                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2147                 }
2148                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2149                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2150                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2151                         vlan->vlan_tci |=
2152                                 rte_be_to_cpu_16(vlan_v->tci &
2153                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2154                 }
2155                 if (vlan_m->inner_type == nic_mask.inner_type)
2156                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2157                                                            vlan_m->inner_type);
2158         }
2159 }
2160
2161 /**
2162  * Validate the push VLAN action.
2163  *
2164  * @param[in] dev
2165  *   Pointer to the rte_eth_dev structure.
2166  * @param[in] action_flags
2167  *   Holds the actions detected until now.
2168  * @param[in] item_flags
2169  *   The items found in this flow rule.
2170  * @param[in] action
2171  *   Pointer to the action structure.
2172  * @param[in] attr
2173  *   Pointer to flow attributes
2174  * @param[out] error
2175  *   Pointer to error structure.
2176  *
2177  * @return
2178  *   0 on success, a negative errno value otherwise and rte_errno is set.
2179  */
2180 static int
2181 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2182                                   uint64_t action_flags,
2183                                   const struct rte_flow_item_vlan *vlan_m,
2184                                   const struct rte_flow_action *action,
2185                                   const struct rte_flow_attr *attr,
2186                                   struct rte_flow_error *error)
2187 {
2188         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2189         const struct mlx5_priv *priv = dev->data->dev_private;
2190
2191         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2192             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2193                 return rte_flow_error_set(error, EINVAL,
2194                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2195                                           "invalid vlan ethertype");
2196         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2197                 return rte_flow_error_set(error, EINVAL,
2198                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2199                                           "wrong action order, port_id should "
2200                                           "be after push VLAN");
2201         if (!attr->transfer && priv->representor)
2202                 return rte_flow_error_set(error, ENOTSUP,
2203                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2204                                           "push vlan action for VF representor "
2205                                           "not supported on NIC table");
2206         if (vlan_m &&
2207             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2208             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2209                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2210             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2211             !(mlx5_flow_find_action
2212                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2213                 return rte_flow_error_set(error, EINVAL,
2214                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2215                                           "not full match mask on VLAN PCP and "
2216                                           "there is no of_set_vlan_pcp action, "
2217                                           "push VLAN action cannot figure out "
2218                                           "PCP value");
2219         if (vlan_m &&
2220             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2221             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2222                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2223             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2224             !(mlx5_flow_find_action
2225                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2226                 return rte_flow_error_set(error, EINVAL,
2227                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2228                                           "not full match mask on VLAN VID and "
2229                                           "there is no of_set_vlan_vid action, "
2230                                           "push VLAN action cannot figure out "
2231                                           "VID value");
2232         (void)attr;
2233         return 0;
2234 }
2235
2236 /**
2237  * Validate the set VLAN PCP.
2238  *
2239  * @param[in] action_flags
2240  *   Holds the actions detected until now.
2241  * @param[in] actions
2242  *   Pointer to the list of actions remaining in the flow rule.
2243  * @param[out] error
2244  *   Pointer to error structure.
2245  *
2246  * @return
2247  *   0 on success, a negative errno value otherwise and rte_errno is set.
2248  */
2249 static int
2250 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2251                                      const struct rte_flow_action actions[],
2252                                      struct rte_flow_error *error)
2253 {
2254         const struct rte_flow_action *action = actions;
2255         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2256
2257         if (conf->vlan_pcp > 7)
2258                 return rte_flow_error_set(error, EINVAL,
2259                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2260                                           "VLAN PCP value is too big");
2261         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2262                 return rte_flow_error_set(error, ENOTSUP,
2263                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2264                                           "set VLAN PCP action must follow "
2265                                           "the push VLAN action");
2266         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2267                 return rte_flow_error_set(error, ENOTSUP,
2268                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2269                                           "Multiple VLAN PCP modification are "
2270                                           "not supported");
2271         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2272                 return rte_flow_error_set(error, EINVAL,
2273                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2274                                           "wrong action order, port_id should "
2275                                           "be after set VLAN PCP");
2276         return 0;
2277 }
2278
2279 /**
2280  * Validate the set VLAN VID.
2281  *
2282  * @param[in] item_flags
2283  *   Holds the items detected in this rule.
2284  * @param[in] action_flags
2285  *   Holds the actions detected until now.
2286  * @param[in] actions
2287  *   Pointer to the list of actions remaining in the flow rule.
2288  * @param[out] error
2289  *   Pointer to error structure.
2290  *
2291  * @return
2292  *   0 on success, a negative errno value otherwise and rte_errno is set.
2293  */
2294 static int
2295 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2296                                      uint64_t action_flags,
2297                                      const struct rte_flow_action actions[],
2298                                      struct rte_flow_error *error)
2299 {
2300         const struct rte_flow_action *action = actions;
2301         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2302
2303         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2304                 return rte_flow_error_set(error, EINVAL,
2305                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2306                                           "VLAN VID value is too big");
2307         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2308             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2309                 return rte_flow_error_set(error, ENOTSUP,
2310                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2311                                           "set VLAN VID action must follow push"
2312                                           " VLAN action or match on VLAN item");
2313         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2314                 return rte_flow_error_set(error, ENOTSUP,
2315                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2316                                           "Multiple VLAN VID modifications are "
2317                                           "not supported");
2318         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2319                 return rte_flow_error_set(error, EINVAL,
2320                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2321                                           "wrong action order, port_id should "
2322                                           "be after set VLAN VID");
2323         return 0;
2324 }
2325
2326 /*
2327  * Validate the FLAG action.
2328  *
2329  * @param[in] dev
2330  *   Pointer to the rte_eth_dev structure.
2331  * @param[in] action_flags
2332  *   Holds the actions detected until now.
2333  * @param[in] attr
2334  *   Pointer to flow attributes
2335  * @param[out] error
2336  *   Pointer to error structure.
2337  *
2338  * @return
2339  *   0 on success, a negative errno value otherwise and rte_errno is set.
2340  */
2341 static int
2342 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
2343                              uint64_t action_flags,
2344                              const struct rte_flow_attr *attr,
2345                              struct rte_flow_error *error)
2346 {
2347         struct mlx5_priv *priv = dev->data->dev_private;
2348         struct mlx5_dev_config *config = &priv->config;
2349         int ret;
2350
2351         /* Fall back if no extended metadata register support. */
2352         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2353                 return mlx5_flow_validate_action_flag(action_flags, attr,
2354                                                       error);
2355         /* Extensive metadata mode requires registers. */
2356         if (!mlx5_flow_ext_mreg_supported(dev))
2357                 return rte_flow_error_set(error, ENOTSUP,
2358                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2359                                           "no metadata registers "
2360                                           "to support flag action");
2361         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
2362                 return rte_flow_error_set(error, ENOTSUP,
2363                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2364                                           "extended metadata register"
2365                                           " isn't available");
2366         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2367         if (ret < 0)
2368                 return ret;
2369         MLX5_ASSERT(ret > 0);
2370         if (action_flags & MLX5_FLOW_ACTION_MARK)
2371                 return rte_flow_error_set(error, EINVAL,
2372                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2373                                           "can't mark and flag in same flow");
2374         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2375                 return rte_flow_error_set(error, EINVAL,
2376                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2377                                           "can't have 2 flag"
2378                                           " actions in same flow");
2379         return 0;
2380 }
2381
2382 /**
2383  * Validate MARK action.
2384  *
2385  * @param[in] dev
2386  *   Pointer to the rte_eth_dev structure.
2387  * @param[in] action
2388  *   Pointer to action.
2389  * @param[in] action_flags
2390  *   Holds the actions detected until now.
2391  * @param[in] attr
2392  *   Pointer to flow attributes
2393  * @param[out] error
2394  *   Pointer to error structure.
2395  *
2396  * @return
2397  *   0 on success, a negative errno value otherwise and rte_errno is set.
2398  */
2399 static int
2400 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
2401                              const struct rte_flow_action *action,
2402                              uint64_t action_flags,
2403                              const struct rte_flow_attr *attr,
2404                              struct rte_flow_error *error)
2405 {
2406         struct mlx5_priv *priv = dev->data->dev_private;
2407         struct mlx5_dev_config *config = &priv->config;
2408         const struct rte_flow_action_mark *mark = action->conf;
2409         int ret;
2410
2411         /* Fall back if no extended metadata register support. */
2412         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2413                 return mlx5_flow_validate_action_mark(action, action_flags,
2414                                                       attr, error);
2415         /* Extensive metadata mode requires registers. */
2416         if (!mlx5_flow_ext_mreg_supported(dev))
2417                 return rte_flow_error_set(error, ENOTSUP,
2418                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2419                                           "no metadata registers "
2420                                           "to support mark action");
2421         if (!priv->sh->dv_mark_mask)
2422                 return rte_flow_error_set(error, ENOTSUP,
2423                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2424                                           "extended metadata register"
2425                                           " isn't available");
2426         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2427         if (ret < 0)
2428                 return ret;
2429         MLX5_ASSERT(ret > 0);
2430         if (!mark)
2431                 return rte_flow_error_set(error, EINVAL,
2432                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2433                                           "configuration cannot be null");
2434         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
2435                 return rte_flow_error_set(error, EINVAL,
2436                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2437                                           &mark->id,
2438                                           "mark id exceeds the limit");
2439         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2440                 return rte_flow_error_set(error, EINVAL,
2441                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2442                                           "can't flag and mark in same flow");
2443         if (action_flags & MLX5_FLOW_ACTION_MARK)
2444                 return rte_flow_error_set(error, EINVAL,
2445                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2446                                           "can't have 2 mark actions in same"
2447                                           " flow");
2448         return 0;
2449 }
2450
2451 /**
2452  * Validate SET_META action.
2453  *
2454  * @param[in] dev
2455  *   Pointer to the rte_eth_dev structure.
2456  * @param[in] action
2457  *   Pointer to the action structure.
2458  * @param[in] action_flags
2459  *   Holds the actions detected until now.
2460  * @param[in] attr
2461  *   Pointer to flow attributes
2462  * @param[out] error
2463  *   Pointer to error structure.
2464  *
2465  * @return
2466  *   0 on success, a negative errno value otherwise and rte_errno is set.
2467  */
2468 static int
2469 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
2470                                  const struct rte_flow_action *action,
2471                                  uint64_t action_flags __rte_unused,
2472                                  const struct rte_flow_attr *attr,
2473                                  struct rte_flow_error *error)
2474 {
2475         const struct rte_flow_action_set_meta *conf;
2476         uint32_t nic_mask = UINT32_MAX;
2477         int reg;
2478
2479         if (!mlx5_flow_ext_mreg_supported(dev))
2480                 return rte_flow_error_set(error, ENOTSUP,
2481                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2482                                           "extended metadata register"
2483                                           " isn't supported");
2484         reg = flow_dv_get_metadata_reg(dev, attr, error);
2485         if (reg < 0)
2486                 return reg;
2487         if (reg != REG_A && reg != REG_B) {
2488                 struct mlx5_priv *priv = dev->data->dev_private;
2489
2490                 nic_mask = priv->sh->dv_meta_mask;
2491         }
2492         if (!(action->conf))
2493                 return rte_flow_error_set(error, EINVAL,
2494                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2495                                           "configuration cannot be null");
2496         conf = (const struct rte_flow_action_set_meta *)action->conf;
2497         if (!conf->mask)
2498                 return rte_flow_error_set(error, EINVAL,
2499                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2500                                           "zero mask doesn't have any effect");
2501         if (conf->mask & ~nic_mask)
2502                 return rte_flow_error_set(error, EINVAL,
2503                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2504                                           "meta data must be within reg C0");
2505         return 0;
2506 }
2507
2508 /**
2509  * Validate SET_TAG action.
2510  *
2511  * @param[in] dev
2512  *   Pointer to the rte_eth_dev structure.
2513  * @param[in] action
2514  *   Pointer to the action structure.
2515  * @param[in] action_flags
2516  *   Holds the actions detected until now.
2517  * @param[in] attr
2518  *   Pointer to flow attributes
2519  * @param[out] error
2520  *   Pointer to error structure.
2521  *
2522  * @return
2523  *   0 on success, a negative errno value otherwise and rte_errno is set.
2524  */
2525 static int
2526 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
2527                                 const struct rte_flow_action *action,
2528                                 uint64_t action_flags,
2529                                 const struct rte_flow_attr *attr,
2530                                 struct rte_flow_error *error)
2531 {
2532         const struct rte_flow_action_set_tag *conf;
2533         const uint64_t terminal_action_flags =
2534                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
2535                 MLX5_FLOW_ACTION_RSS;
2536         int ret;
2537
2538         if (!mlx5_flow_ext_mreg_supported(dev))
2539                 return rte_flow_error_set(error, ENOTSUP,
2540                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2541                                           "extensive metadata register"
2542                                           " isn't supported");
2543         if (!(action->conf))
2544                 return rte_flow_error_set(error, EINVAL,
2545                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2546                                           "configuration cannot be null");
2547         conf = (const struct rte_flow_action_set_tag *)action->conf;
2548         if (!conf->mask)
2549                 return rte_flow_error_set(error, EINVAL,
2550                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2551                                           "zero mask doesn't have any effect");
2552         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
2553         if (ret < 0)
2554                 return ret;
2555         if (!attr->transfer && attr->ingress &&
2556             (action_flags & terminal_action_flags))
2557                 return rte_flow_error_set(error, EINVAL,
2558                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2559                                           "set_tag has no effect"
2560                                           " with terminal actions");
2561         return 0;
2562 }
2563
2564 /**
2565  * Validate count action.
2566  *
2567  * @param[in] dev
2568  *   Pointer to rte_eth_dev structure.
2569  * @param[out] error
2570  *   Pointer to error structure.
2571  *
2572  * @return
2573  *   0 on success, a negative errno value otherwise and rte_errno is set.
2574  */
2575 static int
2576 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2577                               struct rte_flow_error *error)
2578 {
2579         struct mlx5_priv *priv = dev->data->dev_private;
2580
2581         if (!priv->config.devx)
2582                 goto notsup_err;
2583 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2584         return 0;
2585 #endif
2586 notsup_err:
2587         return rte_flow_error_set
2588                       (error, ENOTSUP,
2589                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2590                        NULL,
2591                        "count action not supported");
2592 }
2593
2594 /**
2595  * Validate the L2 encap action.
2596  *
2597  * @param[in] dev
2598  *   Pointer to the rte_eth_dev structure.
2599  * @param[in] action_flags
2600  *   Holds the actions detected until now.
2601  * @param[in] action
2602  *   Pointer to the action structure.
2603  * @param[in] attr
2604  *   Pointer to flow attributes.
2605  * @param[out] error
2606  *   Pointer to error structure.
2607  *
2608  * @return
2609  *   0 on success, a negative errno value otherwise and rte_errno is set.
2610  */
2611 static int
2612 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
2613                                  uint64_t action_flags,
2614                                  const struct rte_flow_action *action,
2615                                  const struct rte_flow_attr *attr,
2616                                  struct rte_flow_error *error)
2617 {
2618         const struct mlx5_priv *priv = dev->data->dev_private;
2619
2620         if (!(action->conf))
2621                 return rte_flow_error_set(error, EINVAL,
2622                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2623                                           "configuration cannot be null");
2624         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
2625                 return rte_flow_error_set(error, EINVAL,
2626                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2627                                           "can only have a single encap action "
2628                                           "in a flow");
2629         if (!attr->transfer && priv->representor)
2630                 return rte_flow_error_set(error, ENOTSUP,
2631                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2632                                           "encap action for VF representor "
2633                                           "not supported on NIC table");
2634         return 0;
2635 }
2636
2637 /**
2638  * Validate a decap action.
2639  *
2640  * @param[in] dev
2641  *   Pointer to the rte_eth_dev structure.
2642  * @param[in] action_flags
2643  *   Holds the actions detected until now.
2644  * @param[in] attr
2645  *   Pointer to flow attributes
2646  * @param[out] error
2647  *   Pointer to error structure.
2648  *
2649  * @return
2650  *   0 on success, a negative errno value otherwise and rte_errno is set.
2651  */
2652 static int
2653 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
2654                               uint64_t action_flags,
2655                               const struct rte_flow_attr *attr,
2656                               struct rte_flow_error *error)
2657 {
2658         const struct mlx5_priv *priv = dev->data->dev_private;
2659
2660         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
2661             !priv->config.decap_en)
2662                 return rte_flow_error_set(error, ENOTSUP,
2663                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2664                                           "decap is not enabled");
2665         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
2666                 return rte_flow_error_set(error, ENOTSUP,
2667                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2668                                           action_flags &
2669                                           MLX5_FLOW_ACTION_DECAP ? "can only "
2670                                           "have a single decap action" : "decap "
2671                                           "after encap is not supported");
2672         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2673                 return rte_flow_error_set(error, EINVAL,
2674                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2675                                           "can't have decap action after"
2676                                           " modify action");
2677         if (attr->egress)
2678                 return rte_flow_error_set(error, ENOTSUP,
2679                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2680                                           NULL,
2681                                           "decap action not supported for "
2682                                           "egress");
2683         if (!attr->transfer && priv->representor)
2684                 return rte_flow_error_set(error, ENOTSUP,
2685                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2686                                           "decap action for VF representor "
2687                                           "not supported on NIC table");
2688         return 0;
2689 }
2690
2691 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
2692
2693 /**
2694  * Validate the raw encap and decap actions.
2695  *
2696  * @param[in] dev
2697  *   Pointer to the rte_eth_dev structure.
2698  * @param[in] decap
2699  *   Pointer to the decap action.
2700  * @param[in] encap
2701  *   Pointer to the encap action.
2702  * @param[in] attr
2703  *   Pointer to flow attributes
2704  * @param[in/out] action_flags
2705  *   Holds the actions detected until now.
2706  * @param[out] actions_n
2707  *   pointer to the number of actions counter.
2708  * @param[out] error
2709  *   Pointer to error structure.
2710  *
2711  * @return
2712  *   0 on success, a negative errno value otherwise and rte_errno is set.
2713  */
2714 static int
2715 flow_dv_validate_action_raw_encap_decap
2716         (struct rte_eth_dev *dev,
2717          const struct rte_flow_action_raw_decap *decap,
2718          const struct rte_flow_action_raw_encap *encap,
2719          const struct rte_flow_attr *attr, uint64_t *action_flags,
2720          int *actions_n, struct rte_flow_error *error)
2721 {
2722         const struct mlx5_priv *priv = dev->data->dev_private;
2723         int ret;
2724
2725         if (encap && (!encap->size || !encap->data))
2726                 return rte_flow_error_set(error, EINVAL,
2727                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2728                                           "raw encap data cannot be empty");
2729         if (decap && encap) {
2730                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
2731                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2732                         /* L3 encap. */
2733                         decap = NULL;
2734                 else if (encap->size <=
2735                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2736                            decap->size >
2737                            MLX5_ENCAPSULATION_DECISION_SIZE)
2738                         /* L3 decap. */
2739                         encap = NULL;
2740                 else if (encap->size >
2741                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2742                            decap->size >
2743                            MLX5_ENCAPSULATION_DECISION_SIZE)
2744                         /* 2 L2 actions: encap and decap. */
2745                         ;
2746                 else
2747                         return rte_flow_error_set(error,
2748                                 ENOTSUP,
2749                                 RTE_FLOW_ERROR_TYPE_ACTION,
2750                                 NULL, "unsupported too small "
2751                                 "raw decap and too small raw "
2752                                 "encap combination");
2753         }
2754         if (decap) {
2755                 ret = flow_dv_validate_action_decap(dev, *action_flags, attr,
2756                                                     error);
2757                 if (ret < 0)
2758                         return ret;
2759                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
2760                 ++(*actions_n);
2761         }
2762         if (encap) {
2763                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
2764                         return rte_flow_error_set(error, ENOTSUP,
2765                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2766                                                   NULL,
2767                                                   "small raw encap size");
2768                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
2769                         return rte_flow_error_set(error, EINVAL,
2770                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2771                                                   NULL,
2772                                                   "more than one encap action");
2773                 if (!attr->transfer && priv->representor)
2774                         return rte_flow_error_set
2775                                         (error, ENOTSUP,
2776                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2777                                          "encap action for VF representor "
2778                                          "not supported on NIC table");
2779                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
2780                 ++(*actions_n);
2781         }
2782         return 0;
2783 }
2784
2785 /**
2786  * Match encap_decap resource.
2787  *
2788  * @param entry
2789  *   Pointer to exist resource entry object.
2790  * @param ctx
2791  *   Pointer to new encap_decap resource.
2792  *
2793  * @return
2794  *   0 on matching, -1 otherwise.
2795  */
2796 static int
2797 flow_dv_encap_decap_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
2798 {
2799         struct mlx5_flow_dv_encap_decap_resource *resource;
2800         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2801
2802         resource = (struct mlx5_flow_dv_encap_decap_resource *)ctx;
2803         cache_resource = container_of(entry,
2804                                       struct mlx5_flow_dv_encap_decap_resource,
2805                                       entry);
2806         if (resource->entry.key == cache_resource->entry.key &&
2807             resource->reformat_type == cache_resource->reformat_type &&
2808             resource->ft_type == cache_resource->ft_type &&
2809             resource->flags == cache_resource->flags &&
2810             resource->size == cache_resource->size &&
2811             !memcmp((const void *)resource->buf,
2812                     (const void *)cache_resource->buf,
2813                     resource->size))
2814                 return 0;
2815         return -1;
2816 }
2817
2818 /**
2819  * Find existing encap/decap resource or create and register a new one.
2820  *
2821  * @param[in, out] dev
2822  *   Pointer to rte_eth_dev structure.
2823  * @param[in, out] resource
2824  *   Pointer to encap/decap resource.
2825  * @parm[in, out] dev_flow
2826  *   Pointer to the dev_flow.
2827  * @param[out] error
2828  *   pointer to error structure.
2829  *
2830  * @return
2831  *   0 on success otherwise -errno and errno is set.
2832  */
2833 static int
2834 flow_dv_encap_decap_resource_register
2835                         (struct rte_eth_dev *dev,
2836                          struct mlx5_flow_dv_encap_decap_resource *resource,
2837                          struct mlx5_flow *dev_flow,
2838                          struct rte_flow_error *error)
2839 {
2840         struct mlx5_priv *priv = dev->data->dev_private;
2841         struct mlx5_dev_ctx_shared *sh = priv->sh;
2842         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2843         struct mlx5dv_dr_domain *domain;
2844         struct mlx5_hlist_entry *entry;
2845         union mlx5_flow_encap_decap_key encap_decap_key = {
2846                 {
2847                         .ft_type = resource->ft_type,
2848                         .refmt_type = resource->reformat_type,
2849                         .buf_size = resource->size,
2850                         .table_level = !!dev_flow->dv.group,
2851                         .cksum = 0,
2852                 }
2853         };
2854         int ret;
2855
2856         resource->flags = dev_flow->dv.group ? 0 : 1;
2857         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2858                 domain = sh->fdb_domain;
2859         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2860                 domain = sh->rx_domain;
2861         else
2862                 domain = sh->tx_domain;
2863         encap_decap_key.cksum = __rte_raw_cksum(resource->buf,
2864                                                 resource->size, 0);
2865         resource->entry.key = encap_decap_key.v64;
2866         /* Lookup a matching resource from cache. */
2867         entry = mlx5_hlist_lookup_ex(sh->encaps_decaps, resource->entry.key,
2868                                      flow_dv_encap_decap_resource_match,
2869                                      (void *)resource);
2870         if (entry) {
2871                 cache_resource = container_of(entry,
2872                         struct mlx5_flow_dv_encap_decap_resource, entry);
2873                 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2874                         (void *)cache_resource,
2875                         rte_atomic32_read(&cache_resource->refcnt));
2876                 rte_atomic32_inc(&cache_resource->refcnt);
2877                 dev_flow->handle->dvh.rix_encap_decap = cache_resource->idx;
2878                 dev_flow->dv.encap_decap = cache_resource;
2879                 return 0;
2880         }
2881         /* Register new encap/decap resource. */
2882         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2883                                        &dev_flow->handle->dvh.rix_encap_decap);
2884         if (!cache_resource)
2885                 return rte_flow_error_set(error, ENOMEM,
2886                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2887                                           "cannot allocate resource memory");
2888         *cache_resource = *resource;
2889         cache_resource->idx = dev_flow->handle->dvh.rix_encap_decap;
2890         ret = mlx5_flow_os_create_flow_action_packet_reformat
2891                                         (sh->ctx, domain, cache_resource,
2892                                          &cache_resource->action);
2893         if (ret) {
2894                 mlx5_free(cache_resource);
2895                 return rte_flow_error_set(error, ENOMEM,
2896                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2897                                           NULL, "cannot create action");
2898         }
2899         rte_atomic32_init(&cache_resource->refcnt);
2900         rte_atomic32_inc(&cache_resource->refcnt);
2901         if (mlx5_hlist_insert_ex(sh->encaps_decaps, &cache_resource->entry,
2902                                  flow_dv_encap_decap_resource_match,
2903                                  (void *)cache_resource)) {
2904                 claim_zero(mlx5_flow_os_destroy_flow_action
2905                                                 (cache_resource->action));
2906                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2907                                 cache_resource->idx);
2908                 return rte_flow_error_set(error, EEXIST,
2909                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2910                                           NULL, "action exist");
2911         }
2912         dev_flow->dv.encap_decap = cache_resource;
2913         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2914                 (void *)cache_resource,
2915                 rte_atomic32_read(&cache_resource->refcnt));
2916         return 0;
2917 }
2918
2919 /**
2920  * Find existing table jump resource or create and register a new one.
2921  *
2922  * @param[in, out] dev
2923  *   Pointer to rte_eth_dev structure.
2924  * @param[in, out] tbl
2925  *   Pointer to flow table resource.
2926  * @parm[in, out] dev_flow
2927  *   Pointer to the dev_flow.
2928  * @param[out] error
2929  *   pointer to error structure.
2930  *
2931  * @return
2932  *   0 on success otherwise -errno and errno is set.
2933  */
2934 static int
2935 flow_dv_jump_tbl_resource_register
2936                         (struct rte_eth_dev *dev __rte_unused,
2937                          struct mlx5_flow_tbl_resource *tbl,
2938                          struct mlx5_flow *dev_flow,
2939                          struct rte_flow_error *error)
2940 {
2941         struct mlx5_flow_tbl_data_entry *tbl_data =
2942                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2943         int cnt, ret;
2944
2945         MLX5_ASSERT(tbl);
2946         cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
2947         if (!cnt) {
2948                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
2949                                 (tbl->obj, &tbl_data->jump.action);
2950                 if (ret)
2951                         return rte_flow_error_set(error, ENOMEM,
2952                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2953                                         NULL, "cannot create jump action");
2954                 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2955                         (void *)&tbl_data->jump, cnt);
2956         } else {
2957                 /* old jump should not make the table ref++. */
2958                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
2959                 MLX5_ASSERT(tbl_data->jump.action);
2960                 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2961                         (void *)&tbl_data->jump, cnt);
2962         }
2963         rte_atomic32_inc(&tbl_data->jump.refcnt);
2964         dev_flow->handle->rix_jump = tbl_data->idx;
2965         dev_flow->dv.jump = &tbl_data->jump;
2966         return 0;
2967 }
2968
2969 /**
2970  * Find existing default miss resource or create and register a new one.
2971  *
2972  * @param[in, out] dev
2973  *   Pointer to rte_eth_dev structure.
2974  * @param[out] error
2975  *   pointer to error structure.
2976  *
2977  * @return
2978  *   0 on success otherwise -errno and errno is set.
2979  */
2980 static int
2981 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
2982                 struct rte_flow_error *error)
2983 {
2984         struct mlx5_priv *priv = dev->data->dev_private;
2985         struct mlx5_dev_ctx_shared *sh = priv->sh;
2986         struct mlx5_flow_default_miss_resource *cache_resource =
2987                         &sh->default_miss;
2988         int cnt = rte_atomic32_read(&cache_resource->refcnt);
2989
2990         if (!cnt) {
2991                 MLX5_ASSERT(cache_resource->action);
2992                 cache_resource->action =
2993                 mlx5_glue->dr_create_flow_action_default_miss();
2994                 if (!cache_resource->action)
2995                         return rte_flow_error_set(error, ENOMEM,
2996                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2997                                         "cannot create default miss action");
2998                 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++",
2999                                 (void *)cache_resource->action, cnt);
3000         }
3001         rte_atomic32_inc(&cache_resource->refcnt);
3002         return 0;
3003 }
3004
3005 /**
3006  * Find existing table port ID resource or create and register a new one.
3007  *
3008  * @param[in, out] dev
3009  *   Pointer to rte_eth_dev structure.
3010  * @param[in, out] resource
3011  *   Pointer to port ID action resource.
3012  * @parm[in, out] dev_flow
3013  *   Pointer to the dev_flow.
3014  * @param[out] error
3015  *   pointer to error structure.
3016  *
3017  * @return
3018  *   0 on success otherwise -errno and errno is set.
3019  */
3020 static int
3021 flow_dv_port_id_action_resource_register
3022                         (struct rte_eth_dev *dev,
3023                          struct mlx5_flow_dv_port_id_action_resource *resource,
3024                          struct mlx5_flow *dev_flow,
3025                          struct rte_flow_error *error)
3026 {
3027         struct mlx5_priv *priv = dev->data->dev_private;
3028         struct mlx5_dev_ctx_shared *sh = priv->sh;
3029         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
3030         uint32_t idx = 0;
3031         int ret;
3032
3033         /* Lookup a matching resource from cache. */
3034         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list,
3035                       idx, cache_resource, next) {
3036                 if (resource->port_id == cache_resource->port_id) {
3037                         DRV_LOG(DEBUG, "port id action resource resource %p: "
3038                                 "refcnt %d++",
3039                                 (void *)cache_resource,
3040                                 rte_atomic32_read(&cache_resource->refcnt));
3041                         rte_atomic32_inc(&cache_resource->refcnt);
3042                         dev_flow->handle->rix_port_id_action = idx;
3043                         dev_flow->dv.port_id_action = cache_resource;
3044                         return 0;
3045                 }
3046         }
3047         /* Register new port id action resource. */
3048         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID],
3049                                        &dev_flow->handle->rix_port_id_action);
3050         if (!cache_resource)
3051                 return rte_flow_error_set(error, ENOMEM,
3052                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3053                                           "cannot allocate resource memory");
3054         *cache_resource = *resource;
3055         ret = mlx5_flow_os_create_flow_action_dest_port
3056                                 (priv->sh->fdb_domain, resource->port_id,
3057                                  &cache_resource->action);
3058         if (ret) {
3059                 mlx5_free(cache_resource);
3060                 return rte_flow_error_set(error, ENOMEM,
3061                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3062                                           NULL, "cannot create action");
3063         }
3064         rte_atomic32_init(&cache_resource->refcnt);
3065         rte_atomic32_inc(&cache_resource->refcnt);
3066         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
3067                      dev_flow->handle->rix_port_id_action, cache_resource,
3068                      next);
3069         dev_flow->dv.port_id_action = cache_resource;
3070         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
3071                 (void *)cache_resource,
3072                 rte_atomic32_read(&cache_resource->refcnt));
3073         return 0;
3074 }
3075
3076 /**
3077  * Find existing push vlan resource or create and register a new one.
3078  *
3079  * @param [in, out] dev
3080  *   Pointer to rte_eth_dev structure.
3081  * @param[in, out] resource
3082  *   Pointer to port ID action resource.
3083  * @parm[in, out] dev_flow
3084  *   Pointer to the dev_flow.
3085  * @param[out] error
3086  *   pointer to error structure.
3087  *
3088  * @return
3089  *   0 on success otherwise -errno and errno is set.
3090  */
3091 static int
3092 flow_dv_push_vlan_action_resource_register
3093                        (struct rte_eth_dev *dev,
3094                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
3095                         struct mlx5_flow *dev_flow,
3096                         struct rte_flow_error *error)
3097 {
3098         struct mlx5_priv *priv = dev->data->dev_private;
3099         struct mlx5_dev_ctx_shared *sh = priv->sh;
3100         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
3101         struct mlx5dv_dr_domain *domain;
3102         uint32_t idx = 0;
3103         int ret;
3104
3105         /* Lookup a matching resource from cache. */
3106         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3107                       sh->push_vlan_action_list, idx, cache_resource, next) {
3108                 if (resource->vlan_tag == cache_resource->vlan_tag &&
3109                     resource->ft_type == cache_resource->ft_type) {
3110                         DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
3111                                 "refcnt %d++",
3112                                 (void *)cache_resource,
3113                                 rte_atomic32_read(&cache_resource->refcnt));
3114                         rte_atomic32_inc(&cache_resource->refcnt);
3115                         dev_flow->handle->dvh.rix_push_vlan = idx;
3116                         dev_flow->dv.push_vlan_res = cache_resource;
3117                         return 0;
3118                 }
3119         }
3120         /* Register new push_vlan action resource. */
3121         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3122                                        &dev_flow->handle->dvh.rix_push_vlan);
3123         if (!cache_resource)
3124                 return rte_flow_error_set(error, ENOMEM,
3125                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3126                                           "cannot allocate resource memory");
3127         *cache_resource = *resource;
3128         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3129                 domain = sh->fdb_domain;
3130         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3131                 domain = sh->rx_domain;
3132         else
3133                 domain = sh->tx_domain;
3134         ret = mlx5_flow_os_create_flow_action_push_vlan
3135                                         (domain, resource->vlan_tag,
3136                                          &cache_resource->action);
3137         if (ret) {
3138                 mlx5_free(cache_resource);
3139                 return rte_flow_error_set(error, ENOMEM,
3140                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3141                                           NULL, "cannot create action");
3142         }
3143         rte_atomic32_init(&cache_resource->refcnt);
3144         rte_atomic32_inc(&cache_resource->refcnt);
3145         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3146                      &sh->push_vlan_action_list,
3147                      dev_flow->handle->dvh.rix_push_vlan,
3148                      cache_resource, next);
3149         dev_flow->dv.push_vlan_res = cache_resource;
3150         DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
3151                 (void *)cache_resource,
3152                 rte_atomic32_read(&cache_resource->refcnt));
3153         return 0;
3154 }
3155 /**
3156  * Get the size of specific rte_flow_item_type hdr size
3157  *
3158  * @param[in] item_type
3159  *   Tested rte_flow_item_type.
3160  *
3161  * @return
3162  *   sizeof struct item_type, 0 if void or irrelevant.
3163  */
3164 static size_t
3165 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
3166 {
3167         size_t retval;
3168
3169         switch (item_type) {
3170         case RTE_FLOW_ITEM_TYPE_ETH:
3171                 retval = sizeof(struct rte_ether_hdr);
3172                 break;
3173         case RTE_FLOW_ITEM_TYPE_VLAN:
3174                 retval = sizeof(struct rte_vlan_hdr);
3175                 break;
3176         case RTE_FLOW_ITEM_TYPE_IPV4:
3177                 retval = sizeof(struct rte_ipv4_hdr);
3178                 break;
3179         case RTE_FLOW_ITEM_TYPE_IPV6:
3180                 retval = sizeof(struct rte_ipv6_hdr);
3181                 break;
3182         case RTE_FLOW_ITEM_TYPE_UDP:
3183                 retval = sizeof(struct rte_udp_hdr);
3184                 break;
3185         case RTE_FLOW_ITEM_TYPE_TCP:
3186                 retval = sizeof(struct rte_tcp_hdr);
3187                 break;
3188         case RTE_FLOW_ITEM_TYPE_VXLAN:
3189         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3190                 retval = sizeof(struct rte_vxlan_hdr);
3191                 break;
3192         case RTE_FLOW_ITEM_TYPE_GRE:
3193         case RTE_FLOW_ITEM_TYPE_NVGRE:
3194                 retval = sizeof(struct rte_gre_hdr);
3195                 break;
3196         case RTE_FLOW_ITEM_TYPE_MPLS:
3197                 retval = sizeof(struct rte_mpls_hdr);
3198                 break;
3199         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
3200         default:
3201                 retval = 0;
3202                 break;
3203         }
3204         return retval;
3205 }
3206
3207 #define MLX5_ENCAP_IPV4_VERSION         0x40
3208 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
3209 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
3210 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
3211 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
3212 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
3213 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
3214
3215 /**
3216  * Convert the encap action data from list of rte_flow_item to raw buffer
3217  *
3218  * @param[in] items
3219  *   Pointer to rte_flow_item objects list.
3220  * @param[out] buf
3221  *   Pointer to the output buffer.
3222  * @param[out] size
3223  *   Pointer to the output buffer size.
3224  * @param[out] error
3225  *   Pointer to the error structure.
3226  *
3227  * @return
3228  *   0 on success, a negative errno value otherwise and rte_errno is set.
3229  */
3230 static int
3231 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
3232                            size_t *size, struct rte_flow_error *error)
3233 {
3234         struct rte_ether_hdr *eth = NULL;
3235         struct rte_vlan_hdr *vlan = NULL;
3236         struct rte_ipv4_hdr *ipv4 = NULL;
3237         struct rte_ipv6_hdr *ipv6 = NULL;
3238         struct rte_udp_hdr *udp = NULL;
3239         struct rte_vxlan_hdr *vxlan = NULL;
3240         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
3241         struct rte_gre_hdr *gre = NULL;
3242         size_t len;
3243         size_t temp_size = 0;
3244
3245         if (!items)
3246                 return rte_flow_error_set(error, EINVAL,
3247                                           RTE_FLOW_ERROR_TYPE_ACTION,
3248                                           NULL, "invalid empty data");
3249         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3250                 len = flow_dv_get_item_hdr_len(items->type);
3251                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
3252                         return rte_flow_error_set(error, EINVAL,
3253                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3254                                                   (void *)items->type,
3255                                                   "items total size is too big"
3256                                                   " for encap action");
3257                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
3258                 switch (items->type) {
3259                 case RTE_FLOW_ITEM_TYPE_ETH:
3260                         eth = (struct rte_ether_hdr *)&buf[temp_size];
3261                         break;
3262                 case RTE_FLOW_ITEM_TYPE_VLAN:
3263                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
3264                         if (!eth)
3265                                 return rte_flow_error_set(error, EINVAL,
3266                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3267                                                 (void *)items->type,
3268                                                 "eth header not found");
3269                         if (!eth->ether_type)
3270                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
3271                         break;
3272                 case RTE_FLOW_ITEM_TYPE_IPV4:
3273                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
3274                         if (!vlan && !eth)
3275                                 return rte_flow_error_set(error, EINVAL,
3276                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3277                                                 (void *)items->type,
3278                                                 "neither eth nor vlan"
3279                                                 " header found");
3280                         if (vlan && !vlan->eth_proto)
3281                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3282                         else if (eth && !eth->ether_type)
3283                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3284                         if (!ipv4->version_ihl)
3285                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
3286                                                     MLX5_ENCAP_IPV4_IHL_MIN;
3287                         if (!ipv4->time_to_live)
3288                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
3289                         break;
3290                 case RTE_FLOW_ITEM_TYPE_IPV6:
3291                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
3292                         if (!vlan && !eth)
3293                                 return rte_flow_error_set(error, EINVAL,
3294                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3295                                                 (void *)items->type,
3296                                                 "neither eth nor vlan"
3297                                                 " header found");
3298                         if (vlan && !vlan->eth_proto)
3299                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3300                         else if (eth && !eth->ether_type)
3301                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3302                         if (!ipv6->vtc_flow)
3303                                 ipv6->vtc_flow =
3304                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
3305                         if (!ipv6->hop_limits)
3306                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
3307                         break;
3308                 case RTE_FLOW_ITEM_TYPE_UDP:
3309                         udp = (struct rte_udp_hdr *)&buf[temp_size];
3310                         if (!ipv4 && !ipv6)
3311                                 return rte_flow_error_set(error, EINVAL,
3312                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3313                                                 (void *)items->type,
3314                                                 "ip header not found");
3315                         if (ipv4 && !ipv4->next_proto_id)
3316                                 ipv4->next_proto_id = IPPROTO_UDP;
3317                         else if (ipv6 && !ipv6->proto)
3318                                 ipv6->proto = IPPROTO_UDP;
3319                         break;
3320                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3321                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
3322                         if (!udp)
3323                                 return rte_flow_error_set(error, EINVAL,
3324                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3325                                                 (void *)items->type,
3326                                                 "udp header not found");
3327                         if (!udp->dst_port)
3328                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
3329                         if (!vxlan->vx_flags)
3330                                 vxlan->vx_flags =
3331                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
3332                         break;
3333                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3334                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
3335                         if (!udp)
3336                                 return rte_flow_error_set(error, EINVAL,
3337                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3338                                                 (void *)items->type,
3339                                                 "udp header not found");
3340                         if (!vxlan_gpe->proto)
3341                                 return rte_flow_error_set(error, EINVAL,
3342                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3343                                                 (void *)items->type,
3344                                                 "next protocol not found");
3345                         if (!udp->dst_port)
3346                                 udp->dst_port =
3347                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
3348                         if (!vxlan_gpe->vx_flags)
3349                                 vxlan_gpe->vx_flags =
3350                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
3351                         break;
3352                 case RTE_FLOW_ITEM_TYPE_GRE:
3353                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3354                         gre = (struct rte_gre_hdr *)&buf[temp_size];
3355                         if (!gre->proto)
3356                                 return rte_flow_error_set(error, EINVAL,
3357                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3358                                                 (void *)items->type,
3359                                                 "next protocol not found");
3360                         if (!ipv4 && !ipv6)
3361                                 return rte_flow_error_set(error, EINVAL,
3362                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3363                                                 (void *)items->type,
3364                                                 "ip header not found");
3365                         if (ipv4 && !ipv4->next_proto_id)
3366                                 ipv4->next_proto_id = IPPROTO_GRE;
3367                         else if (ipv6 && !ipv6->proto)
3368                                 ipv6->proto = IPPROTO_GRE;
3369                         break;
3370                 case RTE_FLOW_ITEM_TYPE_VOID:
3371                         break;
3372                 default:
3373                         return rte_flow_error_set(error, EINVAL,
3374                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3375                                                   (void *)items->type,
3376                                                   "unsupported item type");
3377                         break;
3378                 }
3379                 temp_size += len;
3380         }
3381         *size = temp_size;
3382         return 0;
3383 }
3384
3385 static int
3386 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
3387 {
3388         struct rte_ether_hdr *eth = NULL;
3389         struct rte_vlan_hdr *vlan = NULL;
3390         struct rte_ipv6_hdr *ipv6 = NULL;
3391         struct rte_udp_hdr *udp = NULL;
3392         char *next_hdr;
3393         uint16_t proto;
3394
3395         eth = (struct rte_ether_hdr *)data;
3396         next_hdr = (char *)(eth + 1);
3397         proto = RTE_BE16(eth->ether_type);
3398
3399         /* VLAN skipping */
3400         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
3401                 vlan = (struct rte_vlan_hdr *)next_hdr;
3402                 proto = RTE_BE16(vlan->eth_proto);
3403                 next_hdr += sizeof(struct rte_vlan_hdr);
3404         }
3405
3406         /* HW calculates IPv4 csum. no need to proceed */
3407         if (proto == RTE_ETHER_TYPE_IPV4)
3408                 return 0;
3409
3410         /* non IPv4/IPv6 header. not supported */
3411         if (proto != RTE_ETHER_TYPE_IPV6) {
3412                 return rte_flow_error_set(error, ENOTSUP,
3413                                           RTE_FLOW_ERROR_TYPE_ACTION,
3414                                           NULL, "Cannot offload non IPv4/IPv6");
3415         }
3416
3417         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
3418
3419         /* ignore non UDP */
3420         if (ipv6->proto != IPPROTO_UDP)
3421                 return 0;
3422
3423         udp = (struct rte_udp_hdr *)(ipv6 + 1);
3424         udp->dgram_cksum = 0;
3425
3426         return 0;
3427 }
3428
3429 /**
3430  * Convert L2 encap action to DV specification.
3431  *
3432  * @param[in] dev
3433  *   Pointer to rte_eth_dev structure.
3434  * @param[in] action
3435  *   Pointer to action structure.
3436  * @param[in, out] dev_flow
3437  *   Pointer to the mlx5_flow.
3438  * @param[in] transfer
3439  *   Mark if the flow is E-Switch flow.
3440  * @param[out] error
3441  *   Pointer to the error structure.
3442  *
3443  * @return
3444  *   0 on success, a negative errno value otherwise and rte_errno is set.
3445  */
3446 static int
3447 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
3448                                const struct rte_flow_action *action,
3449                                struct mlx5_flow *dev_flow,
3450                                uint8_t transfer,
3451                                struct rte_flow_error *error)
3452 {
3453         const struct rte_flow_item *encap_data;
3454         const struct rte_flow_action_raw_encap *raw_encap_data;
3455         struct mlx5_flow_dv_encap_decap_resource res = {
3456                 .reformat_type =
3457                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
3458                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3459                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
3460         };
3461
3462         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3463                 raw_encap_data =
3464                         (const struct rte_flow_action_raw_encap *)action->conf;
3465                 res.size = raw_encap_data->size;
3466                 memcpy(res.buf, raw_encap_data->data, res.size);
3467         } else {
3468                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
3469                         encap_data =
3470                                 ((const struct rte_flow_action_vxlan_encap *)
3471                                                 action->conf)->definition;
3472                 else
3473                         encap_data =
3474                                 ((const struct rte_flow_action_nvgre_encap *)
3475                                                 action->conf)->definition;
3476                 if (flow_dv_convert_encap_data(encap_data, res.buf,
3477                                                &res.size, error))
3478                         return -rte_errno;
3479         }
3480         if (flow_dv_zero_encap_udp_csum(res.buf, error))
3481                 return -rte_errno;
3482         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3483                 return rte_flow_error_set(error, EINVAL,
3484                                           RTE_FLOW_ERROR_TYPE_ACTION,
3485                                           NULL, "can't create L2 encap action");
3486         return 0;
3487 }
3488
3489 /**
3490  * Convert L2 decap action to DV specification.
3491  *
3492  * @param[in] dev
3493  *   Pointer to rte_eth_dev structure.
3494  * @param[in, out] dev_flow
3495  *   Pointer to the mlx5_flow.
3496  * @param[in] transfer
3497  *   Mark if the flow is E-Switch flow.
3498  * @param[out] error
3499  *   Pointer to the error structure.
3500  *
3501  * @return
3502  *   0 on success, a negative errno value otherwise and rte_errno is set.
3503  */
3504 static int
3505 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
3506                                struct mlx5_flow *dev_flow,
3507                                uint8_t transfer,
3508                                struct rte_flow_error *error)
3509 {
3510         struct mlx5_flow_dv_encap_decap_resource res = {
3511                 .size = 0,
3512                 .reformat_type =
3513                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
3514                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3515                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
3516         };
3517
3518         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3519                 return rte_flow_error_set(error, EINVAL,
3520                                           RTE_FLOW_ERROR_TYPE_ACTION,
3521                                           NULL, "can't create L2 decap action");
3522         return 0;
3523 }
3524
3525 /**
3526  * Convert raw decap/encap (L3 tunnel) action to DV specification.
3527  *
3528  * @param[in] dev
3529  *   Pointer to rte_eth_dev structure.
3530  * @param[in] action
3531  *   Pointer to action structure.
3532  * @param[in, out] dev_flow
3533  *   Pointer to the mlx5_flow.
3534  * @param[in] attr
3535  *   Pointer to the flow attributes.
3536  * @param[out] error
3537  *   Pointer to the error structure.
3538  *
3539  * @return
3540  *   0 on success, a negative errno value otherwise and rte_errno is set.
3541  */
3542 static int
3543 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
3544                                 const struct rte_flow_action *action,
3545                                 struct mlx5_flow *dev_flow,
3546                                 const struct rte_flow_attr *attr,
3547                                 struct rte_flow_error *error)
3548 {
3549         const struct rte_flow_action_raw_encap *encap_data;
3550         struct mlx5_flow_dv_encap_decap_resource res;
3551
3552         memset(&res, 0, sizeof(res));
3553         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
3554         res.size = encap_data->size;
3555         memcpy(res.buf, encap_data->data, res.size);
3556         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
3557                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
3558                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3559         if (attr->transfer)
3560                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3561         else
3562                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3563                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3564         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3565                 return rte_flow_error_set(error, EINVAL,
3566                                           RTE_FLOW_ERROR_TYPE_ACTION,
3567                                           NULL, "can't create encap action");
3568         return 0;
3569 }
3570
3571 /**
3572  * Create action push VLAN.
3573  *
3574  * @param[in] dev
3575  *   Pointer to rte_eth_dev structure.
3576  * @param[in] attr
3577  *   Pointer to the flow attributes.
3578  * @param[in] vlan
3579  *   Pointer to the vlan to push to the Ethernet header.
3580  * @param[in, out] dev_flow
3581  *   Pointer to the mlx5_flow.
3582  * @param[out] error
3583  *   Pointer to the error structure.
3584  *
3585  * @return
3586  *   0 on success, a negative errno value otherwise and rte_errno is set.
3587  */
3588 static int
3589 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
3590                                 const struct rte_flow_attr *attr,
3591                                 const struct rte_vlan_hdr *vlan,
3592                                 struct mlx5_flow *dev_flow,
3593                                 struct rte_flow_error *error)
3594 {
3595         struct mlx5_flow_dv_push_vlan_action_resource res;
3596
3597         memset(&res, 0, sizeof(res));
3598         res.vlan_tag =
3599                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
3600                                  vlan->vlan_tci);
3601         if (attr->transfer)
3602                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3603         else
3604                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3605                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3606         return flow_dv_push_vlan_action_resource_register
3607                                             (dev, &res, dev_flow, error);
3608 }
3609
3610 static int fdb_mirror;
3611
3612 /**
3613  * Validate the modify-header actions.
3614  *
3615  * @param[in] action_flags
3616  *   Holds the actions detected until now.
3617  * @param[in] action
3618  *   Pointer to the modify action.
3619  * @param[out] error
3620  *   Pointer to error structure.
3621  *
3622  * @return
3623  *   0 on success, a negative errno value otherwise and rte_errno is set.
3624  */
3625 static int
3626 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
3627                                    const struct rte_flow_action *action,
3628                                    struct rte_flow_error *error)
3629 {
3630         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
3631                 return rte_flow_error_set(error, EINVAL,
3632                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3633                                           NULL, "action configuration not set");
3634         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3635                 return rte_flow_error_set(error, EINVAL,
3636                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3637                                           "can't have encap action before"
3638                                           " modify action");
3639         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror)
3640                 return rte_flow_error_set(error, EINVAL,
3641                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3642                                           "can't support sample action before"
3643                                           " modify action for E-Switch"
3644                                           " mirroring");
3645         return 0;
3646 }
3647
3648 /**
3649  * Validate the modify-header MAC address actions.
3650  *
3651  * @param[in] action_flags
3652  *   Holds the actions detected until now.
3653  * @param[in] action
3654  *   Pointer to the modify action.
3655  * @param[in] item_flags
3656  *   Holds the items detected.
3657  * @param[out] error
3658  *   Pointer to error structure.
3659  *
3660  * @return
3661  *   0 on success, a negative errno value otherwise and rte_errno is set.
3662  */
3663 static int
3664 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
3665                                    const struct rte_flow_action *action,
3666                                    const uint64_t item_flags,
3667                                    struct rte_flow_error *error)
3668 {
3669         int ret = 0;
3670
3671         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3672         if (!ret) {
3673                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
3674                         return rte_flow_error_set(error, EINVAL,
3675                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3676                                                   NULL,
3677                                                   "no L2 item in pattern");
3678         }
3679         return ret;
3680 }
3681
3682 /**
3683  * Validate the modify-header IPv4 address actions.
3684  *
3685  * @param[in] action_flags
3686  *   Holds the actions detected until now.
3687  * @param[in] action
3688  *   Pointer to the modify action.
3689  * @param[in] item_flags
3690  *   Holds the items detected.
3691  * @param[out] error
3692  *   Pointer to error structure.
3693  *
3694  * @return
3695  *   0 on success, a negative errno value otherwise and rte_errno is set.
3696  */
3697 static int
3698 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3699                                     const struct rte_flow_action *action,
3700                                     const uint64_t item_flags,
3701                                     struct rte_flow_error *error)
3702 {
3703         int ret = 0;
3704         uint64_t layer;
3705
3706         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3707         if (!ret) {
3708                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3709                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3710                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3711                 if (!(item_flags & layer))
3712                         return rte_flow_error_set(error, EINVAL,
3713                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3714                                                   NULL,
3715                                                   "no ipv4 item in pattern");
3716         }
3717         return ret;
3718 }
3719
3720 /**
3721  * Validate the modify-header IPv6 address actions.
3722  *
3723  * @param[in] action_flags
3724  *   Holds the actions detected until now.
3725  * @param[in] action
3726  *   Pointer to the modify action.
3727  * @param[in] item_flags
3728  *   Holds the items detected.
3729  * @param[out] error
3730  *   Pointer to error structure.
3731  *
3732  * @return
3733  *   0 on success, a negative errno value otherwise and rte_errno is set.
3734  */
3735 static int
3736 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3737                                     const struct rte_flow_action *action,
3738                                     const uint64_t item_flags,
3739                                     struct rte_flow_error *error)
3740 {
3741         int ret = 0;
3742         uint64_t layer;
3743
3744         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3745         if (!ret) {
3746                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3747                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3748                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3749                 if (!(item_flags & layer))
3750                         return rte_flow_error_set(error, EINVAL,
3751                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3752                                                   NULL,
3753                                                   "no ipv6 item in pattern");
3754         }
3755         return ret;
3756 }
3757
3758 /**
3759  * Validate the modify-header TP actions.
3760  *
3761  * @param[in] action_flags
3762  *   Holds the actions detected until now.
3763  * @param[in] action
3764  *   Pointer to the modify action.
3765  * @param[in] item_flags
3766  *   Holds the items detected.
3767  * @param[out] error
3768  *   Pointer to error structure.
3769  *
3770  * @return
3771  *   0 on success, a negative errno value otherwise and rte_errno is set.
3772  */
3773 static int
3774 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3775                                   const struct rte_flow_action *action,
3776                                   const uint64_t item_flags,
3777                                   struct rte_flow_error *error)
3778 {
3779         int ret = 0;
3780         uint64_t layer;
3781
3782         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3783         if (!ret) {
3784                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3785                                  MLX5_FLOW_LAYER_INNER_L4 :
3786                                  MLX5_FLOW_LAYER_OUTER_L4;
3787                 if (!(item_flags & layer))
3788                         return rte_flow_error_set(error, EINVAL,
3789                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3790                                                   NULL, "no transport layer "
3791                                                   "in pattern");
3792         }
3793         return ret;
3794 }
3795
3796 /**
3797  * Validate the modify-header actions of increment/decrement
3798  * TCP Sequence-number.
3799  *
3800  * @param[in] action_flags
3801  *   Holds the actions detected until now.
3802  * @param[in] action
3803  *   Pointer to the modify action.
3804  * @param[in] item_flags
3805  *   Holds the items detected.
3806  * @param[out] error
3807  *   Pointer to error structure.
3808  *
3809  * @return
3810  *   0 on success, a negative errno value otherwise and rte_errno is set.
3811  */
3812 static int
3813 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3814                                        const struct rte_flow_action *action,
3815                                        const uint64_t item_flags,
3816                                        struct rte_flow_error *error)
3817 {
3818         int ret = 0;
3819         uint64_t layer;
3820
3821         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3822         if (!ret) {
3823                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3824                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3825                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3826                 if (!(item_flags & layer))
3827                         return rte_flow_error_set(error, EINVAL,
3828                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3829                                                   NULL, "no TCP item in"
3830                                                   " pattern");
3831                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3832                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3833                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3834                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3835                         return rte_flow_error_set(error, EINVAL,
3836                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3837                                                   NULL,
3838                                                   "cannot decrease and increase"
3839                                                   " TCP sequence number"
3840                                                   " at the same time");
3841         }
3842         return ret;
3843 }
3844
3845 /**
3846  * Validate the modify-header actions of increment/decrement
3847  * TCP Acknowledgment number.
3848  *
3849  * @param[in] action_flags
3850  *   Holds the actions detected until now.
3851  * @param[in] action
3852  *   Pointer to the modify action.
3853  * @param[in] item_flags
3854  *   Holds the items detected.
3855  * @param[out] error
3856  *   Pointer to error structure.
3857  *
3858  * @return
3859  *   0 on success, a negative errno value otherwise and rte_errno is set.
3860  */
3861 static int
3862 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3863                                        const struct rte_flow_action *action,
3864                                        const uint64_t item_flags,
3865                                        struct rte_flow_error *error)
3866 {
3867         int ret = 0;
3868         uint64_t layer;
3869
3870         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3871         if (!ret) {
3872                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3873                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3874                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3875                 if (!(item_flags & layer))
3876                         return rte_flow_error_set(error, EINVAL,
3877                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3878                                                   NULL, "no TCP item in"
3879                                                   " pattern");
3880                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3881                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3882                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3883                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3884                         return rte_flow_error_set(error, EINVAL,
3885                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3886                                                   NULL,
3887                                                   "cannot decrease and increase"
3888                                                   " TCP acknowledgment number"
3889                                                   " at the same time");
3890         }
3891         return ret;
3892 }
3893
3894 /**
3895  * Validate the modify-header TTL actions.
3896  *
3897  * @param[in] action_flags
3898  *   Holds the actions detected until now.
3899  * @param[in] action
3900  *   Pointer to the modify action.
3901  * @param[in] item_flags
3902  *   Holds the items detected.
3903  * @param[out] error
3904  *   Pointer to error structure.
3905  *
3906  * @return
3907  *   0 on success, a negative errno value otherwise and rte_errno is set.
3908  */
3909 static int
3910 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3911                                    const struct rte_flow_action *action,
3912                                    const uint64_t item_flags,
3913                                    struct rte_flow_error *error)
3914 {
3915         int ret = 0;
3916         uint64_t layer;
3917
3918         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3919         if (!ret) {
3920                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3921                                  MLX5_FLOW_LAYER_INNER_L3 :
3922                                  MLX5_FLOW_LAYER_OUTER_L3;
3923                 if (!(item_flags & layer))
3924                         return rte_flow_error_set(error, EINVAL,
3925                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3926                                                   NULL,
3927                                                   "no IP protocol in pattern");
3928         }
3929         return ret;
3930 }
3931
3932 /**
3933  * Validate jump action.
3934  *
3935  * @param[in] action
3936  *   Pointer to the jump action.
3937  * @param[in] action_flags
3938  *   Holds the actions detected until now.
3939  * @param[in] attributes
3940  *   Pointer to flow attributes
3941  * @param[in] external
3942  *   Action belongs to flow rule created by request external to PMD.
3943  * @param[out] error
3944  *   Pointer to error structure.
3945  *
3946  * @return
3947  *   0 on success, a negative errno value otherwise and rte_errno is set.
3948  */
3949 static int
3950 flow_dv_validate_action_jump(const struct rte_flow_action *action,
3951                              uint64_t action_flags,
3952                              const struct rte_flow_attr *attributes,
3953                              bool external, struct rte_flow_error *error)
3954 {
3955         uint32_t target_group, table;
3956         int ret = 0;
3957
3958         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3959                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3960                 return rte_flow_error_set(error, EINVAL,
3961                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3962                                           "can't have 2 fate actions in"
3963                                           " same flow");
3964         if (action_flags & MLX5_FLOW_ACTION_METER)
3965                 return rte_flow_error_set(error, ENOTSUP,
3966                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3967                                           "jump with meter not support");
3968         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror)
3969                 return rte_flow_error_set(error, EINVAL,
3970                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3971                                           "E-Switch mirroring can't support"
3972                                           " Sample action and jump action in"
3973                                           " same flow now");
3974         if (!action->conf)
3975                 return rte_flow_error_set(error, EINVAL,
3976                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3977                                           NULL, "action configuration not set");
3978         target_group =
3979                 ((const struct rte_flow_action_jump *)action->conf)->group;
3980         ret = mlx5_flow_group_to_table(attributes, external, target_group,
3981                                        true, &table, error);
3982         if (ret)
3983                 return ret;
3984         if (attributes->group == target_group)
3985                 return rte_flow_error_set(error, EINVAL,
3986                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3987                                           "target group must be other than"
3988                                           " the current flow group");
3989         return 0;
3990 }
3991
3992 /*
3993  * Validate the port_id action.
3994  *
3995  * @param[in] dev
3996  *   Pointer to rte_eth_dev structure.
3997  * @param[in] action_flags
3998  *   Bit-fields that holds the actions detected until now.
3999  * @param[in] action
4000  *   Port_id RTE action structure.
4001  * @param[in] attr
4002  *   Attributes of flow that includes this action.
4003  * @param[out] error
4004  *   Pointer to error structure.
4005  *
4006  * @return
4007  *   0 on success, a negative errno value otherwise and rte_errno is set.
4008  */
4009 static int
4010 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
4011                                 uint64_t action_flags,
4012                                 const struct rte_flow_action *action,
4013                                 const struct rte_flow_attr *attr,
4014                                 struct rte_flow_error *error)
4015 {
4016         const struct rte_flow_action_port_id *port_id;
4017         struct mlx5_priv *act_priv;
4018         struct mlx5_priv *dev_priv;
4019         uint16_t port;
4020
4021         if (!attr->transfer)
4022                 return rte_flow_error_set(error, ENOTSUP,
4023                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4024                                           NULL,
4025                                           "port id action is valid in transfer"
4026                                           " mode only");
4027         if (!action || !action->conf)
4028                 return rte_flow_error_set(error, ENOTSUP,
4029                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4030                                           NULL,
4031                                           "port id action parameters must be"
4032                                           " specified");
4033         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
4034                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4035                 return rte_flow_error_set(error, EINVAL,
4036                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4037                                           "can have only one fate actions in"
4038                                           " a flow");
4039         dev_priv = mlx5_dev_to_eswitch_info(dev);
4040         if (!dev_priv)
4041                 return rte_flow_error_set(error, rte_errno,
4042                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4043                                           NULL,
4044                                           "failed to obtain E-Switch info");
4045         port_id = action->conf;
4046         port = port_id->original ? dev->data->port_id : port_id->id;
4047         act_priv = mlx5_port_to_eswitch_info(port, false);
4048         if (!act_priv)
4049                 return rte_flow_error_set
4050                                 (error, rte_errno,
4051                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
4052                                  "failed to obtain E-Switch port id for port");
4053         if (act_priv->domain_id != dev_priv->domain_id)
4054                 return rte_flow_error_set
4055                                 (error, EINVAL,
4056                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4057                                  "port does not belong to"
4058                                  " E-Switch being configured");
4059         return 0;
4060 }
4061
4062 /**
4063  * Get the maximum number of modify header actions.
4064  *
4065  * @param dev
4066  *   Pointer to rte_eth_dev structure.
4067  * @param flags
4068  *   Flags bits to check if root level.
4069  *
4070  * @return
4071  *   Max number of modify header actions device can support.
4072  */
4073 static inline unsigned int
4074 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
4075                               uint64_t flags)
4076 {
4077         /*
4078          * There's no way to directly query the max capacity from FW.
4079          * The maximal value on root table should be assumed to be supported.
4080          */
4081         if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
4082                 return MLX5_MAX_MODIFY_NUM;
4083         else
4084                 return MLX5_ROOT_TBL_MODIFY_NUM;
4085 }
4086
4087 /**
4088  * Validate the meter action.
4089  *
4090  * @param[in] dev
4091  *   Pointer to rte_eth_dev structure.
4092  * @param[in] action_flags
4093  *   Bit-fields that holds the actions detected until now.
4094  * @param[in] action
4095  *   Pointer to the meter action.
4096  * @param[in] attr
4097  *   Attributes of flow that includes this action.
4098  * @param[out] error
4099  *   Pointer to error structure.
4100  *
4101  * @return
4102  *   0 on success, a negative errno value otherwise and rte_ernno is set.
4103  */
4104 static int
4105 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
4106                                 uint64_t action_flags,
4107                                 const struct rte_flow_action *action,
4108                                 const struct rte_flow_attr *attr,
4109                                 struct rte_flow_error *error)
4110 {
4111         struct mlx5_priv *priv = dev->data->dev_private;
4112         const struct rte_flow_action_meter *am = action->conf;
4113         struct mlx5_flow_meter *fm;
4114
4115         if (!am)
4116                 return rte_flow_error_set(error, EINVAL,
4117                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4118                                           "meter action conf is NULL");
4119
4120         if (action_flags & MLX5_FLOW_ACTION_METER)
4121                 return rte_flow_error_set(error, ENOTSUP,
4122                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4123                                           "meter chaining not support");
4124         if (action_flags & MLX5_FLOW_ACTION_JUMP)
4125                 return rte_flow_error_set(error, ENOTSUP,
4126                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4127                                           "meter with jump not support");
4128         if (!priv->mtr_en)
4129                 return rte_flow_error_set(error, ENOTSUP,
4130                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4131                                           NULL,
4132                                           "meter action not supported");
4133         fm = mlx5_flow_meter_find(priv, am->mtr_id);
4134         if (!fm)
4135                 return rte_flow_error_set(error, EINVAL,
4136                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4137                                           "Meter not found");
4138         if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
4139               (!fm->ingress && !attr->ingress && attr->egress) ||
4140               (!fm->egress && !attr->egress && attr->ingress))))
4141                 return rte_flow_error_set(error, EINVAL,
4142                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4143                                           "Flow attributes are either invalid "
4144                                           "or have a conflict with current "
4145                                           "meter attributes");
4146         return 0;
4147 }
4148
4149 /**
4150  * Validate the age action.
4151  *
4152  * @param[in] action_flags
4153  *   Holds the actions detected until now.
4154  * @param[in] action
4155  *   Pointer to the age action.
4156  * @param[in] dev
4157  *   Pointer to the Ethernet device structure.
4158  * @param[out] error
4159  *   Pointer to error structure.
4160  *
4161  * @return
4162  *   0 on success, a negative errno value otherwise and rte_errno is set.
4163  */
4164 static int
4165 flow_dv_validate_action_age(uint64_t action_flags,
4166                             const struct rte_flow_action *action,
4167                             struct rte_eth_dev *dev,
4168                             struct rte_flow_error *error)
4169 {
4170         struct mlx5_priv *priv = dev->data->dev_private;
4171         const struct rte_flow_action_age *age = action->conf;
4172
4173         if (!priv->config.devx || priv->counter_fallback)
4174                 return rte_flow_error_set(error, ENOTSUP,
4175                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4176                                           NULL,
4177                                           "age action not supported");
4178         if (!(action->conf))
4179                 return rte_flow_error_set(error, EINVAL,
4180                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4181                                           "configuration cannot be null");
4182         if (age->timeout >= UINT16_MAX / 2 / 10)
4183                 return rte_flow_error_set(error, ENOTSUP,
4184                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4185                                           "Max age time: 3275 seconds");
4186         if (action_flags & MLX5_FLOW_ACTION_AGE)
4187                 return rte_flow_error_set(error, EINVAL,
4188                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4189                                           "Duplicate age ctions set");
4190         return 0;
4191 }
4192
4193 /**
4194  * Validate the modify-header IPv4 DSCP actions.
4195  *
4196  * @param[in] action_flags
4197  *   Holds the actions detected until now.
4198  * @param[in] action
4199  *   Pointer to the modify action.
4200  * @param[in] item_flags
4201  *   Holds the items detected.
4202  * @param[out] error
4203  *   Pointer to error structure.
4204  *
4205  * @return
4206  *   0 on success, a negative errno value otherwise and rte_errno is set.
4207  */
4208 static int
4209 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
4210                                          const struct rte_flow_action *action,
4211                                          const uint64_t item_flags,
4212                                          struct rte_flow_error *error)
4213 {
4214         int ret = 0;
4215
4216         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4217         if (!ret) {
4218                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
4219                         return rte_flow_error_set(error, EINVAL,
4220                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4221                                                   NULL,
4222                                                   "no ipv4 item in pattern");
4223         }
4224         return ret;
4225 }
4226
4227 /**
4228  * Validate the modify-header IPv6 DSCP actions.
4229  *
4230  * @param[in] action_flags
4231  *   Holds the actions detected until now.
4232  * @param[in] action
4233  *   Pointer to the modify action.
4234  * @param[in] item_flags
4235  *   Holds the items detected.
4236  * @param[out] error
4237  *   Pointer to error structure.
4238  *
4239  * @return
4240  *   0 on success, a negative errno value otherwise and rte_errno is set.
4241  */
4242 static int
4243 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
4244                                          const struct rte_flow_action *action,
4245                                          const uint64_t item_flags,
4246                                          struct rte_flow_error *error)
4247 {
4248         int ret = 0;
4249
4250         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4251         if (!ret) {
4252                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
4253                         return rte_flow_error_set(error, EINVAL,
4254                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4255                                                   NULL,
4256                                                   "no ipv6 item in pattern");
4257         }
4258         return ret;
4259 }
4260
4261 /**
4262  * Match modify-header resource.
4263  *
4264  * @param entry
4265  *   Pointer to exist resource entry object.
4266  * @param ctx
4267  *   Pointer to new modify-header resource.
4268  *
4269  * @return
4270  *   0 on matching, -1 otherwise.
4271  */
4272 static int
4273 flow_dv_modify_hdr_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
4274 {
4275         struct mlx5_flow_dv_modify_hdr_resource *resource;
4276         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4277         uint32_t actions_len;
4278
4279         resource = (struct mlx5_flow_dv_modify_hdr_resource *)ctx;
4280         cache_resource = container_of(entry,
4281                                       struct mlx5_flow_dv_modify_hdr_resource,
4282                                       entry);
4283         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4284         if (resource->entry.key == cache_resource->entry.key &&
4285             resource->ft_type == cache_resource->ft_type &&
4286             resource->actions_num == cache_resource->actions_num &&
4287             resource->flags == cache_resource->flags &&
4288             !memcmp((const void *)resource->actions,
4289                     (const void *)cache_resource->actions,
4290                     actions_len))
4291                 return 0;
4292         return -1;
4293 }
4294
4295 /**
4296  * Validate the sample action.
4297  *
4298  * @param[in] action_flags
4299  *   Holds the actions detected until now.
4300  * @param[in] action
4301  *   Pointer to the sample action.
4302  * @param[in] dev
4303  *   Pointer to the Ethernet device structure.
4304  * @param[in] attr
4305  *   Attributes of flow that includes this action.
4306  * @param[out] error
4307  *   Pointer to error structure.
4308  *
4309  * @return
4310  *   0 on success, a negative errno value otherwise and rte_errno is set.
4311  */
4312 static int
4313 flow_dv_validate_action_sample(uint64_t action_flags,
4314                                const struct rte_flow_action *action,
4315                                struct rte_eth_dev *dev,
4316                                const struct rte_flow_attr *attr,
4317                                struct rte_flow_error *error)
4318 {
4319         struct mlx5_priv *priv = dev->data->dev_private;
4320         struct mlx5_dev_config *dev_conf = &priv->config;
4321         const struct rte_flow_action_sample *sample = action->conf;
4322         const struct rte_flow_action *act;
4323         uint64_t sub_action_flags = 0;
4324         uint16_t queue_index = 0xFFFF;
4325         int actions_n = 0;
4326         int ret;
4327         fdb_mirror = 0;
4328
4329         if (!sample)
4330                 return rte_flow_error_set(error, EINVAL,
4331                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4332                                           "configuration cannot be NULL");
4333         if (sample->ratio == 0)
4334                 return rte_flow_error_set(error, EINVAL,
4335                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4336                                           "ratio value starts from 1");
4337         if (!priv->config.devx || (sample->ratio > 0 && !priv->sampler_en))
4338                 return rte_flow_error_set(error, ENOTSUP,
4339                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4340                                           NULL,
4341                                           "sample action not supported");
4342         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
4343                 return rte_flow_error_set(error, EINVAL,
4344                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4345                                           "Multiple sample actions not "
4346                                           "supported");
4347         if (action_flags & MLX5_FLOW_ACTION_METER)
4348                 return rte_flow_error_set(error, EINVAL,
4349                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4350                                           "wrong action order, meter should "
4351                                           "be after sample action");
4352         if (action_flags & MLX5_FLOW_ACTION_JUMP)
4353                 return rte_flow_error_set(error, EINVAL,
4354                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4355                                           "wrong action order, jump should "
4356                                           "be after sample action");
4357         act = sample->actions;
4358         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
4359                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
4360                         return rte_flow_error_set(error, ENOTSUP,
4361                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4362                                                   act, "too many actions");
4363                 switch (act->type) {
4364                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4365                         ret = mlx5_flow_validate_action_queue(act,
4366                                                               sub_action_flags,
4367                                                               dev,
4368                                                               attr, error);
4369                         if (ret < 0)
4370                                 return ret;
4371                         queue_index = ((const struct rte_flow_action_queue *)
4372                                                         (act->conf))->index;
4373                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
4374                         ++actions_n;
4375                         break;
4376                 case RTE_FLOW_ACTION_TYPE_MARK:
4377                         ret = flow_dv_validate_action_mark(dev, act,
4378                                                            sub_action_flags,
4379                                                            attr, error);
4380                         if (ret < 0)
4381                                 return ret;
4382                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
4383                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
4384                                                 MLX5_FLOW_ACTION_MARK_EXT;
4385                         else
4386                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
4387                         ++actions_n;
4388                         break;
4389                 case RTE_FLOW_ACTION_TYPE_COUNT:
4390                         ret = flow_dv_validate_action_count(dev, error);
4391                         if (ret < 0)
4392                                 return ret;
4393                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
4394                         ++actions_n;
4395                         break;
4396                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
4397                         ret = flow_dv_validate_action_port_id(dev,
4398                                                               sub_action_flags,
4399                                                               act,
4400                                                               attr,
4401                                                               error);
4402                         if (ret)
4403                                 return ret;
4404                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
4405                         ++actions_n;
4406                         break;
4407                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4408                         ret = flow_dv_validate_action_raw_encap_decap
4409                                 (dev, NULL, act->conf, attr, &sub_action_flags,
4410                                  &actions_n, error);
4411                         if (ret < 0)
4412                                 return ret;
4413                         ++actions_n;
4414                         break;
4415                 default:
4416                         return rte_flow_error_set(error, ENOTSUP,
4417                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4418                                                   NULL,
4419                                                   "Doesn't support optional "
4420                                                   "action");
4421                 }
4422         }
4423         if (attr->ingress && !attr->transfer) {
4424                 if (!(sub_action_flags & MLX5_FLOW_ACTION_QUEUE))
4425                         return rte_flow_error_set(error, EINVAL,
4426                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4427                                                   NULL,
4428                                                   "Ingress must has a dest "
4429                                                   "QUEUE for Sample");
4430         } else if (attr->egress && !attr->transfer) {
4431                 return rte_flow_error_set(error, ENOTSUP,
4432                                           RTE_FLOW_ERROR_TYPE_ACTION,
4433                                           NULL,
4434                                           "Sample Only support Ingress "
4435                                           "or E-Switch");
4436         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
4437                 MLX5_ASSERT(attr->transfer);
4438                 if (sample->ratio > 1)
4439                         return rte_flow_error_set(error, ENOTSUP,
4440                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4441                                                   NULL,
4442                                                   "E-Switch doesn't support "
4443                                                   "any optional action "
4444                                                   "for sampling");
4445                 fdb_mirror = 1;
4446                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
4447                         return rte_flow_error_set(error, ENOTSUP,
4448                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4449                                                   NULL,
4450                                                   "unsupported action QUEUE");
4451                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
4452                         return rte_flow_error_set(error, EINVAL,
4453                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4454                                                   NULL,
4455                                                   "E-Switch must has a dest "
4456                                                   "port for mirroring");
4457         }
4458         /* Continue validation for Xcap actions.*/
4459         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
4460             (queue_index == 0xFFFF ||
4461              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
4462                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
4463                      MLX5_FLOW_XCAP_ACTIONS)
4464                         return rte_flow_error_set(error, ENOTSUP,
4465                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4466                                                   NULL, "encap and decap "
4467                                                   "combination aren't "
4468                                                   "supported");
4469                 if (!attr->transfer && attr->ingress && (sub_action_flags &
4470                                                         MLX5_FLOW_ACTION_ENCAP))
4471                         return rte_flow_error_set(error, ENOTSUP,
4472                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4473                                                   NULL, "encap is not supported"
4474                                                   " for ingress traffic");
4475         }
4476         return 0;
4477 }
4478
4479 /**
4480  * Find existing modify-header resource or create and register a new one.
4481  *
4482  * @param dev[in, out]
4483  *   Pointer to rte_eth_dev structure.
4484  * @param[in, out] resource
4485  *   Pointer to modify-header resource.
4486  * @parm[in, out] dev_flow
4487  *   Pointer to the dev_flow.
4488  * @param[out] error
4489  *   pointer to error structure.
4490  *
4491  * @return
4492  *   0 on success otherwise -errno and errno is set.
4493  */
4494 static int
4495 flow_dv_modify_hdr_resource_register
4496                         (struct rte_eth_dev *dev,
4497                          struct mlx5_flow_dv_modify_hdr_resource *resource,
4498                          struct mlx5_flow *dev_flow,
4499                          struct rte_flow_error *error)
4500 {
4501         struct mlx5_priv *priv = dev->data->dev_private;
4502         struct mlx5_dev_ctx_shared *sh = priv->sh;
4503         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4504         struct mlx5dv_dr_domain *ns;
4505         uint32_t actions_len;
4506         struct mlx5_hlist_entry *entry;
4507         union mlx5_flow_modify_hdr_key hdr_mod_key = {
4508                 {
4509                         .ft_type = resource->ft_type,
4510                         .actions_num = resource->actions_num,
4511                         .group = dev_flow->dv.group,
4512                         .cksum = 0,
4513                 }
4514         };
4515         int ret;
4516
4517         resource->flags = dev_flow->dv.group ? 0 :
4518                           MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4519         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
4520                                     resource->flags))
4521                 return rte_flow_error_set(error, EOVERFLOW,
4522                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4523                                           "too many modify header items");
4524         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
4525                 ns = sh->fdb_domain;
4526         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
4527                 ns = sh->tx_domain;
4528         else
4529                 ns = sh->rx_domain;
4530         /* Lookup a matching resource from cache. */
4531         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4532         hdr_mod_key.cksum = __rte_raw_cksum(resource->actions, actions_len, 0);
4533         resource->entry.key = hdr_mod_key.v64;
4534         entry = mlx5_hlist_lookup_ex(sh->modify_cmds, resource->entry.key,
4535                                      flow_dv_modify_hdr_resource_match,
4536                                      (void *)resource);
4537         if (entry) {
4538                 cache_resource = container_of(entry,
4539                                         struct mlx5_flow_dv_modify_hdr_resource,
4540                                         entry);
4541                 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
4542                         (void *)cache_resource,
4543                         rte_atomic32_read(&cache_resource->refcnt));
4544                 rte_atomic32_inc(&cache_resource->refcnt);
4545                 dev_flow->handle->dvh.modify_hdr = cache_resource;
4546                 return 0;
4547
4548         }
4549         /* Register new modify-header resource. */
4550         cache_resource = mlx5_malloc(MLX5_MEM_ZERO,
4551                                     sizeof(*cache_resource) + actions_len, 0,
4552                                     SOCKET_ID_ANY);
4553         if (!cache_resource)
4554                 return rte_flow_error_set(error, ENOMEM,
4555                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4556                                           "cannot allocate resource memory");
4557         *cache_resource = *resource;
4558         rte_memcpy(cache_resource->actions, resource->actions, actions_len);
4559         ret = mlx5_flow_os_create_flow_action_modify_header
4560                                         (sh->ctx, ns, cache_resource,
4561                                          actions_len, &cache_resource->action);
4562         if (ret) {
4563                 mlx5_free(cache_resource);
4564                 return rte_flow_error_set(error, ENOMEM,
4565                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4566                                           NULL, "cannot create action");
4567         }
4568         rte_atomic32_init(&cache_resource->refcnt);
4569         rte_atomic32_inc(&cache_resource->refcnt);
4570         if (mlx5_hlist_insert_ex(sh->modify_cmds, &cache_resource->entry,
4571                                  flow_dv_modify_hdr_resource_match,
4572                                  (void *)cache_resource)) {
4573                 claim_zero(mlx5_flow_os_destroy_flow_action
4574                                                 (cache_resource->action));
4575                 mlx5_free(cache_resource);
4576                 return rte_flow_error_set(error, EEXIST,
4577                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4578                                           NULL, "action exist");
4579         }
4580         dev_flow->handle->dvh.modify_hdr = cache_resource;
4581         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
4582                 (void *)cache_resource,
4583                 rte_atomic32_read(&cache_resource->refcnt));
4584         return 0;
4585 }
4586
4587 /**
4588  * Get DV flow counter by index.
4589  *
4590  * @param[in] dev
4591  *   Pointer to the Ethernet device structure.
4592  * @param[in] idx
4593  *   mlx5 flow counter index in the container.
4594  * @param[out] ppool
4595  *   mlx5 flow counter pool in the container,
4596  *
4597  * @return
4598  *   Pointer to the counter, NULL otherwise.
4599  */
4600 static struct mlx5_flow_counter *
4601 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
4602                            uint32_t idx,
4603                            struct mlx5_flow_counter_pool **ppool)
4604 {
4605         struct mlx5_priv *priv = dev->data->dev_private;
4606         struct mlx5_pools_container *cont;
4607         struct mlx5_flow_counter_pool *pool;
4608         uint32_t batch = 0, age = 0;
4609
4610         idx--;
4611         age = MLX_CNT_IS_AGE(idx);
4612         idx = age ? idx - MLX5_CNT_AGE_OFFSET : idx;
4613         if (idx >= MLX5_CNT_BATCH_OFFSET) {
4614                 idx -= MLX5_CNT_BATCH_OFFSET;
4615                 batch = 1;
4616         }
4617         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4618         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n);
4619         pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
4620         MLX5_ASSERT(pool);
4621         if (ppool)
4622                 *ppool = pool;
4623         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
4624 }
4625
4626 /**
4627  * Check the devx counter belongs to the pool.
4628  *
4629  * @param[in] pool
4630  *   Pointer to the counter pool.
4631  * @param[in] id
4632  *   The counter devx ID.
4633  *
4634  * @return
4635  *   True if counter belongs to the pool, false otherwise.
4636  */
4637 static bool
4638 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
4639 {
4640         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
4641                    MLX5_COUNTERS_PER_POOL;
4642
4643         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
4644                 return true;
4645         return false;
4646 }
4647
4648 /**
4649  * Get a pool by devx counter ID.
4650  *
4651  * @param[in] cont
4652  *   Pointer to the counter container.
4653  * @param[in] id
4654  *   The counter devx ID.
4655  *
4656  * @return
4657  *   The counter pool pointer if exists, NULL otherwise,
4658  */
4659 static struct mlx5_flow_counter_pool *
4660 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
4661 {
4662         uint32_t i;
4663
4664         /* Check last used pool. */
4665         if (cont->last_pool_idx != POOL_IDX_INVALID &&
4666             flow_dv_is_counter_in_pool(cont->pools[cont->last_pool_idx], id))
4667                 return cont->pools[cont->last_pool_idx];
4668         /* ID out of range means no suitable pool in the container. */
4669         if (id > cont->max_id || id < cont->min_id)
4670                 return NULL;
4671         /*
4672          * Find the pool from the end of the container, since mostly counter
4673          * ID is sequence increasing, and the last pool should be the needed
4674          * one.
4675          */
4676         i = rte_atomic16_read(&cont->n_valid);
4677         while (i--) {
4678                 struct mlx5_flow_counter_pool *pool = cont->pools[i];
4679
4680                 if (flow_dv_is_counter_in_pool(pool, id))
4681                         return pool;
4682         }
4683         return NULL;
4684 }
4685
4686 /**
4687  * Allocate a new memory for the counter values wrapped by all the needed
4688  * management.
4689  *
4690  * @param[in] dev
4691  *   Pointer to the Ethernet device structure.
4692  * @param[in] raws_n
4693  *   The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
4694  *
4695  * @return
4696  *   The new memory management pointer on success, otherwise NULL and rte_errno
4697  *   is set.
4698  */
4699 static struct mlx5_counter_stats_mem_mng *
4700 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
4701 {
4702         struct mlx5_priv *priv = dev->data->dev_private;
4703         struct mlx5_dev_ctx_shared *sh = priv->sh;
4704         struct mlx5_devx_mkey_attr mkey_attr;
4705         struct mlx5_counter_stats_mem_mng *mem_mng;
4706         volatile struct flow_counter_stats *raw_data;
4707         int size = (sizeof(struct flow_counter_stats) *
4708                         MLX5_COUNTERS_PER_POOL +
4709                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
4710                         sizeof(struct mlx5_counter_stats_mem_mng);
4711         size_t pgsize = rte_mem_page_size();
4712         if (pgsize == (size_t)-1) {
4713                 DRV_LOG(ERR, "Failed to get mem page size");
4714                 rte_errno = ENOMEM;
4715                 return NULL;
4716         }
4717         uint8_t *mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize,
4718                                   SOCKET_ID_ANY);
4719         int i;
4720
4721         if (!mem) {
4722                 rte_errno = ENOMEM;
4723                 return NULL;
4724         }
4725         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
4726         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
4727         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
4728                                                  IBV_ACCESS_LOCAL_WRITE);
4729         if (!mem_mng->umem) {
4730                 rte_errno = errno;
4731                 mlx5_free(mem);
4732                 return NULL;
4733         }
4734         mkey_attr.addr = (uintptr_t)mem;
4735         mkey_attr.size = size;
4736         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
4737         mkey_attr.pd = sh->pdn;
4738         mkey_attr.log_entity_size = 0;
4739         mkey_attr.pg_access = 0;
4740         mkey_attr.klm_array = NULL;
4741         mkey_attr.klm_num = 0;
4742         if (priv->config.hca_attr.relaxed_ordering_write &&
4743                 priv->config.hca_attr.relaxed_ordering_read  &&
4744                 !haswell_broadwell_cpu)
4745                 mkey_attr.relaxed_ordering = 1;
4746         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
4747         if (!mem_mng->dm) {
4748                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
4749                 rte_errno = errno;
4750                 mlx5_free(mem);
4751                 return NULL;
4752         }
4753         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
4754         raw_data = (volatile struct flow_counter_stats *)mem;
4755         for (i = 0; i < raws_n; ++i) {
4756                 mem_mng->raws[i].mem_mng = mem_mng;
4757                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
4758         }
4759         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
4760         return mem_mng;
4761 }
4762
4763 /**
4764  * Resize a counter container.
4765  *
4766  * @param[in] dev
4767  *   Pointer to the Ethernet device structure.
4768  * @param[in] batch
4769  *   Whether the pool is for counter that was allocated by batch command.
4770  * @param[in] age
4771  *   Whether the pool is for Aging counter.
4772  *
4773  * @return
4774  *   0 on success, otherwise negative errno value and rte_errno is set.
4775  */
4776 static int
4777 flow_dv_container_resize(struct rte_eth_dev *dev,
4778                                 uint32_t batch, uint32_t age)
4779 {
4780         struct mlx5_priv *priv = dev->data->dev_private;
4781         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4782                                                                age);
4783         struct mlx5_counter_stats_mem_mng *mem_mng = NULL;
4784         void *old_pools = cont->pools;
4785         uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
4786         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
4787         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
4788
4789         if (!pools) {
4790                 rte_errno = ENOMEM;
4791                 return -ENOMEM;
4792         }
4793         if (old_pools)
4794                 memcpy(pools, old_pools, cont->n *
4795                                        sizeof(struct mlx5_flow_counter_pool *));
4796         /*
4797          * Fallback mode query the counter directly, no background query
4798          * resources are needed.
4799          */
4800         if (!priv->counter_fallback) {
4801                 int i;
4802
4803                 mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
4804                           MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
4805                 if (!mem_mng) {
4806                         mlx5_free(pools);
4807                         return -ENOMEM;
4808                 }
4809                 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
4810                         LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
4811                                          mem_mng->raws +
4812                                          MLX5_CNT_CONTAINER_RESIZE +
4813                                          i, next);
4814         }
4815         rte_spinlock_lock(&cont->resize_sl);
4816         cont->n = resize;
4817         cont->mem_mng = mem_mng;
4818         cont->pools = pools;
4819         rte_spinlock_unlock(&cont->resize_sl);
4820         if (old_pools)
4821                 mlx5_free(old_pools);
4822         return 0;
4823 }
4824
4825 /**
4826  * Query a devx flow counter.
4827  *
4828  * @param[in] dev
4829  *   Pointer to the Ethernet device structure.
4830  * @param[in] cnt
4831  *   Index to the flow counter.
4832  * @param[out] pkts
4833  *   The statistics value of packets.
4834  * @param[out] bytes
4835  *   The statistics value of bytes.
4836  *
4837  * @return
4838  *   0 on success, otherwise a negative errno value and rte_errno is set.
4839  */
4840 static inline int
4841 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
4842                      uint64_t *bytes)
4843 {
4844         struct mlx5_priv *priv = dev->data->dev_private;
4845         struct mlx5_flow_counter_pool *pool = NULL;
4846         struct mlx5_flow_counter *cnt;
4847         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4848         int offset;
4849
4850         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4851         MLX5_ASSERT(pool);
4852         if (counter < MLX5_CNT_BATCH_OFFSET) {
4853                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4854                 if (priv->counter_fallback)
4855                         return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
4856                                         0, pkts, bytes, 0, NULL, NULL, 0);
4857         }
4858
4859         rte_spinlock_lock(&pool->sl);
4860         /*
4861          * The single counters allocation may allocate smaller ID than the
4862          * current allocated in parallel to the host reading.
4863          * In this case the new counter values must be reported as 0.
4864          */
4865         if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) {
4866                 *pkts = 0;
4867                 *bytes = 0;
4868         } else {
4869                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
4870                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
4871                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
4872         }
4873         rte_spinlock_unlock(&pool->sl);
4874         return 0;
4875 }
4876
4877 /**
4878  * Create and initialize a new counter pool.
4879  *
4880  * @param[in] dev
4881  *   Pointer to the Ethernet device structure.
4882  * @param[out] dcs
4883  *   The devX counter handle.
4884  * @param[in] batch
4885  *   Whether the pool is for counter that was allocated by batch command.
4886  * @param[in] age
4887  *   Whether the pool is for counter that was allocated for aging.
4888  * @param[in/out] cont_cur
4889  *   Pointer to the container pointer, it will be update in pool resize.
4890  *
4891  * @return
4892  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
4893  */
4894 static struct mlx5_flow_counter_pool *
4895 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
4896                     uint32_t batch, uint32_t age)
4897 {
4898         struct mlx5_priv *priv = dev->data->dev_private;
4899         struct mlx5_flow_counter_pool *pool;
4900         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4901                                                                age);
4902         int16_t n_valid = rte_atomic16_read(&cont->n_valid);
4903         uint32_t size = sizeof(*pool);
4904
4905         if (cont->n == n_valid && flow_dv_container_resize(dev, batch, age))
4906                 return NULL;
4907         size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
4908         size += (batch ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
4909         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE);
4910         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
4911         if (!pool) {
4912                 rte_errno = ENOMEM;
4913                 return NULL;
4914         }
4915         pool->min_dcs = dcs;
4916         if (!priv->counter_fallback)
4917                 pool->raw = cont->mem_mng->raws + n_valid %
4918                                                       MLX5_CNT_CONTAINER_RESIZE;
4919         pool->raw_hw = NULL;
4920         pool->type = 0;
4921         pool->type |= (batch ? 0 :  CNT_POOL_TYPE_EXT);
4922         pool->type |= (!age ? 0 :  CNT_POOL_TYPE_AGE);
4923         pool->query_gen = 0;
4924         rte_spinlock_init(&pool->sl);
4925         TAILQ_INIT(&pool->counters[0]);
4926         TAILQ_INIT(&pool->counters[1]);
4927         TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
4928         pool->index = n_valid;
4929         cont->pools[n_valid] = pool;
4930         if (!batch) {
4931                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
4932
4933                 if (base < cont->min_id)
4934                         cont->min_id = base;
4935                 if (base > cont->max_id)
4936                         cont->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
4937                 cont->last_pool_idx = pool->index;
4938         }
4939         /* Pool initialization must be updated before host thread access. */
4940         rte_io_wmb();
4941         rte_atomic16_add(&cont->n_valid, 1);
4942         return pool;
4943 }
4944
4945 /**
4946  * Restore skipped counters in the pool.
4947  *
4948  * As counter pool query requires the first counter dcs
4949  * ID start with 4 alinged, if the pool counters with
4950  * min_dcs ID are not aligned with 4, the counters will
4951  * be skipped.
4952  * Once other min_dcs ID less than these skipped counter
4953  * dcs ID appears, the skipped counters will be safe to
4954  * use.
4955  * Should be called when min_dcs is updated.
4956  *
4957  * @param[in] pool
4958  *   Current counter pool.
4959  * @param[in] last_min_dcs
4960  *   Last min_dcs.
4961  */
4962 static void
4963 flow_dv_counter_restore(struct mlx5_flow_counter_pool *pool,
4964                         struct mlx5_devx_obj *last_min_dcs)
4965 {
4966         struct mlx5_flow_counter_ext *cnt_ext;
4967         uint32_t offset, new_offset;
4968         uint32_t skip_cnt = 0;
4969         uint32_t i;
4970
4971         if (!pool->skip_cnt)
4972                 return;
4973         /*
4974          * If last min_dcs is not valid. The skipped counter may even after
4975          * last min_dcs, set the offset to the whole pool.
4976          */
4977         if (last_min_dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))
4978                 offset = MLX5_COUNTERS_PER_POOL;
4979         else
4980                 offset = last_min_dcs->id % MLX5_COUNTERS_PER_POOL;
4981         new_offset = pool->min_dcs->id % MLX5_COUNTERS_PER_POOL;
4982         /*
4983          * Check the counters from 1 to the last_min_dcs range. Counters
4984          * before new min_dcs indicates pool still has skipped counters.
4985          * Counters be skipped after new min_dcs will be ready to use.
4986          * Offset 0 counter must be empty or min_dcs, start from 1.
4987          */
4988         for (i = 1; i < offset; i++) {
4989                 cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
4990                 if (cnt_ext->skipped) {
4991                         if (i > new_offset) {
4992                                 cnt_ext->skipped = 0;
4993                                 TAILQ_INSERT_TAIL
4994                                         (&pool->counters[pool->query_gen],
4995                                          MLX5_POOL_GET_CNT(pool, i), next);
4996                         } else {
4997                                 skip_cnt++;
4998                         }
4999                 }
5000         }
5001         if (!skip_cnt)
5002                 pool->skip_cnt = 0;
5003 }
5004
5005 /**
5006  * Prepare a new counter and/or a new counter pool.
5007  *
5008  * @param[in] dev
5009  *   Pointer to the Ethernet device structure.
5010  * @param[out] cnt_free
5011  *   Where to put the pointer of a new counter.
5012  * @param[in] batch
5013  *   Whether the pool is for counter that was allocated by batch command.
5014  * @param[in] age
5015  *   Whether the pool is for counter that was allocated for aging.
5016  *
5017  * @return
5018  *   The counter pool pointer and @p cnt_free is set on success,
5019  *   NULL otherwise and rte_errno is set.
5020  */
5021 static struct mlx5_flow_counter_pool *
5022 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
5023                              struct mlx5_flow_counter **cnt_free,
5024                              uint32_t batch, uint32_t age)
5025 {
5026         struct mlx5_priv *priv = dev->data->dev_private;
5027         struct mlx5_pools_container *cont;
5028         struct mlx5_flow_counter_pool *pool;
5029         struct mlx5_counters tmp_tq;
5030         struct mlx5_devx_obj *last_min_dcs;
5031         struct mlx5_devx_obj *dcs = NULL;
5032         struct mlx5_flow_counter *cnt;
5033         uint32_t add2other;
5034         uint32_t i;
5035
5036         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
5037         if (!batch) {
5038 retry:
5039                 add2other = 0;
5040                 /* bulk_bitmap must be 0 for single counter allocation. */
5041                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
5042                 if (!dcs)
5043                         return NULL;
5044                 pool = flow_dv_find_pool_by_id(cont, dcs->id);
5045                 /* Check if counter belongs to exist pool ID range. */
5046                 if (!pool) {
5047                         pool = flow_dv_find_pool_by_id
5048                                (MLX5_CNT_CONTAINER
5049                                (priv->sh, batch, (age ^ 0x1)), dcs->id);
5050                         /*
5051                          * Pool eixsts, counter will be added to the other
5052                          * container, need to reallocate it later.
5053                          */
5054                         if (pool) {
5055                                 add2other = 1;
5056                         } else {
5057                                 pool = flow_dv_pool_create(dev, dcs, batch,
5058                                                            age);
5059                                 if (!pool) {
5060                                         mlx5_devx_cmd_destroy(dcs);
5061                                         return NULL;
5062                                 }
5063                         }
5064                 }
5065                 if ((dcs->id < pool->min_dcs->id ||
5066                     pool->min_dcs->id &
5067                     (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1)) &&
5068                     !(dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))) {
5069                         /*
5070                          * Update the pool min_dcs only if current dcs is
5071                          * valid and exist min_dcs is not valid or greater
5072                          * than new dcs.
5073                          */
5074                         last_min_dcs = pool->min_dcs;
5075                         rte_atomic64_set(&pool->a64_dcs,
5076                                          (int64_t)(uintptr_t)dcs);
5077                         /*
5078                          * Restore any skipped counters if the new min_dcs
5079                          * ID is smaller or min_dcs is not valid.
5080                          */
5081                         if (dcs->id < last_min_dcs->id ||
5082                             last_min_dcs->id &
5083                             (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))
5084                                 flow_dv_counter_restore(pool, last_min_dcs);
5085                 }
5086                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
5087                 cnt = MLX5_POOL_GET_CNT(pool, i);
5088                 cnt->pool = pool;
5089                 MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs;
5090                 /*
5091                  * If min_dcs is not valid, it means the new allocated dcs
5092                  * also fail to become the valid min_dcs, just skip it.
5093                  * Or if min_dcs is valid, and new dcs ID is smaller than
5094                  * min_dcs, but not become the min_dcs, also skip it.
5095                  */
5096                 if (pool->min_dcs->id &
5097                     (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1) ||
5098                     dcs->id < pool->min_dcs->id) {
5099                         MLX5_GET_POOL_CNT_EXT(pool, i)->skipped = 1;
5100                         pool->skip_cnt = 1;
5101                         goto retry;
5102                 }
5103                 if (add2other) {
5104                         TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen],
5105                                           cnt, next);
5106                         goto retry;
5107                 }
5108                 *cnt_free = cnt;
5109                 return pool;
5110         }
5111         /* bulk_bitmap is in 128 counters units. */
5112         if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
5113                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
5114         if (!dcs) {
5115                 rte_errno = ENODATA;
5116                 return NULL;
5117         }
5118         pool = flow_dv_pool_create(dev, dcs, batch, age);
5119         if (!pool) {
5120                 mlx5_devx_cmd_destroy(dcs);
5121                 return NULL;
5122         }
5123         TAILQ_INIT(&tmp_tq);
5124         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
5125                 cnt = MLX5_POOL_GET_CNT(pool, i);
5126                 cnt->pool = pool;
5127                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
5128         }
5129         rte_spinlock_lock(&cont->csl);
5130         TAILQ_CONCAT(&cont->counters, &tmp_tq, next);
5131         rte_spinlock_unlock(&cont->csl);
5132         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
5133         (*cnt_free)->pool = pool;
5134         return pool;
5135 }
5136
5137 /**
5138  * Search for existed shared counter.
5139  *
5140  * @param[in] dev
5141  *   Pointer to the Ethernet device structure.
5142  * @param[in] id
5143  *   The shared counter ID to search.
5144  * @param[out] ppool
5145  *   mlx5 flow counter pool in the container,
5146  *
5147  * @return
5148  *   NULL if not existed, otherwise pointer to the shared extend counter.
5149  */
5150 static struct mlx5_flow_counter_ext *
5151 flow_dv_counter_shared_search(struct rte_eth_dev *dev, uint32_t id,
5152                               struct mlx5_flow_counter_pool **ppool)
5153 {
5154         struct mlx5_priv *priv = dev->data->dev_private;
5155         union mlx5_l3t_data data;
5156         uint32_t cnt_idx;
5157
5158         if (mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) || !data.dword)
5159                 return NULL;
5160         cnt_idx = data.dword;
5161         /*
5162          * Shared counters don't have age info. The counter extend is after
5163          * the counter datat structure.
5164          */
5165         return (struct mlx5_flow_counter_ext *)
5166                ((flow_dv_counter_get_by_idx(dev, cnt_idx, ppool)) + 1);
5167 }
5168
5169 /**
5170  * Allocate a flow counter.
5171  *
5172  * @param[in] dev
5173  *   Pointer to the Ethernet device structure.
5174  * @param[in] shared
5175  *   Indicate if this counter is shared with other flows.
5176  * @param[in] id
5177  *   Counter identifier.
5178  * @param[in] group
5179  *   Counter flow group.
5180  * @param[in] age
5181  *   Whether the counter was allocated for aging.
5182  *
5183  * @return
5184  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
5185  */
5186 static uint32_t
5187 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
5188                       uint16_t group, uint32_t age)
5189 {
5190         struct mlx5_priv *priv = dev->data->dev_private;
5191         struct mlx5_flow_counter_pool *pool = NULL;
5192         struct mlx5_flow_counter *cnt_free = NULL;
5193         struct mlx5_flow_counter_ext *cnt_ext = NULL;
5194         /*
5195          * Currently group 0 flow counter cannot be assigned to a flow if it is
5196          * not the first one in the batch counter allocation, so it is better
5197          * to allocate counters one by one for these flows in a separate
5198          * container.
5199          * A counter can be shared between different groups so need to take
5200          * shared counters from the single container.
5201          */
5202         uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0;
5203         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
5204                                                                age);
5205         uint32_t cnt_idx;
5206
5207         if (!priv->config.devx) {
5208                 rte_errno = ENOTSUP;
5209                 return 0;
5210         }
5211         if (shared) {
5212                 cnt_ext = flow_dv_counter_shared_search(dev, id, &pool);
5213                 if (cnt_ext) {
5214                         if (cnt_ext->ref_cnt + 1 == 0) {
5215                                 rte_errno = E2BIG;
5216                                 return 0;
5217                         }
5218                         cnt_ext->ref_cnt++;
5219                         cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL +
5220                                   (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL)
5221                                   + 1;
5222                         return cnt_idx;
5223                 }
5224         }
5225         /* Get free counters from container. */
5226         rte_spinlock_lock(&cont->csl);
5227         cnt_free = TAILQ_FIRST(&cont->counters);
5228         if (cnt_free)
5229                 TAILQ_REMOVE(&cont->counters, cnt_free, next);
5230         rte_spinlock_unlock(&cont->csl);
5231         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free,
5232                                                        batch, age))
5233                 goto err;
5234         pool = cnt_free->pool;
5235         if (!batch)
5236                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free);
5237         /* Create a DV counter action only in the first time usage. */
5238         if (!cnt_free->action) {
5239                 uint16_t offset;
5240                 struct mlx5_devx_obj *dcs;
5241                 int ret;
5242
5243                 if (batch) {
5244                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
5245                         dcs = pool->min_dcs;
5246                 } else {
5247                         offset = 0;
5248                         dcs = cnt_ext->dcs;
5249                 }
5250                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
5251                                                             &cnt_free->action);
5252                 if (ret) {
5253                         rte_errno = errno;
5254                         goto err;
5255                 }
5256         }
5257         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
5258                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
5259         cnt_idx += batch * MLX5_CNT_BATCH_OFFSET;
5260         cnt_idx += age * MLX5_CNT_AGE_OFFSET;
5261         /* Update the counter reset values. */
5262         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
5263                                  &cnt_free->bytes))
5264                 goto err;
5265         if (cnt_ext) {
5266                 cnt_ext->shared = shared;
5267                 cnt_ext->ref_cnt = 1;
5268                 cnt_ext->id = id;
5269                 if (shared) {
5270                         union mlx5_l3t_data data;
5271
5272                         data.dword = cnt_idx;
5273                         if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
5274                                 return 0;
5275                 }
5276         }
5277         if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on)
5278                 /* Start the asynchronous batch query by the host thread. */
5279                 mlx5_set_query_alarm(priv->sh);
5280         return cnt_idx;
5281 err:
5282         if (cnt_free) {
5283                 cnt_free->pool = pool;
5284                 rte_spinlock_lock(&cont->csl);
5285                 TAILQ_INSERT_TAIL(&cont->counters, cnt_free, next);
5286                 rte_spinlock_unlock(&cont->csl);
5287         }
5288         return 0;
5289 }
5290
5291 /**
5292  * Get age param from counter index.
5293  *
5294  * @param[in] dev
5295  *   Pointer to the Ethernet device structure.
5296  * @param[in] counter
5297  *   Index to the counter handler.
5298  *
5299  * @return
5300  *   The aging parameter specified for the counter index.
5301  */
5302 static struct mlx5_age_param*
5303 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
5304                                 uint32_t counter)
5305 {
5306         struct mlx5_flow_counter *cnt;
5307         struct mlx5_flow_counter_pool *pool = NULL;
5308
5309         flow_dv_counter_get_by_idx(dev, counter, &pool);
5310         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
5311         cnt = MLX5_POOL_GET_CNT(pool, counter);
5312         return MLX5_CNT_TO_AGE(cnt);
5313 }
5314
5315 /**
5316  * Remove a flow counter from aged counter list.
5317  *
5318  * @param[in] dev
5319  *   Pointer to the Ethernet device structure.
5320  * @param[in] counter
5321  *   Index to the counter handler.
5322  * @param[in] cnt
5323  *   Pointer to the counter handler.
5324  */
5325 static void
5326 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
5327                                 uint32_t counter, struct mlx5_flow_counter *cnt)
5328 {
5329         struct mlx5_age_info *age_info;
5330         struct mlx5_age_param *age_param;
5331         struct mlx5_priv *priv = dev->data->dev_private;
5332
5333         age_info = GET_PORT_AGE_INFO(priv);
5334         age_param = flow_dv_counter_idx_get_age(dev, counter);
5335         if (rte_atomic16_cmpset((volatile uint16_t *)
5336                         &age_param->state,
5337                         AGE_CANDIDATE, AGE_FREE)
5338                         != AGE_CANDIDATE) {
5339                 /**
5340                  * We need the lock even it is age timeout,
5341                  * since counter may still in process.
5342                  */
5343                 rte_spinlock_lock(&age_info->aged_sl);
5344                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
5345                 rte_spinlock_unlock(&age_info->aged_sl);
5346         }
5347         rte_atomic16_set(&age_param->state, AGE_FREE);
5348 }
5349 /**
5350  * Release a flow counter.
5351  *
5352  * @param[in] dev
5353  *   Pointer to the Ethernet device structure.
5354  * @param[in] counter
5355  *   Index to the counter handler.
5356  */
5357 static void
5358 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
5359 {
5360         struct mlx5_priv *priv = dev->data->dev_private;
5361         struct mlx5_flow_counter_pool *pool = NULL;
5362         struct mlx5_flow_counter *cnt;
5363         struct mlx5_flow_counter_ext *cnt_ext = NULL;
5364
5365         if (!counter)
5366                 return;
5367         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
5368         MLX5_ASSERT(pool);
5369         if (counter < MLX5_CNT_BATCH_OFFSET) {
5370                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
5371                 if (cnt_ext) {
5372                         if (--cnt_ext->ref_cnt)
5373                                 return;
5374                         if (cnt_ext->shared)
5375                                 mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
5376                                                      cnt_ext->id);
5377                 }
5378         }
5379         if (IS_AGE_POOL(pool))
5380                 flow_dv_counter_remove_from_age(dev, counter, cnt);
5381         cnt->pool = pool;
5382         /*
5383          * Put the counter back to list to be updated in none fallback mode.
5384          * Currently, we are using two list alternately, while one is in query,
5385          * add the freed counter to the other list based on the pool query_gen
5386          * value. After query finishes, add counter the list to the global
5387          * container counter list. The list changes while query starts. In
5388          * this case, lock will not be needed as query callback and release
5389          * function both operate with the different list.
5390          *
5391          */
5392         if (!priv->counter_fallback)
5393                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
5394         else
5395                 TAILQ_INSERT_TAIL(&((MLX5_CNT_CONTAINER
5396                                   (priv->sh, 0, 0))->counters),
5397                                   cnt, next);
5398 }
5399
5400 /**
5401  * Verify the @p attributes will be correctly understood by the NIC and store
5402  * them in the @p flow if everything is correct.
5403  *
5404  * @param[in] dev
5405  *   Pointer to dev struct.
5406  * @param[in] attributes
5407  *   Pointer to flow attributes
5408  * @param[in] external
5409  *   This flow rule is created by request external to PMD.
5410  * @param[out] error
5411  *   Pointer to error structure.
5412  *
5413  * @return
5414  *   - 0 on success and non root table.
5415  *   - 1 on success and root table.
5416  *   - a negative errno value otherwise and rte_errno is set.
5417  */
5418 static int
5419 flow_dv_validate_attributes(struct rte_eth_dev *dev,
5420                             const struct rte_flow_attr *attributes,
5421                             bool external __rte_unused,
5422                             struct rte_flow_error *error)
5423 {
5424         struct mlx5_priv *priv = dev->data->dev_private;
5425         uint32_t priority_max = priv->config.flow_prio - 1;
5426         int ret = 0;
5427
5428 #ifndef HAVE_MLX5DV_DR
5429         if (attributes->group)
5430                 return rte_flow_error_set(error, ENOTSUP,
5431                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
5432                                           NULL,
5433                                           "groups are not supported");
5434 #else
5435         uint32_t table = 0;
5436
5437         ret = mlx5_flow_group_to_table(attributes, external,
5438                                        attributes->group, !!priv->fdb_def_rule,
5439                                        &table, error);
5440         if (ret)
5441                 return ret;
5442         if (!table)
5443                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
5444 #endif
5445         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
5446             attributes->priority >= priority_max)
5447                 return rte_flow_error_set(error, ENOTSUP,
5448                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
5449                                           NULL,
5450                                           "priority out of range");
5451         if (attributes->transfer) {
5452                 if (!priv->config.dv_esw_en)
5453                         return rte_flow_error_set
5454                                 (error, ENOTSUP,
5455                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5456                                  "E-Switch dr is not supported");
5457                 if (!(priv->representor || priv->master))
5458                         return rte_flow_error_set
5459                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5460                                  NULL, "E-Switch configuration can only be"
5461                                  " done by a master or a representor device");
5462                 if (attributes->egress)
5463                         return rte_flow_error_set
5464                                 (error, ENOTSUP,
5465                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
5466                                  "egress is not supported");
5467         }
5468         if (!(attributes->egress ^ attributes->ingress))
5469                 return rte_flow_error_set(error, ENOTSUP,
5470                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
5471                                           "must specify exactly one of "
5472                                           "ingress or egress");
5473         return ret;
5474 }
5475
5476 /**
5477  * Internal validation function. For validating both actions and items.
5478  *
5479  * @param[in] dev
5480  *   Pointer to the rte_eth_dev structure.
5481  * @param[in] attr
5482  *   Pointer to the flow attributes.
5483  * @param[in] items
5484  *   Pointer to the list of items.
5485  * @param[in] actions
5486  *   Pointer to the list of actions.
5487  * @param[in] external
5488  *   This flow rule is created by request external to PMD.
5489  * @param[in] hairpin
5490  *   Number of hairpin TX actions, 0 means classic flow.
5491  * @param[out] error
5492  *   Pointer to the error structure.
5493  *
5494  * @return
5495  *   0 on success, a negative errno value otherwise and rte_errno is set.
5496  */
5497 static int
5498 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
5499                  const struct rte_flow_item items[],
5500                  const struct rte_flow_action actions[],
5501                  bool external, int hairpin, struct rte_flow_error *error)
5502 {
5503         int ret;
5504         uint64_t action_flags = 0;
5505         uint64_t item_flags = 0;
5506         uint64_t last_item = 0;
5507         uint8_t next_protocol = 0xff;
5508         uint16_t ether_type = 0;
5509         int actions_n = 0;
5510         uint8_t item_ipv6_proto = 0;
5511         const struct rte_flow_item *gre_item = NULL;
5512         const struct rte_flow_action_raw_decap *decap;
5513         const struct rte_flow_action_raw_encap *encap;
5514         const struct rte_flow_action_rss *rss;
5515         const struct rte_flow_item_tcp nic_tcp_mask = {
5516                 .hdr = {
5517                         .tcp_flags = 0xFF,
5518                         .src_port = RTE_BE16(UINT16_MAX),
5519                         .dst_port = RTE_BE16(UINT16_MAX),
5520                 }
5521         };
5522         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
5523                 .hdr = {
5524                         .src_addr =
5525                         "\xff\xff\xff\xff\xff\xff\xff\xff"
5526                         "\xff\xff\xff\xff\xff\xff\xff\xff",
5527                         .dst_addr =
5528                         "\xff\xff\xff\xff\xff\xff\xff\xff"
5529                         "\xff\xff\xff\xff\xff\xff\xff\xff",
5530                         .vtc_flow = RTE_BE32(0xffffffff),
5531                         .proto = 0xff,
5532                         .hop_limits = 0xff,
5533                 },
5534                 .has_frag_ext = 1,
5535         };
5536         const struct rte_flow_item_ecpri nic_ecpri_mask = {
5537                 .hdr = {
5538                         .common = {
5539                                 .u32 =
5540                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
5541                                         .type = 0xFF,
5542                                         }).u32),
5543                         },
5544                         .dummy[0] = 0xffffffff,
5545                 },
5546         };
5547         struct mlx5_priv *priv = dev->data->dev_private;
5548         struct mlx5_dev_config *dev_conf = &priv->config;
5549         uint16_t queue_index = 0xFFFF;
5550         const struct rte_flow_item_vlan *vlan_m = NULL;
5551         int16_t rw_act_num = 0;
5552         uint64_t is_root;
5553
5554         if (items == NULL)
5555                 return -1;
5556         ret = flow_dv_validate_attributes(dev, attr, external, error);
5557         if (ret < 0)
5558                 return ret;
5559         is_root = (uint64_t)ret;
5560         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5561                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
5562                 int type = items->type;
5563
5564                 if (!mlx5_flow_os_item_supported(type))
5565                         return rte_flow_error_set(error, ENOTSUP,
5566                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5567                                                   NULL, "item not supported");
5568                 switch (type) {
5569                 case RTE_FLOW_ITEM_TYPE_VOID:
5570                         break;
5571                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
5572                         ret = flow_dv_validate_item_port_id
5573                                         (dev, items, attr, item_flags, error);
5574                         if (ret < 0)
5575                                 return ret;
5576                         last_item = MLX5_FLOW_ITEM_PORT_ID;
5577                         break;
5578                 case RTE_FLOW_ITEM_TYPE_ETH:
5579                         ret = mlx5_flow_validate_item_eth(items, item_flags,
5580                                                           error);
5581                         if (ret < 0)
5582                                 return ret;
5583                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
5584                                              MLX5_FLOW_LAYER_OUTER_L2;
5585                         if (items->mask != NULL && items->spec != NULL) {
5586                                 ether_type =
5587                                         ((const struct rte_flow_item_eth *)
5588                                          items->spec)->type;
5589                                 ether_type &=
5590                                         ((const struct rte_flow_item_eth *)
5591                                          items->mask)->type;
5592                                 ether_type = rte_be_to_cpu_16(ether_type);
5593                         } else {
5594                                 ether_type = 0;
5595                         }
5596                         break;
5597                 case RTE_FLOW_ITEM_TYPE_VLAN:
5598                         ret = flow_dv_validate_item_vlan(items, item_flags,
5599                                                          dev, error);
5600                         if (ret < 0)
5601                                 return ret;
5602                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
5603                                              MLX5_FLOW_LAYER_OUTER_VLAN;
5604                         if (items->mask != NULL && items->spec != NULL) {
5605                                 ether_type =
5606                                         ((const struct rte_flow_item_vlan *)
5607                                          items->spec)->inner_type;
5608                                 ether_type &=
5609                                         ((const struct rte_flow_item_vlan *)
5610                                          items->mask)->inner_type;
5611                                 ether_type = rte_be_to_cpu_16(ether_type);
5612                         } else {
5613                                 ether_type = 0;
5614                         }
5615                         /* Store outer VLAN mask for of_push_vlan action. */
5616                         if (!tunnel)
5617                                 vlan_m = items->mask;
5618                         break;
5619                 case RTE_FLOW_ITEM_TYPE_IPV4:
5620                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5621                                                   &item_flags, &tunnel);
5622                         ret = flow_dv_validate_item_ipv4(items, item_flags,
5623                                                          last_item, ether_type,
5624                                                          error);
5625                         if (ret < 0)
5626                                 return ret;
5627                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5628                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5629                         if (items->mask != NULL &&
5630                             ((const struct rte_flow_item_ipv4 *)
5631                              items->mask)->hdr.next_proto_id) {
5632                                 next_protocol =
5633                                         ((const struct rte_flow_item_ipv4 *)
5634                                          (items->spec))->hdr.next_proto_id;
5635                                 next_protocol &=
5636                                         ((const struct rte_flow_item_ipv4 *)
5637                                          (items->mask))->hdr.next_proto_id;
5638                         } else {
5639                                 /* Reset for inner layer. */
5640                                 next_protocol = 0xff;
5641                         }
5642                         break;
5643                 case RTE_FLOW_ITEM_TYPE_IPV6:
5644                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5645                                                   &item_flags, &tunnel);
5646                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
5647                                                            last_item,
5648                                                            ether_type,
5649                                                            &nic_ipv6_mask,
5650                                                            error);
5651                         if (ret < 0)
5652                                 return ret;
5653                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5654                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5655                         if (items->mask != NULL &&
5656                             ((const struct rte_flow_item_ipv6 *)
5657                              items->mask)->hdr.proto) {
5658                                 item_ipv6_proto =
5659                                         ((const struct rte_flow_item_ipv6 *)
5660                                          items->spec)->hdr.proto;
5661                                 next_protocol =
5662                                         ((const struct rte_flow_item_ipv6 *)
5663                                          items->spec)->hdr.proto;
5664                                 next_protocol &=
5665                                         ((const struct rte_flow_item_ipv6 *)
5666                                          items->mask)->hdr.proto;
5667                         } else {
5668                                 /* Reset for inner layer. */
5669                                 next_protocol = 0xff;
5670                         }
5671                         break;
5672                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
5673                         ret = flow_dv_validate_item_ipv6_frag_ext(items,
5674                                                                   item_flags,
5675                                                                   error);
5676                         if (ret < 0)
5677                                 return ret;
5678                         last_item = tunnel ?
5679                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
5680                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
5681                         if (items->mask != NULL &&
5682                             ((const struct rte_flow_item_ipv6_frag_ext *)
5683                              items->mask)->hdr.next_header) {
5684                                 next_protocol =
5685                                 ((const struct rte_flow_item_ipv6_frag_ext *)
5686                                  items->spec)->hdr.next_header;
5687                                 next_protocol &=
5688                                 ((const struct rte_flow_item_ipv6_frag_ext *)
5689                                  items->mask)->hdr.next_header;
5690                         } else {
5691                                 /* Reset for inner layer. */
5692                                 next_protocol = 0xff;
5693                         }
5694                         break;
5695                 case RTE_FLOW_ITEM_TYPE_TCP:
5696                         ret = mlx5_flow_validate_item_tcp
5697                                                 (items, item_flags,
5698                                                  next_protocol,
5699                                                  &nic_tcp_mask,
5700                                                  error);
5701                         if (ret < 0)
5702                                 return ret;
5703                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5704                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5705                         break;
5706                 case RTE_FLOW_ITEM_TYPE_UDP:
5707                         ret = mlx5_flow_validate_item_udp(items, item_flags,
5708                                                           next_protocol,
5709                                                           error);
5710                         if (ret < 0)
5711                                 return ret;
5712                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5713                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5714                         break;
5715                 case RTE_FLOW_ITEM_TYPE_GRE:
5716                         ret = mlx5_flow_validate_item_gre(items, item_flags,
5717                                                           next_protocol, error);
5718                         if (ret < 0)
5719                                 return ret;
5720                         gre_item = items;
5721                         last_item = MLX5_FLOW_LAYER_GRE;
5722                         break;
5723                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5724                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
5725                                                             next_protocol,
5726                                                             error);
5727                         if (ret < 0)
5728                                 return ret;
5729                         last_item = MLX5_FLOW_LAYER_NVGRE;
5730                         break;
5731                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5732                         ret = mlx5_flow_validate_item_gre_key
5733                                 (items, item_flags, gre_item, error);
5734                         if (ret < 0)
5735                                 return ret;
5736                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5737                         break;
5738                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5739                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
5740                                                             error);
5741                         if (ret < 0)
5742                                 return ret;
5743                         last_item = MLX5_FLOW_LAYER_VXLAN;
5744                         break;
5745                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5746                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
5747                                                                 item_flags, dev,
5748                                                                 error);
5749                         if (ret < 0)
5750                                 return ret;
5751                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5752                         break;
5753                 case RTE_FLOW_ITEM_TYPE_GENEVE:
5754                         ret = mlx5_flow_validate_item_geneve(items,
5755                                                              item_flags, dev,
5756                                                              error);
5757                         if (ret < 0)
5758                                 return ret;
5759                         last_item = MLX5_FLOW_LAYER_GENEVE;
5760                         break;
5761                 case RTE_FLOW_ITEM_TYPE_MPLS:
5762                         ret = mlx5_flow_validate_item_mpls(dev, items,
5763                                                            item_flags,
5764                                                            last_item, error);
5765                         if (ret < 0)
5766                                 return ret;
5767                         last_item = MLX5_FLOW_LAYER_MPLS;
5768                         break;
5769
5770                 case RTE_FLOW_ITEM_TYPE_MARK:
5771                         ret = flow_dv_validate_item_mark(dev, items, attr,
5772                                                          error);
5773                         if (ret < 0)
5774                                 return ret;
5775                         last_item = MLX5_FLOW_ITEM_MARK;
5776                         break;
5777                 case RTE_FLOW_ITEM_TYPE_META:
5778                         ret = flow_dv_validate_item_meta(dev, items, attr,
5779                                                          error);
5780                         if (ret < 0)
5781                                 return ret;
5782                         last_item = MLX5_FLOW_ITEM_METADATA;
5783                         break;
5784                 case RTE_FLOW_ITEM_TYPE_ICMP:
5785                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
5786                                                            next_protocol,
5787                                                            error);
5788                         if (ret < 0)
5789                                 return ret;
5790                         last_item = MLX5_FLOW_LAYER_ICMP;
5791                         break;
5792                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5793                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
5794                                                             next_protocol,
5795                                                             error);
5796                         if (ret < 0)
5797                                 return ret;
5798                         item_ipv6_proto = IPPROTO_ICMPV6;
5799                         last_item = MLX5_FLOW_LAYER_ICMP6;
5800                         break;
5801                 case RTE_FLOW_ITEM_TYPE_TAG:
5802                         ret = flow_dv_validate_item_tag(dev, items,
5803                                                         attr, error);
5804                         if (ret < 0)
5805                                 return ret;
5806                         last_item = MLX5_FLOW_ITEM_TAG;
5807                         break;
5808                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
5809                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
5810                         break;
5811                 case RTE_FLOW_ITEM_TYPE_GTP:
5812                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
5813                                                         error);
5814                         if (ret < 0)
5815                                 return ret;
5816                         last_item = MLX5_FLOW_LAYER_GTP;
5817                         break;
5818                 case RTE_FLOW_ITEM_TYPE_ECPRI:
5819                         /* Capacity will be checked in the translate stage. */
5820                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
5821                                                             last_item,
5822                                                             ether_type,
5823                                                             &nic_ecpri_mask,
5824                                                             error);
5825                         if (ret < 0)
5826                                 return ret;
5827                         last_item = MLX5_FLOW_LAYER_ECPRI;
5828                         break;
5829                 default:
5830                         return rte_flow_error_set(error, ENOTSUP,
5831                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5832                                                   NULL, "item not supported");
5833                 }
5834                 item_flags |= last_item;
5835         }
5836         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5837                 int type = actions->type;
5838
5839                 if (!mlx5_flow_os_action_supported(type))
5840                         return rte_flow_error_set(error, ENOTSUP,
5841                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5842                                                   actions,
5843                                                   "action not supported");
5844                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5845                         return rte_flow_error_set(error, ENOTSUP,
5846                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5847                                                   actions, "too many actions");
5848                 switch (type) {
5849                 case RTE_FLOW_ACTION_TYPE_VOID:
5850                         break;
5851                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5852                         ret = flow_dv_validate_action_port_id(dev,
5853                                                               action_flags,
5854                                                               actions,
5855                                                               attr,
5856                                                               error);
5857                         if (ret)
5858                                 return ret;
5859                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5860                         ++actions_n;
5861                         break;
5862                 case RTE_FLOW_ACTION_TYPE_FLAG:
5863                         ret = flow_dv_validate_action_flag(dev, action_flags,
5864                                                            attr, error);
5865                         if (ret < 0)
5866                                 return ret;
5867                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5868                                 /* Count all modify-header actions as one. */
5869                                 if (!(action_flags &
5870                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5871                                         ++actions_n;
5872                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
5873                                                 MLX5_FLOW_ACTION_MARK_EXT;
5874                         } else {
5875                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
5876                                 ++actions_n;
5877                         }
5878                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5879                         break;
5880                 case RTE_FLOW_ACTION_TYPE_MARK:
5881                         ret = flow_dv_validate_action_mark(dev, actions,
5882                                                            action_flags,
5883                                                            attr, error);
5884                         if (ret < 0)
5885                                 return ret;
5886                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5887                                 /* Count all modify-header actions as one. */
5888                                 if (!(action_flags &
5889                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5890                                         ++actions_n;
5891                                 action_flags |= MLX5_FLOW_ACTION_MARK |
5892                                                 MLX5_FLOW_ACTION_MARK_EXT;
5893                         } else {
5894                                 action_flags |= MLX5_FLOW_ACTION_MARK;
5895                                 ++actions_n;
5896                         }
5897                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5898                         break;
5899                 case RTE_FLOW_ACTION_TYPE_SET_META:
5900                         ret = flow_dv_validate_action_set_meta(dev, actions,
5901                                                                action_flags,
5902                                                                attr, error);
5903                         if (ret < 0)
5904                                 return ret;
5905                         /* Count all modify-header actions as one action. */
5906                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5907                                 ++actions_n;
5908                         action_flags |= MLX5_FLOW_ACTION_SET_META;
5909                         rw_act_num += MLX5_ACT_NUM_SET_META;
5910                         break;
5911                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5912                         ret = flow_dv_validate_action_set_tag(dev, actions,
5913                                                               action_flags,
5914                                                               attr, error);
5915                         if (ret < 0)
5916                                 return ret;
5917                         /* Count all modify-header actions as one action. */
5918                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5919                                 ++actions_n;
5920                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
5921                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5922                         break;
5923                 case RTE_FLOW_ACTION_TYPE_DROP:
5924                         ret = mlx5_flow_validate_action_drop(action_flags,
5925                                                              attr, error);
5926                         if (ret < 0)
5927                                 return ret;
5928                         action_flags |= MLX5_FLOW_ACTION_DROP;
5929                         ++actions_n;
5930                         break;
5931                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5932                         ret = mlx5_flow_validate_action_queue(actions,
5933                                                               action_flags, dev,
5934                                                               attr, error);
5935                         if (ret < 0)
5936                                 return ret;
5937                         queue_index = ((const struct rte_flow_action_queue *)
5938                                                         (actions->conf))->index;
5939                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
5940                         ++actions_n;
5941                         break;
5942                 case RTE_FLOW_ACTION_TYPE_RSS:
5943                         rss = actions->conf;
5944                         ret = mlx5_flow_validate_action_rss(actions,
5945                                                             action_flags, dev,
5946                                                             attr, item_flags,
5947                                                             error);
5948                         if (ret < 0)
5949                                 return ret;
5950                         if (rss != NULL && rss->queue_num)
5951                                 queue_index = rss->queue[0];
5952                         action_flags |= MLX5_FLOW_ACTION_RSS;
5953                         ++actions_n;
5954                         break;
5955                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
5956                         ret =
5957                         mlx5_flow_validate_action_default_miss(action_flags,
5958                                         attr, error);
5959                         if (ret < 0)
5960                                 return ret;
5961                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
5962                         ++actions_n;
5963                         break;
5964                 case RTE_FLOW_ACTION_TYPE_COUNT:
5965                         ret = flow_dv_validate_action_count(dev, error);
5966                         if (ret < 0)
5967                                 return ret;
5968                         action_flags |= MLX5_FLOW_ACTION_COUNT;
5969                         ++actions_n;
5970                         break;
5971                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5972                         if (flow_dv_validate_action_pop_vlan(dev,
5973                                                              action_flags,
5974                                                              actions,
5975                                                              item_flags, attr,
5976                                                              error))
5977                                 return -rte_errno;
5978                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5979                         ++actions_n;
5980                         break;
5981                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5982                         ret = flow_dv_validate_action_push_vlan(dev,
5983                                                                 action_flags,
5984                                                                 vlan_m,
5985                                                                 actions, attr,
5986                                                                 error);
5987                         if (ret < 0)
5988                                 return ret;
5989                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5990                         ++actions_n;
5991                         break;
5992                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5993                         ret = flow_dv_validate_action_set_vlan_pcp
5994                                                 (action_flags, actions, error);
5995                         if (ret < 0)
5996                                 return ret;
5997                         /* Count PCP with push_vlan command. */
5998                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
5999                         break;
6000                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
6001                         ret = flow_dv_validate_action_set_vlan_vid
6002                                                 (item_flags, action_flags,
6003                                                  actions, error);
6004                         if (ret < 0)
6005                                 return ret;
6006                         /* Count VID with push_vlan command. */
6007                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
6008                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
6009                         break;
6010                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
6011                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
6012                         ret = flow_dv_validate_action_l2_encap(dev,
6013                                                                action_flags,
6014                                                                actions, attr,
6015                                                                error);
6016                         if (ret < 0)
6017                                 return ret;
6018                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
6019                         ++actions_n;
6020                         break;
6021                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
6022                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
6023                         ret = flow_dv_validate_action_decap(dev, action_flags,
6024                                                             attr, error);
6025                         if (ret < 0)
6026                                 return ret;
6027                         action_flags |= MLX5_FLOW_ACTION_DECAP;
6028                         ++actions_n;
6029                         break;
6030                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
6031                         ret = flow_dv_validate_action_raw_encap_decap
6032                                 (dev, NULL, actions->conf, attr, &action_flags,
6033                                  &actions_n, error);
6034                         if (ret < 0)
6035                                 return ret;
6036                         break;
6037                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
6038                         decap = actions->conf;
6039                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
6040                                 ;
6041                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
6042                                 encap = NULL;
6043                                 actions--;
6044                         } else {
6045                                 encap = actions->conf;
6046                         }
6047                         ret = flow_dv_validate_action_raw_encap_decap
6048                                            (dev,
6049                                             decap ? decap : &empty_decap, encap,
6050                                             attr, &action_flags, &actions_n,
6051                                             error);
6052                         if (ret < 0)
6053                                 return ret;
6054                         break;
6055                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
6056                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
6057                         ret = flow_dv_validate_action_modify_mac(action_flags,
6058                                                                  actions,
6059                                                                  item_flags,
6060                                                                  error);
6061                         if (ret < 0)
6062                                 return ret;
6063                         /* Count all modify-header actions as one action. */
6064                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6065                                 ++actions_n;
6066                         action_flags |= actions->type ==
6067                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
6068                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
6069                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
6070                         /*
6071                          * Even if the source and destination MAC addresses have
6072                          * overlap in the header with 4B alignment, the convert
6073                          * function will handle them separately and 4 SW actions
6074                          * will be created. And 2 actions will be added each
6075                          * time no matter how many bytes of address will be set.
6076                          */
6077                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
6078                         break;
6079                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
6080                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
6081                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
6082                                                                   actions,
6083                                                                   item_flags,
6084                                                                   error);
6085                         if (ret < 0)
6086                                 return ret;
6087                         /* Count all modify-header actions as one action. */
6088                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6089                                 ++actions_n;
6090                         action_flags |= actions->type ==
6091                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
6092                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
6093                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
6094                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
6095                         break;
6096                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
6097                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
6098                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
6099                                                                   actions,
6100                                                                   item_flags,
6101                                                                   error);
6102                         if (ret < 0)
6103                                 return ret;
6104                         if (item_ipv6_proto == IPPROTO_ICMPV6)
6105                                 return rte_flow_error_set(error, ENOTSUP,
6106                                         RTE_FLOW_ERROR_TYPE_ACTION,
6107                                         actions,
6108                                         "Can't change header "
6109                                         "with ICMPv6 proto");
6110                         /* Count all modify-header actions as one action. */
6111                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6112                                 ++actions_n;
6113                         action_flags |= actions->type ==
6114                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
6115                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
6116                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
6117                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
6118                         break;
6119                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
6120                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
6121                         ret = flow_dv_validate_action_modify_tp(action_flags,
6122                                                                 actions,
6123                                                                 item_flags,
6124                                                                 error);
6125                         if (ret < 0)
6126                                 return ret;
6127                         /* Count all modify-header actions as one action. */
6128                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6129                                 ++actions_n;
6130                         action_flags |= actions->type ==
6131                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
6132                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
6133                                                 MLX5_FLOW_ACTION_SET_TP_DST;
6134                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
6135                         break;
6136                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
6137                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
6138                         ret = flow_dv_validate_action_modify_ttl(action_flags,
6139                                                                  actions,
6140                                                                  item_flags,
6141                                                                  error);
6142                         if (ret < 0)
6143                                 return ret;
6144                         /* Count all modify-header actions as one action. */
6145                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6146                                 ++actions_n;
6147                         action_flags |= actions->type ==
6148                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
6149                                                 MLX5_FLOW_ACTION_SET_TTL :
6150                                                 MLX5_FLOW_ACTION_DEC_TTL;
6151                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
6152                         break;
6153                 case RTE_FLOW_ACTION_TYPE_JUMP:
6154                         ret = flow_dv_validate_action_jump(actions,
6155                                                            action_flags,
6156                                                            attr, external,
6157                                                            error);
6158                         if (ret)
6159                                 return ret;
6160                         ++actions_n;
6161                         action_flags |= MLX5_FLOW_ACTION_JUMP;
6162                         break;
6163                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
6164                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
6165                         ret = flow_dv_validate_action_modify_tcp_seq
6166                                                                 (action_flags,
6167                                                                  actions,
6168                                                                  item_flags,
6169                                                                  error);
6170                         if (ret < 0)
6171                                 return ret;
6172                         /* Count all modify-header actions as one action. */
6173                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6174                                 ++actions_n;
6175                         action_flags |= actions->type ==
6176                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
6177                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
6178                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
6179                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
6180                         break;
6181                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
6182                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
6183                         ret = flow_dv_validate_action_modify_tcp_ack
6184                                                                 (action_flags,
6185                                                                  actions,
6186                                                                  item_flags,
6187                                                                  error);
6188                         if (ret < 0)
6189                                 return ret;
6190                         /* Count all modify-header actions as one action. */
6191                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6192                                 ++actions_n;
6193                         action_flags |= actions->type ==
6194                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
6195                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
6196                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
6197                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
6198                         break;
6199                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
6200                         break;
6201                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
6202                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
6203                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
6204                         break;
6205                 case RTE_FLOW_ACTION_TYPE_METER:
6206                         ret = mlx5_flow_validate_action_meter(dev,
6207                                                               action_flags,
6208                                                               actions, attr,
6209                                                               error);
6210                         if (ret < 0)
6211                                 return ret;
6212                         action_flags |= MLX5_FLOW_ACTION_METER;
6213                         ++actions_n;
6214                         /* Meter action will add one more TAG action. */
6215                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
6216                         break;
6217                 case RTE_FLOW_ACTION_TYPE_AGE:
6218                         ret = flow_dv_validate_action_age(action_flags,
6219                                                           actions, dev,
6220                                                           error);
6221                         if (ret < 0)
6222                                 return ret;
6223                         action_flags |= MLX5_FLOW_ACTION_AGE;
6224                         ++actions_n;
6225                         break;
6226                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
6227                         ret = flow_dv_validate_action_modify_ipv4_dscp
6228                                                          (action_flags,
6229                                                           actions,
6230                                                           item_flags,
6231                                                           error);
6232                         if (ret < 0)
6233                                 return ret;
6234                         /* Count all modify-header actions as one action. */
6235                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6236                                 ++actions_n;
6237                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
6238                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
6239                         break;
6240                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
6241                         ret = flow_dv_validate_action_modify_ipv6_dscp
6242                                                                 (action_flags,
6243                                                                  actions,
6244                                                                  item_flags,
6245                                                                  error);
6246                         if (ret < 0)
6247                                 return ret;
6248                         /* Count all modify-header actions as one action. */
6249                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6250                                 ++actions_n;
6251                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
6252                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
6253                         break;
6254                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
6255                         ret = flow_dv_validate_action_sample(action_flags,
6256                                                              actions, dev,
6257                                                              attr, error);
6258                         if (ret < 0)
6259                                 return ret;
6260                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
6261                         ++actions_n;
6262                         break;
6263                 default:
6264                         return rte_flow_error_set(error, ENOTSUP,
6265                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6266                                                   actions,
6267                                                   "action not supported");
6268                 }
6269         }
6270         /*
6271          * Validate the drop action mutual exclusion with other actions.
6272          * Drop action is mutually-exclusive with any other action, except for
6273          * Count action.
6274          */
6275         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
6276             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
6277                 return rte_flow_error_set(error, EINVAL,
6278                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
6279                                           "Drop action is mutually-exclusive "
6280                                           "with any other action, except for "
6281                                           "Count action");
6282         /* Eswitch has few restrictions on using items and actions */
6283         if (attr->transfer) {
6284                 if (!mlx5_flow_ext_mreg_supported(dev) &&
6285                     action_flags & MLX5_FLOW_ACTION_FLAG)
6286                         return rte_flow_error_set(error, ENOTSUP,
6287                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6288                                                   NULL,
6289                                                   "unsupported action FLAG");
6290                 if (!mlx5_flow_ext_mreg_supported(dev) &&
6291                     action_flags & MLX5_FLOW_ACTION_MARK)
6292                         return rte_flow_error_set(error, ENOTSUP,
6293                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6294                                                   NULL,
6295                                                   "unsupported action MARK");
6296                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
6297                         return rte_flow_error_set(error, ENOTSUP,
6298                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6299                                                   NULL,
6300                                                   "unsupported action QUEUE");
6301                 if (action_flags & MLX5_FLOW_ACTION_RSS)
6302                         return rte_flow_error_set(error, ENOTSUP,
6303                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6304                                                   NULL,
6305                                                   "unsupported action RSS");
6306                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
6307                         return rte_flow_error_set(error, EINVAL,
6308                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6309                                                   actions,
6310                                                   "no fate action is found");
6311         } else {
6312                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
6313                         return rte_flow_error_set(error, EINVAL,
6314                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6315                                                   actions,
6316                                                   "no fate action is found");
6317         }
6318         /* Continue validation for Xcap and VLAN actions.*/
6319         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
6320                              MLX5_FLOW_VLAN_ACTIONS)) &&
6321             (queue_index == 0xFFFF ||
6322              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
6323                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
6324                     MLX5_FLOW_XCAP_ACTIONS)
6325                         return rte_flow_error_set(error, ENOTSUP,
6326                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6327                                                   NULL, "encap and decap "
6328                                                   "combination aren't supported");
6329                 if (!attr->transfer && attr->ingress) {
6330                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
6331                                 return rte_flow_error_set
6332                                                 (error, ENOTSUP,
6333                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6334                                                  NULL, "encap is not supported"
6335                                                  " for ingress traffic");
6336                         else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
6337                                 return rte_flow_error_set
6338                                                 (error, ENOTSUP,
6339                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6340                                                  NULL, "push VLAN action not "
6341                                                  "supported for ingress");
6342                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
6343                                         MLX5_FLOW_VLAN_ACTIONS)
6344                                 return rte_flow_error_set
6345                                                 (error, ENOTSUP,
6346                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6347                                                  NULL, "no support for "
6348                                                  "multiple VLAN actions");
6349                 }
6350         }
6351         /* Hairpin flow will add one more TAG action. */
6352         if (hairpin > 0)
6353                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6354         /* extra metadata enabled: one more TAG action will be add. */
6355         if (dev_conf->dv_flow_en &&
6356             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
6357             mlx5_flow_ext_mreg_supported(dev))
6358                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6359         if ((uint32_t)rw_act_num >
6360                         flow_dv_modify_hdr_action_max(dev, is_root)) {
6361                 return rte_flow_error_set(error, ENOTSUP,
6362                                           RTE_FLOW_ERROR_TYPE_ACTION,
6363                                           NULL, "too many header modify"
6364                                           " actions to support");
6365         }
6366         return 0;
6367 }
6368
6369 /**
6370  * Internal preparation function. Allocates the DV flow size,
6371  * this size is constant.
6372  *
6373  * @param[in] dev
6374  *   Pointer to the rte_eth_dev structure.
6375  * @param[in] attr
6376  *   Pointer to the flow attributes.
6377  * @param[in] items
6378  *   Pointer to the list of items.
6379  * @param[in] actions
6380  *   Pointer to the list of actions.
6381  * @param[out] error
6382  *   Pointer to the error structure.
6383  *
6384  * @return
6385  *   Pointer to mlx5_flow object on success,
6386  *   otherwise NULL and rte_errno is set.
6387  */
6388 static struct mlx5_flow *
6389 flow_dv_prepare(struct rte_eth_dev *dev,
6390                 const struct rte_flow_attr *attr __rte_unused,
6391                 const struct rte_flow_item items[] __rte_unused,
6392                 const struct rte_flow_action actions[] __rte_unused,
6393                 struct rte_flow_error *error)
6394 {
6395         uint32_t handle_idx = 0;
6396         struct mlx5_flow *dev_flow;
6397         struct mlx5_flow_handle *dev_handle;
6398         struct mlx5_priv *priv = dev->data->dev_private;
6399
6400         /* In case of corrupting the memory. */
6401         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
6402                 rte_flow_error_set(error, ENOSPC,
6403                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6404                                    "not free temporary device flow");
6405                 return NULL;
6406         }
6407         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
6408                                    &handle_idx);
6409         if (!dev_handle) {
6410                 rte_flow_error_set(error, ENOMEM,
6411                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6412                                    "not enough memory to create flow handle");
6413                 return NULL;
6414         }
6415         /* No multi-thread supporting. */
6416         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
6417         dev_flow->handle = dev_handle;
6418         dev_flow->handle_idx = handle_idx;
6419         /*
6420          * In some old rdma-core releases, before continuing, a check of the
6421          * length of matching parameter will be done at first. It needs to use
6422          * the length without misc4 param. If the flow has misc4 support, then
6423          * the length needs to be adjusted accordingly. Each param member is
6424          * aligned with a 64B boundary naturally.
6425          */
6426         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) -
6427                                   MLX5_ST_SZ_BYTES(fte_match_set_misc4);
6428         /*
6429          * The matching value needs to be cleared to 0 before using. In the
6430          * past, it will be automatically cleared when using rte_*alloc
6431          * API. The time consumption will be almost the same as before.
6432          */
6433         memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
6434         dev_flow->ingress = attr->ingress;
6435         dev_flow->dv.transfer = attr->transfer;
6436         return dev_flow;
6437 }
6438
6439 #ifdef RTE_LIBRTE_MLX5_DEBUG
6440 /**
6441  * Sanity check for match mask and value. Similar to check_valid_spec() in
6442  * kernel driver. If unmasked bit is present in value, it returns failure.
6443  *
6444  * @param match_mask
6445  *   pointer to match mask buffer.
6446  * @param match_value
6447  *   pointer to match value buffer.
6448  *
6449  * @return
6450  *   0 if valid, -EINVAL otherwise.
6451  */
6452 static int
6453 flow_dv_check_valid_spec(void *match_mask, void *match_value)
6454 {
6455         uint8_t *m = match_mask;
6456         uint8_t *v = match_value;
6457         unsigned int i;
6458
6459         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
6460                 if (v[i] & ~m[i]) {
6461                         DRV_LOG(ERR,
6462                                 "match_value differs from match_criteria"
6463                                 " %p[%u] != %p[%u]",
6464                                 match_value, i, match_mask, i);
6465                         return -EINVAL;
6466                 }
6467         }
6468         return 0;
6469 }
6470 #endif
6471
6472 /**
6473  * Add match of ip_version.
6474  *
6475  * @param[in] group
6476  *   Flow group.
6477  * @param[in] headers_v
6478  *   Values header pointer.
6479  * @param[in] headers_m
6480  *   Masks header pointer.
6481  * @param[in] ip_version
6482  *   The IP version to set.
6483  */
6484 static inline void
6485 flow_dv_set_match_ip_version(uint32_t group,
6486                              void *headers_v,
6487                              void *headers_m,
6488                              uint8_t ip_version)
6489 {
6490         if (group == 0)
6491                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
6492         else
6493                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
6494                          ip_version);
6495         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
6496         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
6497         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
6498 }
6499
6500 /**
6501  * Add Ethernet item to matcher and to the value.
6502  *
6503  * @param[in, out] matcher
6504  *   Flow matcher.
6505  * @param[in, out] key
6506  *   Flow matcher value.
6507  * @param[in] item
6508  *   Flow pattern to translate.
6509  * @param[in] inner
6510  *   Item is inner pattern.
6511  */
6512 static void
6513 flow_dv_translate_item_eth(void *matcher, void *key,
6514                            const struct rte_flow_item *item, int inner,
6515                            uint32_t group)
6516 {
6517         const struct rte_flow_item_eth *eth_m = item->mask;
6518         const struct rte_flow_item_eth *eth_v = item->spec;
6519         const struct rte_flow_item_eth nic_mask = {
6520                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6521                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6522                 .type = RTE_BE16(0xffff),
6523         };
6524         void *headers_m;
6525         void *headers_v;
6526         char *l24_v;
6527         unsigned int i;
6528
6529         if (!eth_v)
6530                 return;
6531         if (!eth_m)
6532                 eth_m = &nic_mask;
6533         if (inner) {
6534                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6535                                          inner_headers);
6536                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6537         } else {
6538                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6539                                          outer_headers);
6540                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6541         }
6542         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
6543                &eth_m->dst, sizeof(eth_m->dst));
6544         /* The value must be in the range of the mask. */
6545         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
6546         for (i = 0; i < sizeof(eth_m->dst); ++i)
6547                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
6548         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
6549                &eth_m->src, sizeof(eth_m->src));
6550         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
6551         /* The value must be in the range of the mask. */
6552         for (i = 0; i < sizeof(eth_m->dst); ++i)
6553                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
6554         if (eth_v->type) {
6555                 /* When ethertype is present set mask for tagged VLAN. */
6556                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6557                 /* Set value for tagged VLAN if ethertype is 802.1Q. */
6558                 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
6559                     eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
6560                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag,
6561                                  1);
6562                         /* Return here to avoid setting match on ethertype. */
6563                         return;
6564                 }
6565         }
6566         /*
6567          * HW supports match on one Ethertype, the Ethertype following the last
6568          * VLAN tag of the packet (see PRM).
6569          * Set match on ethertype only if ETH header is not followed by VLAN.
6570          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6571          * ethertype, and use ip_version field instead.
6572          * eCPRI over Ether layer will use type value 0xAEFE.
6573          */
6574         if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6575             eth_m->type == 0xFFFF) {
6576                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6577         } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6578                    eth_m->type == 0xFFFF) {
6579                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6580         } else {
6581                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6582                          rte_be_to_cpu_16(eth_m->type));
6583                 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6584                                      ethertype);
6585                 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
6586         }
6587 }
6588
6589 /**
6590  * Add VLAN item to matcher and to the value.
6591  *
6592  * @param[in, out] dev_flow
6593  *   Flow descriptor.
6594  * @param[in, out] matcher
6595  *   Flow matcher.
6596  * @param[in, out] key
6597  *   Flow matcher value.
6598  * @param[in] item
6599  *   Flow pattern to translate.
6600  * @param[in] inner
6601  *   Item is inner pattern.
6602  */
6603 static void
6604 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
6605                             void *matcher, void *key,
6606                             const struct rte_flow_item *item,
6607                             int inner, uint32_t group)
6608 {
6609         const struct rte_flow_item_vlan *vlan_m = item->mask;
6610         const struct rte_flow_item_vlan *vlan_v = item->spec;
6611         void *headers_m;
6612         void *headers_v;
6613         uint16_t tci_m;
6614         uint16_t tci_v;
6615
6616         if (inner) {
6617                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6618                                          inner_headers);
6619                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6620         } else {
6621                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6622                                          outer_headers);
6623                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6624                 /*
6625                  * This is workaround, masks are not supported,
6626                  * and pre-validated.
6627                  */
6628                 if (vlan_v)
6629                         dev_flow->handle->vf_vlan.tag =
6630                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
6631         }
6632         /*
6633          * When VLAN item exists in flow, mark packet as tagged,
6634          * even if TCI is not specified.
6635          */
6636         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6637         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
6638         if (!vlan_v)
6639                 return;
6640         if (!vlan_m)
6641                 vlan_m = &rte_flow_item_vlan_mask;
6642         tci_m = rte_be_to_cpu_16(vlan_m->tci);
6643         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
6644         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
6645         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
6646         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
6647         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
6648         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
6649         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
6650         /*
6651          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6652          * ethertype, and use ip_version field instead.
6653          */
6654         if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6655             vlan_m->inner_type == 0xFFFF) {
6656                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6657         } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6658                    vlan_m->inner_type == 0xFFFF) {
6659                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6660         } else {
6661                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6662                          rte_be_to_cpu_16(vlan_m->inner_type));
6663                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
6664                          rte_be_to_cpu_16(vlan_m->inner_type &
6665                                           vlan_v->inner_type));
6666         }
6667 }
6668
6669 /**
6670  * Add IPV4 item to matcher and to the value.
6671  *
6672  * @param[in, out] matcher
6673  *   Flow matcher.
6674  * @param[in, out] key
6675  *   Flow matcher value.
6676  * @param[in] item
6677  *   Flow pattern to translate.
6678  * @param[in] item_flags
6679  *   Bit-fields that holds the items detected until now.
6680  * @param[in] inner
6681  *   Item is inner pattern.
6682  * @param[in] group
6683  *   The group to insert the rule.
6684  */
6685 static void
6686 flow_dv_translate_item_ipv4(void *matcher, void *key,
6687                             const struct rte_flow_item *item,
6688                             const uint64_t item_flags,
6689                             int inner, uint32_t group)
6690 {
6691         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
6692         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
6693         const struct rte_flow_item_ipv4 nic_mask = {
6694                 .hdr = {
6695                         .src_addr = RTE_BE32(0xffffffff),
6696                         .dst_addr = RTE_BE32(0xffffffff),
6697                         .type_of_service = 0xff,
6698                         .next_proto_id = 0xff,
6699                         .time_to_live = 0xff,
6700                 },
6701         };
6702         void *headers_m;
6703         void *headers_v;
6704         char *l24_m;
6705         char *l24_v;
6706         uint8_t tos;
6707
6708         if (inner) {
6709                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6710                                          inner_headers);
6711                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6712         } else {
6713                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6714                                          outer_headers);
6715                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6716         }
6717         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6718         /*
6719          * On outer header (which must contains L2), or inner header with L2,
6720          * set cvlan_tag mask bit to mark this packet as untagged.
6721          * This should be done even if item->spec is empty.
6722          */
6723         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6724                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6725         if (!ipv4_v)
6726                 return;
6727         if (!ipv4_m)
6728                 ipv4_m = &nic_mask;
6729         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6730                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6731         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6732                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6733         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
6734         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
6735         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6736                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6737         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6738                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6739         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
6740         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
6741         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
6742         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
6743                  ipv4_m->hdr.type_of_service);
6744         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
6745         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
6746                  ipv4_m->hdr.type_of_service >> 2);
6747         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
6748         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6749                  ipv4_m->hdr.next_proto_id);
6750         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6751                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
6752         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6753                  ipv4_m->hdr.time_to_live);
6754         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6755                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
6756         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
6757                  !!(ipv4_m->hdr.fragment_offset));
6758         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
6759                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
6760 }
6761
6762 /**
6763  * Add IPV6 item to matcher and to the value.
6764  *
6765  * @param[in, out] matcher
6766  *   Flow matcher.
6767  * @param[in, out] key
6768  *   Flow matcher value.
6769  * @param[in] item
6770  *   Flow pattern to translate.
6771  * @param[in] item_flags
6772  *   Bit-fields that holds the items detected until now.
6773  * @param[in] inner
6774  *   Item is inner pattern.
6775  * @param[in] group
6776  *   The group to insert the rule.
6777  */
6778 static void
6779 flow_dv_translate_item_ipv6(void *matcher, void *key,
6780                             const struct rte_flow_item *item,
6781                             const uint64_t item_flags,
6782                             int inner, uint32_t group)
6783 {
6784         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
6785         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
6786         const struct rte_flow_item_ipv6 nic_mask = {
6787                 .hdr = {
6788                         .src_addr =
6789                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6790                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6791                         .dst_addr =
6792                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6793                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6794                         .vtc_flow = RTE_BE32(0xffffffff),
6795                         .proto = 0xff,
6796                         .hop_limits = 0xff,
6797                 },
6798         };
6799         void *headers_m;
6800         void *headers_v;
6801         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6802         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6803         char *l24_m;
6804         char *l24_v;
6805         uint32_t vtc_m;
6806         uint32_t vtc_v;
6807         int i;
6808         int size;
6809
6810         if (inner) {
6811                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6812                                          inner_headers);
6813                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6814         } else {
6815                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6816                                          outer_headers);
6817                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6818         }
6819         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6820         /*
6821          * On outer header (which must contains L2), or inner header with L2,
6822          * set cvlan_tag mask bit to mark this packet as untagged.
6823          * This should be done even if item->spec is empty.
6824          */
6825         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6826                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6827         if (!ipv6_v)
6828                 return;
6829         if (!ipv6_m)
6830                 ipv6_m = &nic_mask;
6831         size = sizeof(ipv6_m->hdr.dst_addr);
6832         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6833                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6834         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6835                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6836         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
6837         for (i = 0; i < size; ++i)
6838                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
6839         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6840                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6841         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6842                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6843         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
6844         for (i = 0; i < size; ++i)
6845                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
6846         /* TOS. */
6847         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
6848         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
6849         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
6850         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
6851         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
6852         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
6853         /* Label. */
6854         if (inner) {
6855                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
6856                          vtc_m);
6857                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
6858                          vtc_v);
6859         } else {
6860                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
6861                          vtc_m);
6862                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
6863                          vtc_v);
6864         }
6865         /* Protocol. */
6866         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6867                  ipv6_m->hdr.proto);
6868         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6869                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
6870         /* Hop limit. */
6871         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6872                  ipv6_m->hdr.hop_limits);
6873         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6874                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
6875         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
6876                  !!(ipv6_m->has_frag_ext));
6877         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
6878                  !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext));
6879 }
6880
6881 /**
6882  * Add IPV6 fragment extension item to matcher and to the value.
6883  *
6884  * @param[in, out] matcher
6885  *   Flow matcher.
6886  * @param[in, out] key
6887  *   Flow matcher value.
6888  * @param[in] item
6889  *   Flow pattern to translate.
6890  * @param[in] inner
6891  *   Item is inner pattern.
6892  */
6893 static void
6894 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key,
6895                                      const struct rte_flow_item *item,
6896                                      int inner)
6897 {
6898         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask;
6899         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec;
6900         const struct rte_flow_item_ipv6_frag_ext nic_mask = {
6901                 .hdr = {
6902                         .next_header = 0xff,
6903                         .frag_data = RTE_BE16(0xffff),
6904                 },
6905         };
6906         void *headers_m;
6907         void *headers_v;
6908
6909         if (inner) {
6910                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6911                                          inner_headers);
6912                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6913         } else {
6914                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6915                                          outer_headers);
6916                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6917         }
6918         /* IPv6 fragment extension item exists, so packet is IP fragment. */
6919         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
6920         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1);
6921         if (!ipv6_frag_ext_v)
6922                 return;
6923         if (!ipv6_frag_ext_m)
6924                 ipv6_frag_ext_m = &nic_mask;
6925         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6926                  ipv6_frag_ext_m->hdr.next_header);
6927         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6928                  ipv6_frag_ext_v->hdr.next_header &
6929                  ipv6_frag_ext_m->hdr.next_header);
6930 }
6931
6932 /**
6933  * Add TCP item to matcher and to the value.
6934  *
6935  * @param[in, out] matcher
6936  *   Flow matcher.
6937  * @param[in, out] key
6938  *   Flow matcher value.
6939  * @param[in] item
6940  *   Flow pattern to translate.
6941  * @param[in] inner
6942  *   Item is inner pattern.
6943  */
6944 static void
6945 flow_dv_translate_item_tcp(void *matcher, void *key,
6946                            const struct rte_flow_item *item,
6947                            int inner)
6948 {
6949         const struct rte_flow_item_tcp *tcp_m = item->mask;
6950         const struct rte_flow_item_tcp *tcp_v = item->spec;
6951         void *headers_m;
6952         void *headers_v;
6953
6954         if (inner) {
6955                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6956                                          inner_headers);
6957                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6958         } else {
6959                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6960                                          outer_headers);
6961                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6962         }
6963         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6964         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
6965         if (!tcp_v)
6966                 return;
6967         if (!tcp_m)
6968                 tcp_m = &rte_flow_item_tcp_mask;
6969         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
6970                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
6971         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
6972                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
6973         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
6974                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
6975         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
6976                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
6977         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
6978                  tcp_m->hdr.tcp_flags);
6979         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
6980                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
6981 }
6982
6983 /**
6984  * Add UDP item to matcher and to the value.
6985  *
6986  * @param[in, out] matcher
6987  *   Flow matcher.
6988  * @param[in, out] key
6989  *   Flow matcher value.
6990  * @param[in] item
6991  *   Flow pattern to translate.
6992  * @param[in] inner
6993  *   Item is inner pattern.
6994  */
6995 static void
6996 flow_dv_translate_item_udp(void *matcher, void *key,
6997                            const struct rte_flow_item *item,
6998                            int inner)
6999 {
7000         const struct rte_flow_item_udp *udp_m = item->mask;
7001         const struct rte_flow_item_udp *udp_v = item->spec;
7002         void *headers_m;
7003         void *headers_v;
7004
7005         if (inner) {
7006                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7007                                          inner_headers);
7008                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7009         } else {
7010                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7011                                          outer_headers);
7012                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7013         }
7014         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
7015         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
7016         if (!udp_v)
7017                 return;
7018         if (!udp_m)
7019                 udp_m = &rte_flow_item_udp_mask;
7020         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
7021                  rte_be_to_cpu_16(udp_m->hdr.src_port));
7022         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
7023                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
7024         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
7025                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
7026         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
7027                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
7028 }
7029
7030 /**
7031  * Add GRE optional Key item to matcher and to the value.
7032  *
7033  * @param[in, out] matcher
7034  *   Flow matcher.
7035  * @param[in, out] key
7036  *   Flow matcher value.
7037  * @param[in] item
7038  *   Flow pattern to translate.
7039  * @param[in] inner
7040  *   Item is inner pattern.
7041  */
7042 static void
7043 flow_dv_translate_item_gre_key(void *matcher, void *key,
7044                                    const struct rte_flow_item *item)
7045 {
7046         const rte_be32_t *key_m = item->mask;
7047         const rte_be32_t *key_v = item->spec;
7048         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7049         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7050         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
7051
7052         /* GRE K bit must be on and should already be validated */
7053         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
7054         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
7055         if (!key_v)
7056                 return;
7057         if (!key_m)
7058                 key_m = &gre_key_default_mask;
7059         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
7060                  rte_be_to_cpu_32(*key_m) >> 8);
7061         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
7062                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
7063         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
7064                  rte_be_to_cpu_32(*key_m) & 0xFF);
7065         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
7066                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
7067 }
7068
7069 /**
7070  * Add GRE item to matcher and to the value.
7071  *
7072  * @param[in, out] matcher
7073  *   Flow matcher.
7074  * @param[in, out] key
7075  *   Flow matcher value.
7076  * @param[in] item
7077  *   Flow pattern to translate.
7078  * @param[in] inner
7079  *   Item is inner pattern.
7080  */
7081 static void
7082 flow_dv_translate_item_gre(void *matcher, void *key,
7083                            const struct rte_flow_item *item,
7084                            int inner)
7085 {
7086         const struct rte_flow_item_gre *gre_m = item->mask;
7087         const struct rte_flow_item_gre *gre_v = item->spec;
7088         void *headers_m;
7089         void *headers_v;
7090         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7091         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7092         struct {
7093                 union {
7094                         __extension__
7095                         struct {
7096                                 uint16_t version:3;
7097                                 uint16_t rsvd0:9;
7098                                 uint16_t s_present:1;
7099                                 uint16_t k_present:1;
7100                                 uint16_t rsvd_bit1:1;
7101                                 uint16_t c_present:1;
7102                         };
7103                         uint16_t value;
7104                 };
7105         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
7106
7107         if (inner) {
7108                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7109                                          inner_headers);
7110                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7111         } else {
7112                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7113                                          outer_headers);
7114                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7115         }
7116         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
7117         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
7118         if (!gre_v)
7119                 return;
7120         if (!gre_m)
7121                 gre_m = &rte_flow_item_gre_mask;
7122         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
7123                  rte_be_to_cpu_16(gre_m->protocol));
7124         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
7125                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
7126         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
7127         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
7128         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
7129                  gre_crks_rsvd0_ver_m.c_present);
7130         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
7131                  gre_crks_rsvd0_ver_v.c_present &
7132                  gre_crks_rsvd0_ver_m.c_present);
7133         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
7134                  gre_crks_rsvd0_ver_m.k_present);
7135         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
7136                  gre_crks_rsvd0_ver_v.k_present &
7137                  gre_crks_rsvd0_ver_m.k_present);
7138         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
7139                  gre_crks_rsvd0_ver_m.s_present);
7140         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
7141                  gre_crks_rsvd0_ver_v.s_present &
7142                  gre_crks_rsvd0_ver_m.s_present);
7143 }
7144
7145 /**
7146  * Add NVGRE item to matcher and to the value.
7147  *
7148  * @param[in, out] matcher
7149  *   Flow matcher.
7150  * @param[in, out] key
7151  *   Flow matcher value.
7152  * @param[in] item
7153  *   Flow pattern to translate.
7154  * @param[in] inner
7155  *   Item is inner pattern.
7156  */
7157 static void
7158 flow_dv_translate_item_nvgre(void *matcher, void *key,
7159                              const struct rte_flow_item *item,
7160                              int inner)
7161 {
7162         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
7163         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
7164         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7165         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7166         const char *tni_flow_id_m;
7167         const char *tni_flow_id_v;
7168         char *gre_key_m;
7169         char *gre_key_v;
7170         int size;
7171         int i;
7172
7173         /* For NVGRE, GRE header fields must be set with defined values. */
7174         const struct rte_flow_item_gre gre_spec = {
7175                 .c_rsvd0_ver = RTE_BE16(0x2000),
7176                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
7177         };
7178         const struct rte_flow_item_gre gre_mask = {
7179                 .c_rsvd0_ver = RTE_BE16(0xB000),
7180                 .protocol = RTE_BE16(UINT16_MAX),
7181         };
7182         const struct rte_flow_item gre_item = {
7183                 .spec = &gre_spec,
7184                 .mask = &gre_mask,
7185                 .last = NULL,
7186         };
7187         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
7188         if (!nvgre_v)
7189                 return;
7190         if (!nvgre_m)
7191                 nvgre_m = &rte_flow_item_nvgre_mask;
7192         tni_flow_id_m = (const char *)nvgre_m->tni;
7193         tni_flow_id_v = (const char *)nvgre_v->tni;
7194         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
7195         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
7196         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
7197         memcpy(gre_key_m, tni_flow_id_m, size);
7198         for (i = 0; i < size; ++i)
7199                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
7200 }
7201
7202 /**
7203  * Add VXLAN item to matcher and to the value.
7204  *
7205  * @param[in, out] matcher
7206  *   Flow matcher.
7207  * @param[in, out] key
7208  *   Flow matcher value.
7209  * @param[in] item
7210  *   Flow pattern to translate.
7211  * @param[in] inner
7212  *   Item is inner pattern.
7213  */
7214 static void
7215 flow_dv_translate_item_vxlan(void *matcher, void *key,
7216                              const struct rte_flow_item *item,
7217                              int inner)
7218 {
7219         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
7220         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
7221         void *headers_m;
7222         void *headers_v;
7223         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7224         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7225         char *vni_m;
7226         char *vni_v;
7227         uint16_t dport;
7228         int size;
7229         int i;
7230
7231         if (inner) {
7232                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7233                                          inner_headers);
7234                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7235         } else {
7236                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7237                                          outer_headers);
7238                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7239         }
7240         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
7241                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
7242         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7243                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7244                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7245         }
7246         if (!vxlan_v)
7247                 return;
7248         if (!vxlan_m)
7249                 vxlan_m = &rte_flow_item_vxlan_mask;
7250         size = sizeof(vxlan_m->vni);
7251         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
7252         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
7253         memcpy(vni_m, vxlan_m->vni, size);
7254         for (i = 0; i < size; ++i)
7255                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
7256 }
7257
7258 /**
7259  * Add VXLAN-GPE item to matcher and to the value.
7260  *
7261  * @param[in, out] matcher
7262  *   Flow matcher.
7263  * @param[in, out] key
7264  *   Flow matcher value.
7265  * @param[in] item
7266  *   Flow pattern to translate.
7267  * @param[in] inner
7268  *   Item is inner pattern.
7269  */
7270
7271 static void
7272 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
7273                                  const struct rte_flow_item *item, int inner)
7274 {
7275         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
7276         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
7277         void *headers_m;
7278         void *headers_v;
7279         void *misc_m =
7280                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
7281         void *misc_v =
7282                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7283         char *vni_m;
7284         char *vni_v;
7285         uint16_t dport;
7286         int size;
7287         int i;
7288         uint8_t flags_m = 0xff;
7289         uint8_t flags_v = 0xc;
7290
7291         if (inner) {
7292                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7293                                          inner_headers);
7294                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7295         } else {
7296                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7297                                          outer_headers);
7298                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7299         }
7300         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
7301                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
7302         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7303                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7304                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7305         }
7306         if (!vxlan_v)
7307                 return;
7308         if (!vxlan_m)
7309                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
7310         size = sizeof(vxlan_m->vni);
7311         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
7312         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
7313         memcpy(vni_m, vxlan_m->vni, size);
7314         for (i = 0; i < size; ++i)
7315                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
7316         if (vxlan_m->flags) {
7317                 flags_m = vxlan_m->flags;
7318                 flags_v = vxlan_v->flags;
7319         }
7320         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
7321         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
7322         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
7323                  vxlan_m->protocol);
7324         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
7325                  vxlan_v->protocol);
7326 }
7327
7328 /**
7329  * Add Geneve item to matcher and to the value.
7330  *
7331  * @param[in, out] matcher
7332  *   Flow matcher.
7333  * @param[in, out] key
7334  *   Flow matcher value.
7335  * @param[in] item
7336  *   Flow pattern to translate.
7337  * @param[in] inner
7338  *   Item is inner pattern.
7339  */
7340
7341 static void
7342 flow_dv_translate_item_geneve(void *matcher, void *key,
7343                               const struct rte_flow_item *item, int inner)
7344 {
7345         const struct rte_flow_item_geneve *geneve_m = item->mask;
7346         const struct rte_flow_item_geneve *geneve_v = item->spec;
7347         void *headers_m;
7348         void *headers_v;
7349         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7350         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7351         uint16_t dport;
7352         uint16_t gbhdr_m;
7353         uint16_t gbhdr_v;
7354         char *vni_m;
7355         char *vni_v;
7356         size_t size, i;
7357
7358         if (inner) {
7359                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7360                                          inner_headers);
7361                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7362         } else {
7363                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7364                                          outer_headers);
7365                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7366         }
7367         dport = MLX5_UDP_PORT_GENEVE;
7368         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7369                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7370                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7371         }
7372         if (!geneve_v)
7373                 return;
7374         if (!geneve_m)
7375                 geneve_m = &rte_flow_item_geneve_mask;
7376         size = sizeof(geneve_m->vni);
7377         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
7378         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
7379         memcpy(vni_m, geneve_m->vni, size);
7380         for (i = 0; i < size; ++i)
7381                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
7382         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
7383                  rte_be_to_cpu_16(geneve_m->protocol));
7384         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
7385                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
7386         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
7387         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
7388         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
7389                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7390         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
7391                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7392         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
7393                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7394         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
7395                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
7396                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7397 }
7398
7399 /**
7400  * Add MPLS item to matcher and to the value.
7401  *
7402  * @param[in, out] matcher
7403  *   Flow matcher.
7404  * @param[in, out] key
7405  *   Flow matcher value.
7406  * @param[in] item
7407  *   Flow pattern to translate.
7408  * @param[in] prev_layer
7409  *   The protocol layer indicated in previous item.
7410  * @param[in] inner
7411  *   Item is inner pattern.
7412  */
7413 static void
7414 flow_dv_translate_item_mpls(void *matcher, void *key,
7415                             const struct rte_flow_item *item,
7416                             uint64_t prev_layer,
7417                             int inner)
7418 {
7419         const uint32_t *in_mpls_m = item->mask;
7420         const uint32_t *in_mpls_v = item->spec;
7421         uint32_t *out_mpls_m = 0;
7422         uint32_t *out_mpls_v = 0;
7423         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7424         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7425         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
7426                                      misc_parameters_2);
7427         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7428         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
7429         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7430
7431         switch (prev_layer) {
7432         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7433                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
7434                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
7435                          MLX5_UDP_PORT_MPLS);
7436                 break;
7437         case MLX5_FLOW_LAYER_GRE:
7438                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
7439                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
7440                          RTE_ETHER_TYPE_MPLS);
7441                 break;
7442         default:
7443                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
7444                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
7445                          IPPROTO_MPLS);
7446                 break;
7447         }
7448         if (!in_mpls_v)
7449                 return;
7450         if (!in_mpls_m)
7451                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
7452         switch (prev_layer) {
7453         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7454                 out_mpls_m =
7455                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7456                                                  outer_first_mpls_over_udp);
7457                 out_mpls_v =
7458                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7459                                                  outer_first_mpls_over_udp);
7460                 break;
7461         case MLX5_FLOW_LAYER_GRE:
7462                 out_mpls_m =
7463                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7464                                                  outer_first_mpls_over_gre);
7465                 out_mpls_v =
7466                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7467                                                  outer_first_mpls_over_gre);
7468                 break;
7469         default:
7470                 /* Inner MPLS not over GRE is not supported. */
7471                 if (!inner) {
7472                         out_mpls_m =
7473                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7474                                                          misc2_m,
7475                                                          outer_first_mpls);
7476                         out_mpls_v =
7477                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7478                                                          misc2_v,
7479                                                          outer_first_mpls);
7480                 }
7481                 break;
7482         }
7483         if (out_mpls_m && out_mpls_v) {
7484                 *out_mpls_m = *in_mpls_m;
7485                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
7486         }
7487 }
7488
7489 /**
7490  * Add metadata register item to matcher
7491  *
7492  * @param[in, out] matcher
7493  *   Flow matcher.
7494  * @param[in, out] key
7495  *   Flow matcher value.
7496  * @param[in] reg_type
7497  *   Type of device metadata register
7498  * @param[in] value
7499  *   Register value
7500  * @param[in] mask
7501  *   Register mask
7502  */
7503 static void
7504 flow_dv_match_meta_reg(void *matcher, void *key,
7505                        enum modify_reg reg_type,
7506                        uint32_t data, uint32_t mask)
7507 {
7508         void *misc2_m =
7509                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
7510         void *misc2_v =
7511                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7512         uint32_t temp;
7513
7514         data &= mask;
7515         switch (reg_type) {
7516         case REG_A:
7517                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
7518                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
7519                 break;
7520         case REG_B:
7521                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
7522                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
7523                 break;
7524         case REG_C_0:
7525                 /*
7526                  * The metadata register C0 field might be divided into
7527                  * source vport index and META item value, we should set
7528                  * this field according to specified mask, not as whole one.
7529                  */
7530                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
7531                 temp |= mask;
7532                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
7533                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
7534                 temp &= ~mask;
7535                 temp |= data;
7536                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
7537                 break;
7538         case REG_C_1:
7539                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
7540                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
7541                 break;
7542         case REG_C_2:
7543                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
7544                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
7545                 break;
7546         case REG_C_3:
7547                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
7548                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
7549                 break;
7550         case REG_C_4:
7551                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
7552                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
7553                 break;
7554         case REG_C_5:
7555                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
7556                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
7557                 break;
7558         case REG_C_6:
7559                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
7560                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
7561                 break;
7562         case REG_C_7:
7563                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
7564                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
7565                 break;
7566         default:
7567                 MLX5_ASSERT(false);
7568                 break;
7569         }
7570 }
7571
7572 /**
7573  * Add MARK item to matcher
7574  *
7575  * @param[in] dev
7576  *   The device to configure through.
7577  * @param[in, out] matcher
7578  *   Flow matcher.
7579  * @param[in, out] key
7580  *   Flow matcher value.
7581  * @param[in] item
7582  *   Flow pattern to translate.
7583  */
7584 static void
7585 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
7586                             void *matcher, void *key,
7587                             const struct rte_flow_item *item)
7588 {
7589         struct mlx5_priv *priv = dev->data->dev_private;
7590         const struct rte_flow_item_mark *mark;
7591         uint32_t value;
7592         uint32_t mask;
7593
7594         mark = item->mask ? (const void *)item->mask :
7595                             &rte_flow_item_mark_mask;
7596         mask = mark->id & priv->sh->dv_mark_mask;
7597         mark = (const void *)item->spec;
7598         MLX5_ASSERT(mark);
7599         value = mark->id & priv->sh->dv_mark_mask & mask;
7600         if (mask) {
7601                 enum modify_reg reg;
7602
7603                 /* Get the metadata register index for the mark. */
7604                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
7605                 MLX5_ASSERT(reg > 0);
7606                 if (reg == REG_C_0) {
7607                         struct mlx5_priv *priv = dev->data->dev_private;
7608                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7609                         uint32_t shl_c0 = rte_bsf32(msk_c0);
7610
7611                         mask &= msk_c0;
7612                         mask <<= shl_c0;
7613                         value <<= shl_c0;
7614                 }
7615                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7616         }
7617 }
7618
7619 /**
7620  * Add META item to matcher
7621  *
7622  * @param[in] dev
7623  *   The devich to configure through.
7624  * @param[in, out] matcher
7625  *   Flow matcher.
7626  * @param[in, out] key
7627  *   Flow matcher value.
7628  * @param[in] attr
7629  *   Attributes of flow that includes this item.
7630  * @param[in] item
7631  *   Flow pattern to translate.
7632  */
7633 static void
7634 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
7635                             void *matcher, void *key,
7636                             const struct rte_flow_attr *attr,
7637                             const struct rte_flow_item *item)
7638 {
7639         const struct rte_flow_item_meta *meta_m;
7640         const struct rte_flow_item_meta *meta_v;
7641
7642         meta_m = (const void *)item->mask;
7643         if (!meta_m)
7644                 meta_m = &rte_flow_item_meta_mask;
7645         meta_v = (const void *)item->spec;
7646         if (meta_v) {
7647                 int reg;
7648                 uint32_t value = meta_v->data;
7649                 uint32_t mask = meta_m->data;
7650
7651                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
7652                 if (reg < 0)
7653                         return;
7654                 /*
7655                  * In datapath code there is no endianness
7656                  * coversions for perfromance reasons, all
7657                  * pattern conversions are done in rte_flow.
7658                  */
7659                 value = rte_cpu_to_be_32(value);
7660                 mask = rte_cpu_to_be_32(mask);
7661                 if (reg == REG_C_0) {
7662                         struct mlx5_priv *priv = dev->data->dev_private;
7663                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7664                         uint32_t shl_c0 = rte_bsf32(msk_c0);
7665 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
7666                         uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
7667
7668                         value >>= shr_c0;
7669                         mask >>= shr_c0;
7670 #endif
7671                         value <<= shl_c0;
7672                         mask <<= shl_c0;
7673                         MLX5_ASSERT(msk_c0);
7674                         MLX5_ASSERT(!(~msk_c0 & mask));
7675                 }
7676                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7677         }
7678 }
7679
7680 /**
7681  * Add vport metadata Reg C0 item to matcher
7682  *
7683  * @param[in, out] matcher
7684  *   Flow matcher.
7685  * @param[in, out] key
7686  *   Flow matcher value.
7687  * @param[in] reg
7688  *   Flow pattern to translate.
7689  */
7690 static void
7691 flow_dv_translate_item_meta_vport(void *matcher, void *key,
7692                                   uint32_t value, uint32_t mask)
7693 {
7694         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
7695 }
7696
7697 /**
7698  * Add tag item to matcher
7699  *
7700  * @param[in] dev
7701  *   The devich to configure through.
7702  * @param[in, out] matcher
7703  *   Flow matcher.
7704  * @param[in, out] key
7705  *   Flow matcher value.
7706  * @param[in] item
7707  *   Flow pattern to translate.
7708  */
7709 static void
7710 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
7711                                 void *matcher, void *key,
7712                                 const struct rte_flow_item *item)
7713 {
7714         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
7715         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
7716         uint32_t mask, value;
7717
7718         MLX5_ASSERT(tag_v);
7719         value = tag_v->data;
7720         mask = tag_m ? tag_m->data : UINT32_MAX;
7721         if (tag_v->id == REG_C_0) {
7722                 struct mlx5_priv *priv = dev->data->dev_private;
7723                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7724                 uint32_t shl_c0 = rte_bsf32(msk_c0);
7725
7726                 mask &= msk_c0;
7727                 mask <<= shl_c0;
7728                 value <<= shl_c0;
7729         }
7730         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
7731 }
7732
7733 /**
7734  * Add TAG item to matcher
7735  *
7736  * @param[in] dev
7737  *   The devich to configure through.
7738  * @param[in, out] matcher
7739  *   Flow matcher.
7740  * @param[in, out] key
7741  *   Flow matcher value.
7742  * @param[in] item
7743  *   Flow pattern to translate.
7744  */
7745 static void
7746 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
7747                            void *matcher, void *key,
7748                            const struct rte_flow_item *item)
7749 {
7750         const struct rte_flow_item_tag *tag_v = item->spec;
7751         const struct rte_flow_item_tag *tag_m = item->mask;
7752         enum modify_reg reg;
7753
7754         MLX5_ASSERT(tag_v);
7755         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
7756         /* Get the metadata register index for the tag. */
7757         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
7758         MLX5_ASSERT(reg > 0);
7759         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
7760 }
7761
7762 /**
7763  * Add source vport match to the specified matcher.
7764  *
7765  * @param[in, out] matcher
7766  *   Flow matcher.
7767  * @param[in, out] key
7768  *   Flow matcher value.
7769  * @param[in] port
7770  *   Source vport value to match
7771  * @param[in] mask
7772  *   Mask
7773  */
7774 static void
7775 flow_dv_translate_item_source_vport(void *matcher, void *key,
7776                                     int16_t port, uint16_t mask)
7777 {
7778         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7779         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7780
7781         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
7782         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
7783 }
7784
7785 /**
7786  * Translate port-id item to eswitch match on  port-id.
7787  *
7788  * @param[in] dev
7789  *   The devich to configure through.
7790  * @param[in, out] matcher
7791  *   Flow matcher.
7792  * @param[in, out] key
7793  *   Flow matcher value.
7794  * @param[in] item
7795  *   Flow pattern to translate.
7796  *
7797  * @return
7798  *   0 on success, a negative errno value otherwise.
7799  */
7800 static int
7801 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
7802                                void *key, const struct rte_flow_item *item)
7803 {
7804         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
7805         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
7806         struct mlx5_priv *priv;
7807         uint16_t mask, id;
7808
7809         mask = pid_m ? pid_m->id : 0xffff;
7810         id = pid_v ? pid_v->id : dev->data->port_id;
7811         priv = mlx5_port_to_eswitch_info(id, item == NULL);
7812         if (!priv)
7813                 return -rte_errno;
7814         /* Translate to vport field or to metadata, depending on mode. */
7815         if (priv->vport_meta_mask)
7816                 flow_dv_translate_item_meta_vport(matcher, key,
7817                                                   priv->vport_meta_tag,
7818                                                   priv->vport_meta_mask);
7819         else
7820                 flow_dv_translate_item_source_vport(matcher, key,
7821                                                     priv->vport_id, mask);
7822         return 0;
7823 }
7824
7825 /**
7826  * Add ICMP6 item to matcher and to the value.
7827  *
7828  * @param[in, out] matcher
7829  *   Flow matcher.
7830  * @param[in, out] key
7831  *   Flow matcher value.
7832  * @param[in] item
7833  *   Flow pattern to translate.
7834  * @param[in] inner
7835  *   Item is inner pattern.
7836  */
7837 static void
7838 flow_dv_translate_item_icmp6(void *matcher, void *key,
7839                               const struct rte_flow_item *item,
7840                               int inner)
7841 {
7842         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
7843         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
7844         void *headers_m;
7845         void *headers_v;
7846         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7847                                      misc_parameters_3);
7848         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7849         if (inner) {
7850                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7851                                          inner_headers);
7852                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7853         } else {
7854                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7855                                          outer_headers);
7856                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7857         }
7858         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7859         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
7860         if (!icmp6_v)
7861                 return;
7862         if (!icmp6_m)
7863                 icmp6_m = &rte_flow_item_icmp6_mask;
7864         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
7865         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
7866                  icmp6_v->type & icmp6_m->type);
7867         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
7868         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
7869                  icmp6_v->code & icmp6_m->code);
7870 }
7871
7872 /**
7873  * Add ICMP item to matcher and to the value.
7874  *
7875  * @param[in, out] matcher
7876  *   Flow matcher.
7877  * @param[in, out] key
7878  *   Flow matcher value.
7879  * @param[in] item
7880  *   Flow pattern to translate.
7881  * @param[in] inner
7882  *   Item is inner pattern.
7883  */
7884 static void
7885 flow_dv_translate_item_icmp(void *matcher, void *key,
7886                             const struct rte_flow_item *item,
7887                             int inner)
7888 {
7889         const struct rte_flow_item_icmp *icmp_m = item->mask;
7890         const struct rte_flow_item_icmp *icmp_v = item->spec;
7891         uint32_t icmp_header_data_m = 0;
7892         uint32_t icmp_header_data_v = 0;
7893         void *headers_m;
7894         void *headers_v;
7895         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7896                                      misc_parameters_3);
7897         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7898         if (inner) {
7899                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7900                                          inner_headers);
7901                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7902         } else {
7903                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7904                                          outer_headers);
7905                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7906         }
7907         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7908         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
7909         if (!icmp_v)
7910                 return;
7911         if (!icmp_m)
7912                 icmp_m = &rte_flow_item_icmp_mask;
7913         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
7914                  icmp_m->hdr.icmp_type);
7915         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
7916                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
7917         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
7918                  icmp_m->hdr.icmp_code);
7919         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
7920                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
7921         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
7922         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
7923         if (icmp_header_data_m) {
7924                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
7925                 icmp_header_data_v |=
7926                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
7927                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
7928                          icmp_header_data_m);
7929                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
7930                          icmp_header_data_v & icmp_header_data_m);
7931         }
7932 }
7933
7934 /**
7935  * Add GTP item to matcher and to the value.
7936  *
7937  * @param[in, out] matcher
7938  *   Flow matcher.
7939  * @param[in, out] key
7940  *   Flow matcher value.
7941  * @param[in] item
7942  *   Flow pattern to translate.
7943  * @param[in] inner
7944  *   Item is inner pattern.
7945  */
7946 static void
7947 flow_dv_translate_item_gtp(void *matcher, void *key,
7948                            const struct rte_flow_item *item, int inner)
7949 {
7950         const struct rte_flow_item_gtp *gtp_m = item->mask;
7951         const struct rte_flow_item_gtp *gtp_v = item->spec;
7952         void *headers_m;
7953         void *headers_v;
7954         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7955                                      misc_parameters_3);
7956         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7957         uint16_t dport = RTE_GTPU_UDP_PORT;
7958
7959         if (inner) {
7960                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7961                                          inner_headers);
7962                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7963         } else {
7964                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7965                                          outer_headers);
7966                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7967         }
7968         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7969                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7970                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7971         }
7972         if (!gtp_v)
7973                 return;
7974         if (!gtp_m)
7975                 gtp_m = &rte_flow_item_gtp_mask;
7976         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
7977                  gtp_m->v_pt_rsv_flags);
7978         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
7979                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
7980         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
7981         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
7982                  gtp_v->msg_type & gtp_m->msg_type);
7983         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
7984                  rte_be_to_cpu_32(gtp_m->teid));
7985         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
7986                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
7987 }
7988
7989 /**
7990  * Add eCPRI item to matcher and to the value.
7991  *
7992  * @param[in] dev
7993  *   The devich to configure through.
7994  * @param[in, out] matcher
7995  *   Flow matcher.
7996  * @param[in, out] key
7997  *   Flow matcher value.
7998  * @param[in] item
7999  *   Flow pattern to translate.
8000  * @param[in] samples
8001  *   Sample IDs to be used in the matching.
8002  */
8003 static void
8004 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
8005                              void *key, const struct rte_flow_item *item)
8006 {
8007         struct mlx5_priv *priv = dev->data->dev_private;
8008         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
8009         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
8010         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
8011                                      misc_parameters_4);
8012         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
8013         uint32_t *samples;
8014         void *dw_m;
8015         void *dw_v;
8016
8017         if (!ecpri_v)
8018                 return;
8019         if (!ecpri_m)
8020                 ecpri_m = &rte_flow_item_ecpri_mask;
8021         /*
8022          * Maximal four DW samples are supported in a single matching now.
8023          * Two are used now for a eCPRI matching:
8024          * 1. Type: one byte, mask should be 0x00ff0000 in network order
8025          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
8026          *    if any.
8027          */
8028         if (!ecpri_m->hdr.common.u32)
8029                 return;
8030         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
8031         /* Need to take the whole DW as the mask to fill the entry. */
8032         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
8033                             prog_sample_field_value_0);
8034         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
8035                             prog_sample_field_value_0);
8036         /* Already big endian (network order) in the header. */
8037         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
8038         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32;
8039         /* Sample#0, used for matching type, offset 0. */
8040         MLX5_SET(fte_match_set_misc4, misc4_m,
8041                  prog_sample_field_id_0, samples[0]);
8042         /* It makes no sense to set the sample ID in the mask field. */
8043         MLX5_SET(fte_match_set_misc4, misc4_v,
8044                  prog_sample_field_id_0, samples[0]);
8045         /*
8046          * Checking if message body part needs to be matched.
8047          * Some wildcard rules only matching type field should be supported.
8048          */
8049         if (ecpri_m->hdr.dummy[0]) {
8050                 switch (ecpri_v->hdr.common.type) {
8051                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
8052                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
8053                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
8054                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
8055                                             prog_sample_field_value_1);
8056                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
8057                                             prog_sample_field_value_1);
8058                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
8059                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0];
8060                         /* Sample#1, to match message body, offset 4. */
8061                         MLX5_SET(fte_match_set_misc4, misc4_m,
8062                                  prog_sample_field_id_1, samples[1]);
8063                         MLX5_SET(fte_match_set_misc4, misc4_v,
8064                                  prog_sample_field_id_1, samples[1]);
8065                         break;
8066                 default:
8067                         /* Others, do not match any sample ID. */
8068                         break;
8069                 }
8070         }
8071 }
8072
8073 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
8074
8075 #define HEADER_IS_ZERO(match_criteria, headers)                              \
8076         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
8077                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
8078
8079 /**
8080  * Calculate flow matcher enable bitmap.
8081  *
8082  * @param match_criteria
8083  *   Pointer to flow matcher criteria.
8084  *
8085  * @return
8086  *   Bitmap of enabled fields.
8087  */
8088 static uint8_t
8089 flow_dv_matcher_enable(uint32_t *match_criteria)
8090 {
8091         uint8_t match_criteria_enable;
8092
8093         match_criteria_enable =
8094                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
8095                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
8096         match_criteria_enable |=
8097                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
8098                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
8099         match_criteria_enable |=
8100                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
8101                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
8102         match_criteria_enable |=
8103                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
8104                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
8105         match_criteria_enable |=
8106                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
8107                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
8108         match_criteria_enable |=
8109                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
8110                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
8111         return match_criteria_enable;
8112 }
8113
8114
8115 /**
8116  * Get a flow table.
8117  *
8118  * @param[in, out] dev
8119  *   Pointer to rte_eth_dev structure.
8120  * @param[in] table_id
8121  *   Table id to use.
8122  * @param[in] egress
8123  *   Direction of the table.
8124  * @param[in] transfer
8125  *   E-Switch or NIC flow.
8126  * @param[out] error
8127  *   pointer to error structure.
8128  *
8129  * @return
8130  *   Returns tables resource based on the index, NULL in case of failed.
8131  */
8132 static struct mlx5_flow_tbl_resource *
8133 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
8134                          uint32_t table_id, uint8_t egress,
8135                          uint8_t transfer,
8136                          struct rte_flow_error *error)
8137 {
8138         struct mlx5_priv *priv = dev->data->dev_private;
8139         struct mlx5_dev_ctx_shared *sh = priv->sh;
8140         struct mlx5_flow_tbl_resource *tbl;
8141         union mlx5_flow_tbl_key table_key = {
8142                 {
8143                         .table_id = table_id,
8144                         .reserved = 0,
8145                         .domain = !!transfer,
8146                         .direction = !!egress,
8147                 }
8148         };
8149         struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
8150                                                          table_key.v64);
8151         struct mlx5_flow_tbl_data_entry *tbl_data;
8152         uint32_t idx = 0;
8153         int ret;
8154         void *domain;
8155
8156         if (pos) {
8157                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
8158                                         entry);
8159                 tbl = &tbl_data->tbl;
8160                 rte_atomic32_inc(&tbl->refcnt);
8161                 return tbl;
8162         }
8163         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
8164         if (!tbl_data) {
8165                 rte_flow_error_set(error, ENOMEM,
8166                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8167                                    NULL,
8168                                    "cannot allocate flow table data entry");
8169                 return NULL;
8170         }
8171         tbl_data->idx = idx;
8172         tbl = &tbl_data->tbl;
8173         pos = &tbl_data->entry;
8174         if (transfer)
8175                 domain = sh->fdb_domain;
8176         else if (egress)
8177                 domain = sh->tx_domain;
8178         else
8179                 domain = sh->rx_domain;
8180         ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj);
8181         if (ret) {
8182                 rte_flow_error_set(error, ENOMEM,
8183                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8184                                    NULL, "cannot create flow table object");
8185                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
8186                 return NULL;
8187         }
8188         /*
8189          * No multi-threads now, but still better to initialize the reference
8190          * count before insert it into the hash list.
8191          */
8192         rte_atomic32_init(&tbl->refcnt);
8193         /* Jump action reference count is initialized here. */
8194         rte_atomic32_init(&tbl_data->jump.refcnt);
8195         pos->key = table_key.v64;
8196         ret = mlx5_hlist_insert(sh->flow_tbls, pos);
8197         if (ret < 0) {
8198                 rte_flow_error_set(error, -ret,
8199                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8200                                    "cannot insert flow table data entry");
8201                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
8202                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
8203         }
8204         rte_atomic32_inc(&tbl->refcnt);
8205         return tbl;
8206 }
8207
8208 /**
8209  * Release a flow table.
8210  *
8211  * @param[in] dev
8212  *   Pointer to rte_eth_dev structure.
8213  * @param[in] tbl
8214  *   Table resource to be released.
8215  *
8216  * @return
8217  *   Returns 0 if table was released, else return 1;
8218  */
8219 static int
8220 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
8221                              struct mlx5_flow_tbl_resource *tbl)
8222 {
8223         struct mlx5_priv *priv = dev->data->dev_private;
8224         struct mlx5_dev_ctx_shared *sh = priv->sh;
8225         struct mlx5_flow_tbl_data_entry *tbl_data =
8226                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
8227
8228         if (!tbl)
8229                 return 0;
8230         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
8231                 struct mlx5_hlist_entry *pos = &tbl_data->entry;
8232
8233                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
8234                 tbl->obj = NULL;
8235                 /* remove the entry from the hash list and free memory. */
8236                 mlx5_hlist_remove(sh->flow_tbls, pos);
8237                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
8238                                 tbl_data->idx);
8239                 return 0;
8240         }
8241         return 1;
8242 }
8243
8244 /**
8245  * Register the flow matcher.
8246  *
8247  * @param[in, out] dev
8248  *   Pointer to rte_eth_dev structure.
8249  * @param[in, out] matcher
8250  *   Pointer to flow matcher.
8251  * @param[in, out] key
8252  *   Pointer to flow table key.
8253  * @parm[in, out] dev_flow
8254  *   Pointer to the dev_flow.
8255  * @param[out] error
8256  *   pointer to error structure.
8257  *
8258  * @return
8259  *   0 on success otherwise -errno and errno is set.
8260  */
8261 static int
8262 flow_dv_matcher_register(struct rte_eth_dev *dev,
8263                          struct mlx5_flow_dv_matcher *matcher,
8264                          union mlx5_flow_tbl_key *key,
8265                          struct mlx5_flow *dev_flow,
8266                          struct rte_flow_error *error)
8267 {
8268         struct mlx5_priv *priv = dev->data->dev_private;
8269         struct mlx5_dev_ctx_shared *sh = priv->sh;
8270         struct mlx5_flow_dv_matcher *cache_matcher;
8271         struct mlx5dv_flow_matcher_attr dv_attr = {
8272                 .type = IBV_FLOW_ATTR_NORMAL,
8273                 .match_mask = (void *)&matcher->mask,
8274         };
8275         struct mlx5_flow_tbl_resource *tbl;
8276         struct mlx5_flow_tbl_data_entry *tbl_data;
8277         int ret;
8278
8279         tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
8280                                        key->domain, error);
8281         if (!tbl)
8282                 return -rte_errno;      /* No need to refill the error info */
8283         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
8284         /* Lookup from cache. */
8285         LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
8286                 if (matcher->crc == cache_matcher->crc &&
8287                     matcher->priority == cache_matcher->priority &&
8288                     !memcmp((const void *)matcher->mask.buf,
8289                             (const void *)cache_matcher->mask.buf,
8290                             cache_matcher->mask.size)) {
8291                         DRV_LOG(DEBUG,
8292                                 "%s group %u priority %hd use %s "
8293                                 "matcher %p: refcnt %d++",
8294                                 key->domain ? "FDB" : "NIC", key->table_id,
8295                                 cache_matcher->priority,
8296                                 key->direction ? "tx" : "rx",
8297                                 (void *)cache_matcher,
8298                                 rte_atomic32_read(&cache_matcher->refcnt));
8299                         rte_atomic32_inc(&cache_matcher->refcnt);
8300                         dev_flow->handle->dvh.matcher = cache_matcher;
8301                         /* old matcher should not make the table ref++. */
8302                         flow_dv_tbl_resource_release(dev, tbl);
8303                         return 0;
8304                 }
8305         }
8306         /* Register new matcher. */
8307         cache_matcher = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache_matcher), 0,
8308                                     SOCKET_ID_ANY);
8309         if (!cache_matcher) {
8310                 flow_dv_tbl_resource_release(dev, tbl);
8311                 return rte_flow_error_set(error, ENOMEM,
8312                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8313                                           "cannot allocate matcher memory");
8314         }
8315         *cache_matcher = *matcher;
8316         dv_attr.match_criteria_enable =
8317                 flow_dv_matcher_enable(cache_matcher->mask.buf);
8318         dv_attr.priority = matcher->priority;
8319         if (key->direction)
8320                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
8321         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
8322                                                &cache_matcher->matcher_object);
8323         if (ret) {
8324                 mlx5_free(cache_matcher);
8325 #ifdef HAVE_MLX5DV_DR
8326                 flow_dv_tbl_resource_release(dev, tbl);
8327 #endif
8328                 return rte_flow_error_set(error, ENOMEM,
8329                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8330                                           NULL, "cannot create matcher");
8331         }
8332         /* Save the table information */
8333         cache_matcher->tbl = tbl;
8334         rte_atomic32_init(&cache_matcher->refcnt);
8335         /* only matcher ref++, table ref++ already done above in get API. */
8336         rte_atomic32_inc(&cache_matcher->refcnt);
8337         LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
8338         dev_flow->handle->dvh.matcher = cache_matcher;
8339         DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
8340                 key->domain ? "FDB" : "NIC", key->table_id,
8341                 cache_matcher->priority,
8342                 key->direction ? "tx" : "rx", (void *)cache_matcher,
8343                 rte_atomic32_read(&cache_matcher->refcnt));
8344         return 0;
8345 }
8346
8347 /**
8348  * Find existing tag resource or create and register a new one.
8349  *
8350  * @param dev[in, out]
8351  *   Pointer to rte_eth_dev structure.
8352  * @param[in, out] tag_be24
8353  *   Tag value in big endian then R-shift 8.
8354  * @parm[in, out] dev_flow
8355  *   Pointer to the dev_flow.
8356  * @param[out] error
8357  *   pointer to error structure.
8358  *
8359  * @return
8360  *   0 on success otherwise -errno and errno is set.
8361  */
8362 static int
8363 flow_dv_tag_resource_register
8364                         (struct rte_eth_dev *dev,
8365                          uint32_t tag_be24,
8366                          struct mlx5_flow *dev_flow,
8367                          struct rte_flow_error *error)
8368 {
8369         struct mlx5_priv *priv = dev->data->dev_private;
8370         struct mlx5_dev_ctx_shared *sh = priv->sh;
8371         struct mlx5_flow_dv_tag_resource *cache_resource;
8372         struct mlx5_hlist_entry *entry;
8373         int ret;
8374
8375         /* Lookup a matching resource from cache. */
8376         entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
8377         if (entry) {
8378                 cache_resource = container_of
8379                         (entry, struct mlx5_flow_dv_tag_resource, entry);
8380                 rte_atomic32_inc(&cache_resource->refcnt);
8381                 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
8382                 dev_flow->dv.tag_resource = cache_resource;
8383                 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
8384                         (void *)cache_resource,
8385                         rte_atomic32_read(&cache_resource->refcnt));
8386                 return 0;
8387         }
8388         /* Register new resource. */
8389         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG],
8390                                        &dev_flow->handle->dvh.rix_tag);
8391         if (!cache_resource)
8392                 return rte_flow_error_set(error, ENOMEM,
8393                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8394                                           "cannot allocate resource memory");
8395         cache_resource->entry.key = (uint64_t)tag_be24;
8396         ret = mlx5_flow_os_create_flow_action_tag(tag_be24,
8397                                                   &cache_resource->action);
8398         if (ret) {
8399                 mlx5_free(cache_resource);
8400                 return rte_flow_error_set(error, ENOMEM,
8401                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8402                                           NULL, "cannot create action");
8403         }
8404         rte_atomic32_init(&cache_resource->refcnt);
8405         rte_atomic32_inc(&cache_resource->refcnt);
8406         if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
8407                 mlx5_flow_os_destroy_flow_action(cache_resource->action);
8408                 mlx5_free(cache_resource);
8409                 return rte_flow_error_set(error, EEXIST,
8410                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8411                                           NULL, "cannot insert tag");
8412         }
8413         dev_flow->dv.tag_resource = cache_resource;
8414         DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
8415                 (void *)cache_resource,
8416                 rte_atomic32_read(&cache_resource->refcnt));
8417         return 0;
8418 }
8419
8420 /**
8421  * Release the tag.
8422  *
8423  * @param dev
8424  *   Pointer to Ethernet device.
8425  * @param tag_idx
8426  *   Tag index.
8427  *
8428  * @return
8429  *   1 while a reference on it exists, 0 when freed.
8430  */
8431 static int
8432 flow_dv_tag_release(struct rte_eth_dev *dev,
8433                     uint32_t tag_idx)
8434 {
8435         struct mlx5_priv *priv = dev->data->dev_private;
8436         struct mlx5_dev_ctx_shared *sh = priv->sh;
8437         struct mlx5_flow_dv_tag_resource *tag;
8438
8439         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8440         if (!tag)
8441                 return 0;
8442         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
8443                 dev->data->port_id, (void *)tag,
8444                 rte_atomic32_read(&tag->refcnt));
8445         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
8446                 claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
8447                 mlx5_hlist_remove(sh->tag_table, &tag->entry);
8448                 DRV_LOG(DEBUG, "port %u tag %p: removed",
8449                         dev->data->port_id, (void *)tag);
8450                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8451                 return 0;
8452         }
8453         return 1;
8454 }
8455
8456 /**
8457  * Translate port ID action to vport.
8458  *
8459  * @param[in] dev
8460  *   Pointer to rte_eth_dev structure.
8461  * @param[in] action
8462  *   Pointer to the port ID action.
8463  * @param[out] dst_port_id
8464  *   The target port ID.
8465  * @param[out] error
8466  *   Pointer to the error structure.
8467  *
8468  * @return
8469  *   0 on success, a negative errno value otherwise and rte_errno is set.
8470  */
8471 static int
8472 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
8473                                  const struct rte_flow_action *action,
8474                                  uint32_t *dst_port_id,
8475                                  struct rte_flow_error *error)
8476 {
8477         uint32_t port;
8478         struct mlx5_priv *priv;
8479         const struct rte_flow_action_port_id *conf =
8480                         (const struct rte_flow_action_port_id *)action->conf;
8481
8482         port = conf->original ? dev->data->port_id : conf->id;
8483         priv = mlx5_port_to_eswitch_info(port, false);
8484         if (!priv)
8485                 return rte_flow_error_set(error, -rte_errno,
8486                                           RTE_FLOW_ERROR_TYPE_ACTION,
8487                                           NULL,
8488                                           "No eswitch info was found for port");
8489 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
8490         /*
8491          * This parameter is transferred to
8492          * mlx5dv_dr_action_create_dest_ib_port().
8493          */
8494         *dst_port_id = priv->dev_port;
8495 #else
8496         /*
8497          * Legacy mode, no LAG configurations is supported.
8498          * This parameter is transferred to
8499          * mlx5dv_dr_action_create_dest_vport().
8500          */
8501         *dst_port_id = priv->vport_id;
8502 #endif
8503         return 0;
8504 }
8505
8506 /**
8507  * Create a counter with aging configuration.
8508  *
8509  * @param[in] dev
8510  *   Pointer to rte_eth_dev structure.
8511  * @param[out] count
8512  *   Pointer to the counter action configuration.
8513  * @param[in] age
8514  *   Pointer to the aging action configuration.
8515  *
8516  * @return
8517  *   Index to flow counter on success, 0 otherwise.
8518  */
8519 static uint32_t
8520 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
8521                                 struct mlx5_flow *dev_flow,
8522                                 const struct rte_flow_action_count *count,
8523                                 const struct rte_flow_action_age *age)
8524 {
8525         uint32_t counter;
8526         struct mlx5_age_param *age_param;
8527
8528         counter = flow_dv_counter_alloc(dev,
8529                                 count ? count->shared : 0,
8530                                 count ? count->id : 0,
8531                                 dev_flow->dv.group, !!age);
8532         if (!counter || age == NULL)
8533                 return counter;
8534         age_param  = flow_dv_counter_idx_get_age(dev, counter);
8535         /*
8536          * The counter age accuracy may have a bit delay. Have 3/4
8537          * second bias on the timeount in order to let it age in time.
8538          */
8539         age_param->context = age->context ? age->context :
8540                 (void *)(uintptr_t)(dev_flow->flow_idx);
8541         /*
8542          * The counter age accuracy may have a bit delay. Have 3/4
8543          * second bias on the timeount in order to let it age in time.
8544          */
8545         age_param->timeout = age->timeout * 10 - MLX5_AGING_TIME_DELAY;
8546         /* Set expire time in unit of 0.1 sec. */
8547         age_param->port_id = dev->data->port_id;
8548         age_param->expire = age_param->timeout +
8549                         rte_rdtsc() / (rte_get_tsc_hz() / 10);
8550         rte_atomic16_set(&age_param->state, AGE_CANDIDATE);
8551         return counter;
8552 }
8553 /**
8554  * Add Tx queue matcher
8555  *
8556  * @param[in] dev
8557  *   Pointer to the dev struct.
8558  * @param[in, out] matcher
8559  *   Flow matcher.
8560  * @param[in, out] key
8561  *   Flow matcher value.
8562  * @param[in] item
8563  *   Flow pattern to translate.
8564  * @param[in] inner
8565  *   Item is inner pattern.
8566  */
8567 static void
8568 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
8569                                 void *matcher, void *key,
8570                                 const struct rte_flow_item *item)
8571 {
8572         const struct mlx5_rte_flow_item_tx_queue *queue_m;
8573         const struct mlx5_rte_flow_item_tx_queue *queue_v;
8574         void *misc_m =
8575                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8576         void *misc_v =
8577                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8578         struct mlx5_txq_ctrl *txq;
8579         uint32_t queue;
8580
8581
8582         queue_m = (const void *)item->mask;
8583         if (!queue_m)
8584                 return;
8585         queue_v = (const void *)item->spec;
8586         if (!queue_v)
8587                 return;
8588         txq = mlx5_txq_get(dev, queue_v->queue);
8589         if (!txq)
8590                 return;
8591         queue = txq->obj->sq->id;
8592         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
8593         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
8594                  queue & queue_m->queue);
8595         mlx5_txq_release(dev, queue_v->queue);
8596 }
8597
8598 /**
8599  * Set the hash fields according to the @p flow information.
8600  *
8601  * @param[in] dev_flow
8602  *   Pointer to the mlx5_flow.
8603  * @param[in] rss_desc
8604  *   Pointer to the mlx5_flow_rss_desc.
8605  */
8606 static void
8607 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
8608                        struct mlx5_flow_rss_desc *rss_desc)
8609 {
8610         uint64_t items = dev_flow->handle->layers;
8611         int rss_inner = 0;
8612         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
8613
8614         dev_flow->hash_fields = 0;
8615 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
8616         if (rss_desc->level >= 2) {
8617                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
8618                 rss_inner = 1;
8619         }
8620 #endif
8621         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
8622             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
8623                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
8624                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
8625                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
8626                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
8627                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
8628                         else
8629                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
8630                 }
8631         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
8632                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
8633                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
8634                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
8635                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
8636                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
8637                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
8638                         else
8639                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
8640                 }
8641         }
8642         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
8643             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
8644                 if (rss_types & ETH_RSS_UDP) {
8645                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
8646                                 dev_flow->hash_fields |=
8647                                                 IBV_RX_HASH_SRC_PORT_UDP;
8648                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
8649                                 dev_flow->hash_fields |=
8650                                                 IBV_RX_HASH_DST_PORT_UDP;
8651                         else
8652                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
8653                 }
8654         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
8655                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
8656                 if (rss_types & ETH_RSS_TCP) {
8657                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
8658                                 dev_flow->hash_fields |=
8659                                                 IBV_RX_HASH_SRC_PORT_TCP;
8660                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
8661                                 dev_flow->hash_fields |=
8662                                                 IBV_RX_HASH_DST_PORT_TCP;
8663                         else
8664                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
8665                 }
8666         }
8667 }
8668
8669 /**
8670  * Create an Rx Hash queue.
8671  *
8672  * @param dev
8673  *   Pointer to Ethernet device.
8674  * @param[in] dev_flow
8675  *   Pointer to the mlx5_flow.
8676  * @param[in] rss_desc
8677  *   Pointer to the mlx5_flow_rss_desc.
8678  * @param[out] hrxq_idx
8679  *   Hash Rx queue index.
8680  *
8681  * @return
8682  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
8683  */
8684 static struct mlx5_hrxq *
8685 flow_dv_handle_rx_queue(struct rte_eth_dev *dev,
8686                         struct mlx5_flow *dev_flow,
8687                         struct mlx5_flow_rss_desc *rss_desc,
8688                         uint32_t *hrxq_idx)
8689 {
8690         struct mlx5_priv *priv = dev->data->dev_private;
8691         struct mlx5_flow_handle *dh = dev_flow->handle;
8692         struct mlx5_hrxq *hrxq;
8693
8694         MLX5_ASSERT(rss_desc->queue_num);
8695         *hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
8696                                   MLX5_RSS_HASH_KEY_LEN,
8697                                   dev_flow->hash_fields,
8698                                   rss_desc->queue,
8699                                   rss_desc->queue_num);
8700         if (!*hrxq_idx) {
8701                 *hrxq_idx = mlx5_hrxq_new
8702                                 (dev, rss_desc->key,
8703                                  MLX5_RSS_HASH_KEY_LEN,
8704                                  dev_flow->hash_fields,
8705                                  rss_desc->queue,
8706                                  rss_desc->queue_num,
8707                                  !!(dh->layers &
8708                                  MLX5_FLOW_LAYER_TUNNEL));
8709                 if (!*hrxq_idx)
8710                         return NULL;
8711         }
8712         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
8713                               *hrxq_idx);
8714         return hrxq;
8715 }
8716
8717 /**
8718  * Find existing sample resource or create and register a new one.
8719  *
8720  * @param[in, out] dev
8721  *   Pointer to rte_eth_dev structure.
8722  * @param[in] attr
8723  *   Attributes of flow that includes this item.
8724  * @param[in] resource
8725  *   Pointer to sample resource.
8726  * @parm[in, out] dev_flow
8727  *   Pointer to the dev_flow.
8728  * @param[in, out] sample_dv_actions
8729  *   Pointer to sample actions list.
8730  * @param[out] error
8731  *   pointer to error structure.
8732  *
8733  * @return
8734  *   0 on success otherwise -errno and errno is set.
8735  */
8736 static int
8737 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
8738                          const struct rte_flow_attr *attr,
8739                          struct mlx5_flow_dv_sample_resource *resource,
8740                          struct mlx5_flow *dev_flow,
8741                          void **sample_dv_actions,
8742                          struct rte_flow_error *error)
8743 {
8744         struct mlx5_flow_dv_sample_resource *cache_resource;
8745         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
8746         struct mlx5_priv *priv = dev->data->dev_private;
8747         struct mlx5_dev_ctx_shared *sh = priv->sh;
8748         struct mlx5_flow_tbl_resource *tbl;
8749         uint32_t idx = 0;
8750         const uint32_t next_ft_step = 1;
8751         uint32_t next_ft_id = resource->ft_id + next_ft_step;
8752
8753         /* Lookup a matching resource from cache. */
8754         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_SAMPLE], sh->sample_action_list,
8755                       idx, cache_resource, next) {
8756                 if (resource->ratio == cache_resource->ratio &&
8757                     resource->ft_type == cache_resource->ft_type &&
8758                     resource->ft_id == cache_resource->ft_id &&
8759                     resource->set_action == cache_resource->set_action &&
8760                     !memcmp((void *)&resource->sample_act,
8761                             (void *)&cache_resource->sample_act,
8762                             sizeof(struct mlx5_flow_sub_actions_list))) {
8763                         DRV_LOG(DEBUG, "sample resource %p: refcnt %d++",
8764                                 (void *)cache_resource,
8765                                 __atomic_load_n(&cache_resource->refcnt,
8766                                                 __ATOMIC_RELAXED));
8767                         __atomic_fetch_add(&cache_resource->refcnt, 1,
8768                                            __ATOMIC_RELAXED);
8769                         dev_flow->handle->dvh.rix_sample = idx;
8770                         dev_flow->dv.sample_res = cache_resource;
8771                         return 0;
8772                 }
8773         }
8774         /* Register new sample resource. */
8775         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE],
8776                                        &dev_flow->handle->dvh.rix_sample);
8777         if (!cache_resource)
8778                 return rte_flow_error_set(error, ENOMEM,
8779                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8780                                           NULL,
8781                                           "cannot allocate resource memory");
8782         *cache_resource = *resource;
8783         /* Create normal path table level */
8784         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
8785                                         attr->egress, attr->transfer, error);
8786         if (!tbl) {
8787                 rte_flow_error_set(error, ENOMEM,
8788                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8789                                           NULL,
8790                                           "fail to create normal path table "
8791                                           "for sample");
8792                 goto error;
8793         }
8794         cache_resource->normal_path_tbl = tbl;
8795         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
8796                 cache_resource->default_miss =
8797                                 mlx5_glue->dr_create_flow_action_default_miss();
8798                 if (!cache_resource->default_miss) {
8799                         rte_flow_error_set(error, ENOMEM,
8800                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8801                                                 NULL,
8802                                                 "cannot create default miss "
8803                                                 "action");
8804                         goto error;
8805                 }
8806                 sample_dv_actions[resource->sample_act.actions_num++] =
8807                                                 cache_resource->default_miss;
8808         }
8809         /* Create a DR sample action */
8810         sampler_attr.sample_ratio = cache_resource->ratio;
8811         sampler_attr.default_next_table = tbl->obj;
8812         sampler_attr.num_sample_actions = resource->sample_act.actions_num;
8813         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
8814                                                         &sample_dv_actions[0];
8815         sampler_attr.action = cache_resource->set_action;
8816         cache_resource->verbs_action =
8817                 mlx5_glue->dr_create_flow_action_sampler(&sampler_attr);
8818         if (!cache_resource->verbs_action) {
8819                 rte_flow_error_set(error, ENOMEM,
8820                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8821                                         NULL, "cannot create sample action");
8822                 goto error;
8823         }
8824         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8825         ILIST_INSERT(sh->ipool[MLX5_IPOOL_SAMPLE], &sh->sample_action_list,
8826                      dev_flow->handle->dvh.rix_sample, cache_resource,
8827                      next);
8828         dev_flow->dv.sample_res = cache_resource;
8829         DRV_LOG(DEBUG, "new sample resource %p: refcnt %d++",
8830                 (void *)cache_resource,
8831                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8832         return 0;
8833 error:
8834         if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
8835                 if (cache_resource->default_miss)
8836                         claim_zero(mlx5_glue->destroy_flow_action
8837                                 (cache_resource->default_miss));
8838         } else {
8839                 if (cache_resource->sample_idx.rix_hrxq &&
8840                     !mlx5_hrxq_release(dev,
8841                                 cache_resource->sample_idx.rix_hrxq))
8842                         cache_resource->sample_idx.rix_hrxq = 0;
8843                 if (cache_resource->sample_idx.rix_tag &&
8844                     !flow_dv_tag_release(dev,
8845                                 cache_resource->sample_idx.rix_tag))
8846                         cache_resource->sample_idx.rix_tag = 0;
8847                 if (cache_resource->sample_idx.cnt) {
8848                         flow_dv_counter_release(dev,
8849                                 cache_resource->sample_idx.cnt);
8850                         cache_resource->sample_idx.cnt = 0;
8851                 }
8852         }
8853         if (cache_resource->normal_path_tbl)
8854                 flow_dv_tbl_resource_release(dev,
8855                                 cache_resource->normal_path_tbl);
8856         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE],
8857                                 dev_flow->handle->dvh.rix_sample);
8858         dev_flow->handle->dvh.rix_sample = 0;
8859         return -rte_errno;
8860 }
8861
8862 /**
8863  * Find existing destination array resource or create and register a new one.
8864  *
8865  * @param[in, out] dev
8866  *   Pointer to rte_eth_dev structure.
8867  * @param[in] attr
8868  *   Attributes of flow that includes this item.
8869  * @param[in] resource
8870  *   Pointer to destination array resource.
8871  * @parm[in, out] dev_flow
8872  *   Pointer to the dev_flow.
8873  * @param[out] error
8874  *   pointer to error structure.
8875  *
8876  * @return
8877  *   0 on success otherwise -errno and errno is set.
8878  */
8879 static int
8880 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
8881                          const struct rte_flow_attr *attr,
8882                          struct mlx5_flow_dv_dest_array_resource *resource,
8883                          struct mlx5_flow *dev_flow,
8884                          struct rte_flow_error *error)
8885 {
8886         struct mlx5_flow_dv_dest_array_resource *cache_resource;
8887         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
8888         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
8889         struct mlx5_priv *priv = dev->data->dev_private;
8890         struct mlx5_dev_ctx_shared *sh = priv->sh;
8891         struct mlx5_flow_sub_actions_list *sample_act;
8892         struct mlx5dv_dr_domain *domain;
8893         uint32_t idx = 0;
8894
8895         /* Lookup a matching resource from cache. */
8896         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8897                       sh->dest_array_list,
8898                       idx, cache_resource, next) {
8899                 if (resource->num_of_dest == cache_resource->num_of_dest &&
8900                     resource->ft_type == cache_resource->ft_type &&
8901                     !memcmp((void *)cache_resource->sample_act,
8902                             (void *)resource->sample_act,
8903                            (resource->num_of_dest *
8904                            sizeof(struct mlx5_flow_sub_actions_list)))) {
8905                         DRV_LOG(DEBUG, "dest array resource %p: refcnt %d++",
8906                                 (void *)cache_resource,
8907                                 __atomic_load_n(&cache_resource->refcnt,
8908                                                 __ATOMIC_RELAXED));
8909                         __atomic_fetch_add(&cache_resource->refcnt, 1,
8910                                            __ATOMIC_RELAXED);
8911                         dev_flow->handle->dvh.rix_dest_array = idx;
8912                         dev_flow->dv.dest_array_res = cache_resource;
8913                         return 0;
8914                 }
8915         }
8916         /* Register new destination array resource. */
8917         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8918                                        &dev_flow->handle->dvh.rix_dest_array);
8919         if (!cache_resource)
8920                 return rte_flow_error_set(error, ENOMEM,
8921                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8922                                           NULL,
8923                                           "cannot allocate resource memory");
8924         *cache_resource = *resource;
8925         if (attr->transfer)
8926                 domain = sh->fdb_domain;
8927         else if (attr->ingress)
8928                 domain = sh->rx_domain;
8929         else
8930                 domain = sh->tx_domain;
8931         for (idx = 0; idx < resource->num_of_dest; idx++) {
8932                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
8933                                  mlx5_malloc(MLX5_MEM_ZERO,
8934                                  sizeof(struct mlx5dv_dr_action_dest_attr),
8935                                  0, SOCKET_ID_ANY);
8936                 if (!dest_attr[idx]) {
8937                         rte_flow_error_set(error, ENOMEM,
8938                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8939                                            NULL,
8940                                            "cannot allocate resource memory");
8941                         goto error;
8942                 }
8943                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
8944                 sample_act = &resource->sample_act[idx];
8945                 if (sample_act->action_flags == MLX5_FLOW_ACTION_QUEUE) {
8946                         dest_attr[idx]->dest = sample_act->dr_queue_action;
8947                 } else if (sample_act->action_flags ==
8948                           (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP)) {
8949                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
8950                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
8951                         dest_attr[idx]->dest_reformat->reformat =
8952                                         sample_act->dr_encap_action;
8953                         dest_attr[idx]->dest_reformat->dest =
8954                                         sample_act->dr_port_id_action;
8955                 } else if (sample_act->action_flags ==
8956                            MLX5_FLOW_ACTION_PORT_ID) {
8957                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
8958                 }
8959         }
8960         /* create a dest array actioin */
8961         cache_resource->action = mlx5_glue->dr_create_flow_action_dest_array
8962                                                 (domain,
8963                                                  cache_resource->num_of_dest,
8964                                                  dest_attr);
8965         if (!cache_resource->action) {
8966                 rte_flow_error_set(error, ENOMEM,
8967                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8968                                    NULL,
8969                                    "cannot create destination array action");
8970                 goto error;
8971         }
8972         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8973         ILIST_INSERT(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8974                      &sh->dest_array_list,
8975                      dev_flow->handle->dvh.rix_dest_array, cache_resource,
8976                      next);
8977         dev_flow->dv.dest_array_res = cache_resource;
8978         DRV_LOG(DEBUG, "new destination array resource %p: refcnt %d++",
8979                 (void *)cache_resource,
8980                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8981         for (idx = 0; idx < resource->num_of_dest; idx++)
8982                 mlx5_free(dest_attr[idx]);
8983         return 0;
8984 error:
8985         for (idx = 0; idx < resource->num_of_dest; idx++) {
8986                 struct mlx5_flow_sub_actions_idx *act_res =
8987                                         &cache_resource->sample_idx[idx];
8988                 if (act_res->rix_hrxq &&
8989                     !mlx5_hrxq_release(dev,
8990                                 act_res->rix_hrxq))
8991                         act_res->rix_hrxq = 0;
8992                 if (act_res->rix_encap_decap &&
8993                         !flow_dv_encap_decap_resource_release(dev,
8994                                 act_res->rix_encap_decap))
8995                         act_res->rix_encap_decap = 0;
8996                 if (act_res->rix_port_id_action &&
8997                         !flow_dv_port_id_action_resource_release(dev,
8998                                 act_res->rix_port_id_action))
8999                         act_res->rix_port_id_action = 0;
9000                 if (dest_attr[idx])
9001                         mlx5_free(dest_attr[idx]);
9002         }
9003
9004         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
9005                                 dev_flow->handle->dvh.rix_dest_array);
9006         dev_flow->handle->dvh.rix_dest_array = 0;
9007         return -rte_errno;
9008 }
9009
9010 /**
9011  * Convert Sample action to DV specification.
9012  *
9013  * @param[in] dev
9014  *   Pointer to rte_eth_dev structure.
9015  * @param[in] action
9016  *   Pointer to action structure.
9017  * @param[in, out] dev_flow
9018  *   Pointer to the mlx5_flow.
9019  * @param[in] attr
9020  *   Pointer to the flow attributes.
9021  * @param[in, out] num_of_dest
9022  *   Pointer to the num of destination.
9023  * @param[in, out] sample_actions
9024  *   Pointer to sample actions list.
9025  * @param[in, out] res
9026  *   Pointer to sample resource.
9027  * @param[out] error
9028  *   Pointer to the error structure.
9029  *
9030  * @return
9031  *   0 on success, a negative errno value otherwise and rte_errno is set.
9032  */
9033 static int
9034 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
9035                                 const struct rte_flow_action *action,
9036                                 struct mlx5_flow *dev_flow,
9037                                 const struct rte_flow_attr *attr,
9038                                 uint32_t *num_of_dest,
9039                                 void **sample_actions,
9040                                 struct mlx5_flow_dv_sample_resource *res,
9041                                 struct rte_flow_error *error)
9042 {
9043         struct mlx5_priv *priv = dev->data->dev_private;
9044         const struct rte_flow_action_sample *sample_action;
9045         const struct rte_flow_action *sub_actions;
9046         const struct rte_flow_action_queue *queue;
9047         struct mlx5_flow_sub_actions_list *sample_act;
9048         struct mlx5_flow_sub_actions_idx *sample_idx;
9049         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
9050                                               priv->rss_desc)
9051                                               [!!priv->flow_nested_idx];
9052         uint64_t action_flags = 0;
9053
9054         sample_act = &res->sample_act;
9055         sample_idx = &res->sample_idx;
9056         sample_action = (const struct rte_flow_action_sample *)action->conf;
9057         res->ratio = sample_action->ratio;
9058         sub_actions = sample_action->actions;
9059         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
9060                 int type = sub_actions->type;
9061                 uint32_t pre_rix = 0;
9062                 void *pre_r;
9063                 switch (type) {
9064                 case RTE_FLOW_ACTION_TYPE_QUEUE:
9065                 {
9066                         struct mlx5_hrxq *hrxq;
9067                         uint32_t hrxq_idx;
9068
9069                         queue = sub_actions->conf;
9070                         rss_desc->queue_num = 1;
9071                         rss_desc->queue[0] = queue->index;
9072                         hrxq = flow_dv_handle_rx_queue(dev, dev_flow,
9073                                         rss_desc, &hrxq_idx);
9074                         if (!hrxq)
9075                                 return rte_flow_error_set
9076                                         (error, rte_errno,
9077                                          RTE_FLOW_ERROR_TYPE_ACTION,
9078                                          NULL,
9079                                          "cannot create fate queue");
9080                         sample_act->dr_queue_action = hrxq->action;
9081                         sample_idx->rix_hrxq = hrxq_idx;
9082                         sample_actions[sample_act->actions_num++] =
9083                                                 hrxq->action;
9084                         (*num_of_dest)++;
9085                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
9086                         if (action_flags & MLX5_FLOW_ACTION_MARK)
9087                                 dev_flow->handle->rix_hrxq = hrxq_idx;
9088                         dev_flow->handle->fate_action =
9089                                         MLX5_FLOW_FATE_QUEUE;
9090                         break;
9091                 }
9092                 case RTE_FLOW_ACTION_TYPE_MARK:
9093                 {
9094                         uint32_t tag_be = mlx5_flow_mark_set
9095                                 (((const struct rte_flow_action_mark *)
9096                                 (sub_actions->conf))->id);
9097
9098                         dev_flow->handle->mark = 1;
9099                         pre_rix = dev_flow->handle->dvh.rix_tag;
9100                         /* Save the mark resource before sample */
9101                         pre_r = dev_flow->dv.tag_resource;
9102                         if (flow_dv_tag_resource_register(dev, tag_be,
9103                                                   dev_flow, error))
9104                                 return -rte_errno;
9105                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9106                         sample_act->dr_tag_action =
9107                                 dev_flow->dv.tag_resource->action;
9108                         sample_idx->rix_tag =
9109                                 dev_flow->handle->dvh.rix_tag;
9110                         sample_actions[sample_act->actions_num++] =
9111                                                 sample_act->dr_tag_action;
9112                         /* Recover the mark resource after sample */
9113                         dev_flow->dv.tag_resource = pre_r;
9114                         dev_flow->handle->dvh.rix_tag = pre_rix;
9115                         action_flags |= MLX5_FLOW_ACTION_MARK;
9116                         break;
9117                 }
9118                 case RTE_FLOW_ACTION_TYPE_COUNT:
9119                 {
9120                         uint32_t counter;
9121
9122                         counter = flow_dv_translate_create_counter(dev,
9123                                         dev_flow, sub_actions->conf, 0);
9124                         if (!counter)
9125                                 return rte_flow_error_set
9126                                                 (error, rte_errno,
9127                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9128                                                  NULL,
9129                                                  "cannot create counter"
9130                                                  " object.");
9131                         sample_idx->cnt = counter;
9132                         sample_act->dr_cnt_action =
9133                                   (flow_dv_counter_get_by_idx(dev,
9134                                   counter, NULL))->action;
9135                         sample_actions[sample_act->actions_num++] =
9136                                                 sample_act->dr_cnt_action;
9137                         action_flags |= MLX5_FLOW_ACTION_COUNT;
9138                         break;
9139                 }
9140                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
9141                 {
9142                         struct mlx5_flow_dv_port_id_action_resource
9143                                         port_id_resource;
9144                         uint32_t port_id = 0;
9145
9146                         memset(&port_id_resource, 0, sizeof(port_id_resource));
9147                         /* Save the port id resource before sample */
9148                         pre_rix = dev_flow->handle->rix_port_id_action;
9149                         pre_r = dev_flow->dv.port_id_action;
9150                         if (flow_dv_translate_action_port_id(dev, sub_actions,
9151                                                              &port_id, error))
9152                                 return -rte_errno;
9153                         port_id_resource.port_id = port_id;
9154                         if (flow_dv_port_id_action_resource_register
9155                             (dev, &port_id_resource, dev_flow, error))
9156                                 return -rte_errno;
9157                         sample_act->dr_port_id_action =
9158                                 dev_flow->dv.port_id_action->action;
9159                         sample_idx->rix_port_id_action =
9160                                 dev_flow->handle->rix_port_id_action;
9161                         sample_actions[sample_act->actions_num++] =
9162                                                 sample_act->dr_port_id_action;
9163                         /* Recover the port id resource after sample */
9164                         dev_flow->dv.port_id_action = pre_r;
9165                         dev_flow->handle->rix_port_id_action = pre_rix;
9166                         (*num_of_dest)++;
9167                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9168                         break;
9169                 }
9170                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
9171                         /* Save the encap resource before sample */
9172                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
9173                         pre_r = dev_flow->dv.encap_decap;
9174                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
9175                                                            dev_flow,
9176                                                            attr->transfer,
9177                                                            error))
9178                                 return -rte_errno;
9179                         sample_act->dr_encap_action =
9180                                 dev_flow->dv.encap_decap->action;
9181                         sample_idx->rix_encap_decap =
9182                                 dev_flow->handle->dvh.rix_encap_decap;
9183                         sample_actions[sample_act->actions_num++] =
9184                                                 sample_act->dr_encap_action;
9185                         /* Recover the encap resource after sample */
9186                         dev_flow->dv.encap_decap = pre_r;
9187                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
9188                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9189                         break;
9190                 default:
9191                         return rte_flow_error_set(error, EINVAL,
9192                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9193                                 NULL,
9194                                 "Not support for sampler action");
9195                 }
9196         }
9197         sample_act->action_flags = action_flags;
9198         res->ft_id = dev_flow->dv.group;
9199         if (attr->transfer) {
9200                 union {
9201                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
9202                         uint64_t set_action;
9203                 } action_ctx = { .set_action = 0 };
9204
9205                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
9206                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
9207                          MLX5_MODIFICATION_TYPE_SET);
9208                 MLX5_SET(set_action_in, action_ctx.action_in, field,
9209                          MLX5_MODI_META_REG_C_0);
9210                 MLX5_SET(set_action_in, action_ctx.action_in, data,
9211                          priv->vport_meta_tag);
9212                 res->set_action = action_ctx.set_action;
9213         } else if (attr->ingress) {
9214                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
9215         }
9216         return 0;
9217 }
9218
9219 /**
9220  * Convert Sample action to DV specification.
9221  *
9222  * @param[in] dev
9223  *   Pointer to rte_eth_dev structure.
9224  * @param[in, out] dev_flow
9225  *   Pointer to the mlx5_flow.
9226  * @param[in] attr
9227  *   Pointer to the flow attributes.
9228  * @param[in] num_of_dest
9229  *   The num of destination.
9230  * @param[in, out] res
9231  *   Pointer to sample resource.
9232  * @param[in, out] mdest_res
9233  *   Pointer to destination array resource.
9234  * @param[in] sample_actions
9235  *   Pointer to sample path actions list.
9236  * @param[in] action_flags
9237  *   Holds the actions detected until now.
9238  * @param[out] error
9239  *   Pointer to the error structure.
9240  *
9241  * @return
9242  *   0 on success, a negative errno value otherwise and rte_errno is set.
9243  */
9244 static int
9245 flow_dv_create_action_sample(struct rte_eth_dev *dev,
9246                              struct mlx5_flow *dev_flow,
9247                              const struct rte_flow_attr *attr,
9248                              uint32_t num_of_dest,
9249                              struct mlx5_flow_dv_sample_resource *res,
9250                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
9251                              void **sample_actions,
9252                              uint64_t action_flags,
9253                              struct rte_flow_error *error)
9254 {
9255         struct mlx5_priv *priv = dev->data->dev_private;
9256         /* update normal path action resource into last index of array */
9257         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
9258         struct mlx5_flow_sub_actions_list *sample_act =
9259                                         &mdest_res->sample_act[dest_index];
9260         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
9261                                               priv->rss_desc)
9262                                               [!!priv->flow_nested_idx];
9263         uint32_t normal_idx = 0;
9264         struct mlx5_hrxq *hrxq;
9265         uint32_t hrxq_idx;
9266
9267         if (num_of_dest > 1) {
9268                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
9269                         /* Handle QP action for mirroring */
9270                         hrxq = flow_dv_handle_rx_queue(dev, dev_flow,
9271                                                        rss_desc, &hrxq_idx);
9272                         if (!hrxq)
9273                                 return rte_flow_error_set
9274                                      (error, rte_errno,
9275                                       RTE_FLOW_ERROR_TYPE_ACTION,
9276                                       NULL,
9277                                       "cannot create rx queue");
9278                         normal_idx++;
9279                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
9280                         sample_act->dr_queue_action = hrxq->action;
9281                         if (action_flags & MLX5_FLOW_ACTION_MARK)
9282                                 dev_flow->handle->rix_hrxq = hrxq_idx;
9283                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9284                 }
9285                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
9286                         normal_idx++;
9287                         mdest_res->sample_idx[dest_index].rix_encap_decap =
9288                                 dev_flow->handle->dvh.rix_encap_decap;
9289                         sample_act->dr_encap_action =
9290                                 dev_flow->dv.encap_decap->action;
9291                 }
9292                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
9293                         normal_idx++;
9294                         mdest_res->sample_idx[dest_index].rix_port_id_action =
9295                                 dev_flow->handle->rix_port_id_action;
9296                         sample_act->dr_port_id_action =
9297                                 dev_flow->dv.port_id_action->action;
9298                 }
9299                 sample_act->actions_num = normal_idx;
9300                 /* update sample action resource into first index of array */
9301                 mdest_res->ft_type = res->ft_type;
9302                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
9303                                 sizeof(struct mlx5_flow_sub_actions_idx));
9304                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
9305                                 sizeof(struct mlx5_flow_sub_actions_list));
9306                 mdest_res->num_of_dest = num_of_dest;
9307                 if (flow_dv_dest_array_resource_register(dev, attr, mdest_res,
9308                                                          dev_flow, error))
9309                         return rte_flow_error_set(error, EINVAL,
9310                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9311                                                   NULL, "can't create sample "
9312                                                   "action");
9313         } else {
9314                 if (flow_dv_sample_resource_register(dev, attr, res, dev_flow,
9315                                                      sample_actions, error))
9316                         return rte_flow_error_set(error, EINVAL,
9317                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9318                                                   NULL,
9319                                                   "can't create sample action");
9320         }
9321         return 0;
9322 }
9323
9324 /**
9325  * Fill the flow with DV spec, lock free
9326  * (mutex should be acquired by caller).
9327  *
9328  * @param[in] dev
9329  *   Pointer to rte_eth_dev structure.
9330  * @param[in, out] dev_flow
9331  *   Pointer to the sub flow.
9332  * @param[in] attr
9333  *   Pointer to the flow attributes.
9334  * @param[in] items
9335  *   Pointer to the list of items.
9336  * @param[in] actions
9337  *   Pointer to the list of actions.
9338  * @param[out] error
9339  *   Pointer to the error structure.
9340  *
9341  * @return
9342  *   0 on success, a negative errno value otherwise and rte_errno is set.
9343  */
9344 static int
9345 __flow_dv_translate(struct rte_eth_dev *dev,
9346                     struct mlx5_flow *dev_flow,
9347                     const struct rte_flow_attr *attr,
9348                     const struct rte_flow_item items[],
9349                     const struct rte_flow_action actions[],
9350                     struct rte_flow_error *error)
9351 {
9352         struct mlx5_priv *priv = dev->data->dev_private;
9353         struct mlx5_dev_config *dev_conf = &priv->config;
9354         struct rte_flow *flow = dev_flow->flow;
9355         struct mlx5_flow_handle *handle = dev_flow->handle;
9356         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
9357                                               priv->rss_desc)
9358                                               [!!priv->flow_nested_idx];
9359         uint64_t item_flags = 0;
9360         uint64_t last_item = 0;
9361         uint64_t action_flags = 0;
9362         uint64_t priority = attr->priority;
9363         struct mlx5_flow_dv_matcher matcher = {
9364                 .mask = {
9365                         .size = sizeof(matcher.mask.buf) -
9366                                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
9367                 },
9368         };
9369         int actions_n = 0;
9370         bool actions_end = false;
9371         union {
9372                 struct mlx5_flow_dv_modify_hdr_resource res;
9373                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
9374                             sizeof(struct mlx5_modification_cmd) *
9375                             (MLX5_MAX_MODIFY_NUM + 1)];
9376         } mhdr_dummy;
9377         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
9378         const struct rte_flow_action_count *count = NULL;
9379         const struct rte_flow_action_age *age = NULL;
9380         union flow_dv_attr flow_attr = { .attr = 0 };
9381         uint32_t tag_be;
9382         union mlx5_flow_tbl_key tbl_key;
9383         uint32_t modify_action_position = UINT32_MAX;
9384         void *match_mask = matcher.mask.buf;
9385         void *match_value = dev_flow->dv.value.buf;
9386         uint8_t next_protocol = 0xff;
9387         struct rte_vlan_hdr vlan = { 0 };
9388         struct mlx5_flow_dv_dest_array_resource mdest_res;
9389         struct mlx5_flow_dv_sample_resource sample_res;
9390         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
9391         struct mlx5_flow_sub_actions_list *sample_act;
9392         uint32_t sample_act_pos = UINT32_MAX;
9393         uint32_t num_of_dest = 0;
9394         int tmp_actions_n = 0;
9395         uint32_t table;
9396         int ret = 0;
9397
9398         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
9399         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
9400         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
9401                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
9402         /* update normal path action resource into last index of array */
9403         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
9404         ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
9405                                        !!priv->fdb_def_rule, &table, error);
9406         if (ret)
9407                 return ret;
9408         dev_flow->dv.group = table;
9409         if (attr->transfer)
9410                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
9411         if (priority == MLX5_FLOW_PRIO_RSVD)
9412                 priority = dev_conf->flow_prio - 1;
9413         /* number of actions must be set to 0 in case of dirty stack. */
9414         mhdr_res->actions_num = 0;
9415         for (; !actions_end ; actions++) {
9416                 const struct rte_flow_action_queue *queue;
9417                 const struct rte_flow_action_rss *rss;
9418                 const struct rte_flow_action *action = actions;
9419                 const uint8_t *rss_key;
9420                 const struct rte_flow_action_meter *mtr;
9421                 struct mlx5_flow_tbl_resource *tbl;
9422                 uint32_t port_id = 0;
9423                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
9424                 int action_type = actions->type;
9425                 const struct rte_flow_action *found_action = NULL;
9426                 struct mlx5_flow_meter *fm = NULL;
9427                 uint32_t jump_group = 0;
9428
9429                 if (!mlx5_flow_os_action_supported(action_type))
9430                         return rte_flow_error_set(error, ENOTSUP,
9431                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9432                                                   actions,
9433                                                   "action not supported");
9434                 switch (action_type) {
9435                 case RTE_FLOW_ACTION_TYPE_VOID:
9436                         break;
9437                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
9438                         if (flow_dv_translate_action_port_id(dev, action,
9439                                                              &port_id, error))
9440                                 return -rte_errno;
9441                         port_id_resource.port_id = port_id;
9442                         MLX5_ASSERT(!handle->rix_port_id_action);
9443                         if (flow_dv_port_id_action_resource_register
9444                             (dev, &port_id_resource, dev_flow, error))
9445                                 return -rte_errno;
9446                         dev_flow->dv.actions[actions_n++] =
9447                                         dev_flow->dv.port_id_action->action;
9448                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9449                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
9450                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9451                         num_of_dest++;
9452                         break;
9453                 case RTE_FLOW_ACTION_TYPE_FLAG:
9454                         action_flags |= MLX5_FLOW_ACTION_FLAG;
9455                         dev_flow->handle->mark = 1;
9456                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
9457                                 struct rte_flow_action_mark mark = {
9458                                         .id = MLX5_FLOW_MARK_DEFAULT,
9459                                 };
9460
9461                                 if (flow_dv_convert_action_mark(dev, &mark,
9462                                                                 mhdr_res,
9463                                                                 error))
9464                                         return -rte_errno;
9465                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
9466                                 break;
9467                         }
9468                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
9469                         /*
9470                          * Only one FLAG or MARK is supported per device flow
9471                          * right now. So the pointer to the tag resource must be
9472                          * zero before the register process.
9473                          */
9474                         MLX5_ASSERT(!handle->dvh.rix_tag);
9475                         if (flow_dv_tag_resource_register(dev, tag_be,
9476                                                           dev_flow, error))
9477                                 return -rte_errno;
9478                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9479                         dev_flow->dv.actions[actions_n++] =
9480                                         dev_flow->dv.tag_resource->action;
9481                         break;
9482                 case RTE_FLOW_ACTION_TYPE_MARK:
9483                         action_flags |= MLX5_FLOW_ACTION_MARK;
9484                         dev_flow->handle->mark = 1;
9485                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
9486                                 const struct rte_flow_action_mark *mark =
9487                                         (const struct rte_flow_action_mark *)
9488                                                 actions->conf;
9489
9490                                 if (flow_dv_convert_action_mark(dev, mark,
9491                                                                 mhdr_res,
9492                                                                 error))
9493                                         return -rte_errno;
9494                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
9495                                 break;
9496                         }
9497                         /* Fall-through */
9498                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
9499                         /* Legacy (non-extensive) MARK action. */
9500                         tag_be = mlx5_flow_mark_set
9501                               (((const struct rte_flow_action_mark *)
9502                                (actions->conf))->id);
9503                         MLX5_ASSERT(!handle->dvh.rix_tag);
9504                         if (flow_dv_tag_resource_register(dev, tag_be,
9505                                                           dev_flow, error))
9506                                 return -rte_errno;
9507                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9508                         dev_flow->dv.actions[actions_n++] =
9509                                         dev_flow->dv.tag_resource->action;
9510                         break;
9511                 case RTE_FLOW_ACTION_TYPE_SET_META:
9512                         if (flow_dv_convert_action_set_meta
9513                                 (dev, mhdr_res, attr,
9514                                  (const struct rte_flow_action_set_meta *)
9515                                   actions->conf, error))
9516                                 return -rte_errno;
9517                         action_flags |= MLX5_FLOW_ACTION_SET_META;
9518                         break;
9519                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
9520                         if (flow_dv_convert_action_set_tag
9521                                 (dev, mhdr_res,
9522                                  (const struct rte_flow_action_set_tag *)
9523                                   actions->conf, error))
9524                                 return -rte_errno;
9525                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9526                         break;
9527                 case RTE_FLOW_ACTION_TYPE_DROP:
9528                         action_flags |= MLX5_FLOW_ACTION_DROP;
9529                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
9530                         break;
9531                 case RTE_FLOW_ACTION_TYPE_QUEUE:
9532                         queue = actions->conf;
9533                         rss_desc->queue_num = 1;
9534                         rss_desc->queue[0] = queue->index;
9535                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
9536                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9537                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
9538                         num_of_dest++;
9539                         break;
9540                 case RTE_FLOW_ACTION_TYPE_RSS:
9541                         rss = actions->conf;
9542                         memcpy(rss_desc->queue, rss->queue,
9543                                rss->queue_num * sizeof(uint16_t));
9544                         rss_desc->queue_num = rss->queue_num;
9545                         /* NULL RSS key indicates default RSS key. */
9546                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
9547                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
9548                         /*
9549                          * rss->level and rss.types should be set in advance
9550                          * when expanding items for RSS.
9551                          */
9552                         action_flags |= MLX5_FLOW_ACTION_RSS;
9553                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9554                         break;
9555                 case RTE_FLOW_ACTION_TYPE_AGE:
9556                 case RTE_FLOW_ACTION_TYPE_COUNT:
9557                         if (!dev_conf->devx) {
9558                                 return rte_flow_error_set
9559                                               (error, ENOTSUP,
9560                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9561                                                NULL,
9562                                                "count action not supported");
9563                         }
9564                         /* Save information first, will apply later. */
9565                         if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT)
9566                                 count = action->conf;
9567                         else
9568                                 age = action->conf;
9569                         action_flags |= MLX5_FLOW_ACTION_COUNT;
9570                         break;
9571                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
9572                         dev_flow->dv.actions[actions_n++] =
9573                                                 priv->sh->pop_vlan_action;
9574                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
9575                         break;
9576                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
9577                         if (!(action_flags &
9578                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
9579                                 flow_dev_get_vlan_info_from_items(items, &vlan);
9580                         vlan.eth_proto = rte_be_to_cpu_16
9581                              ((((const struct rte_flow_action_of_push_vlan *)
9582                                                    actions->conf)->ethertype));
9583                         found_action = mlx5_flow_find_action
9584                                         (actions + 1,
9585                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
9586                         if (found_action)
9587                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
9588                         found_action = mlx5_flow_find_action
9589                                         (actions + 1,
9590                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
9591                         if (found_action)
9592                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
9593                         if (flow_dv_create_action_push_vlan
9594                                             (dev, attr, &vlan, dev_flow, error))
9595                                 return -rte_errno;
9596                         dev_flow->dv.actions[actions_n++] =
9597                                         dev_flow->dv.push_vlan_res->action;
9598                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
9599                         break;
9600                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
9601                         /* of_vlan_push action handled this action */
9602                         MLX5_ASSERT(action_flags &
9603                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
9604                         break;
9605                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
9606                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
9607                                 break;
9608                         flow_dev_get_vlan_info_from_items(items, &vlan);
9609                         mlx5_update_vlan_vid_pcp(actions, &vlan);
9610                         /* If no VLAN push - this is a modify header action */
9611                         if (flow_dv_convert_action_modify_vlan_vid
9612                                                 (mhdr_res, actions, error))
9613                                 return -rte_errno;
9614                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
9615                         break;
9616                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
9617                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
9618                         if (flow_dv_create_action_l2_encap(dev, actions,
9619                                                            dev_flow,
9620                                                            attr->transfer,
9621                                                            error))
9622                                 return -rte_errno;
9623                         dev_flow->dv.actions[actions_n++] =
9624                                         dev_flow->dv.encap_decap->action;
9625                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9626                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
9627                                 sample_act->action_flags |=
9628                                                         MLX5_FLOW_ACTION_ENCAP;
9629                         break;
9630                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
9631                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
9632                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
9633                                                            attr->transfer,
9634                                                            error))
9635                                 return -rte_errno;
9636                         dev_flow->dv.actions[actions_n++] =
9637                                         dev_flow->dv.encap_decap->action;
9638                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9639                         break;
9640                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
9641                         /* Handle encap with preceding decap. */
9642                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
9643                                 if (flow_dv_create_action_raw_encap
9644                                         (dev, actions, dev_flow, attr, error))
9645                                         return -rte_errno;
9646                                 dev_flow->dv.actions[actions_n++] =
9647                                         dev_flow->dv.encap_decap->action;
9648                         } else {
9649                                 /* Handle encap without preceding decap. */
9650                                 if (flow_dv_create_action_l2_encap
9651                                     (dev, actions, dev_flow, attr->transfer,
9652                                      error))
9653                                         return -rte_errno;
9654                                 dev_flow->dv.actions[actions_n++] =
9655                                         dev_flow->dv.encap_decap->action;
9656                         }
9657                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9658                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
9659                                 sample_act->action_flags |=
9660                                                         MLX5_FLOW_ACTION_ENCAP;
9661                         break;
9662                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
9663                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
9664                                 ;
9665                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
9666                                 if (flow_dv_create_action_l2_decap
9667                                     (dev, dev_flow, attr->transfer, error))
9668                                         return -rte_errno;
9669                                 dev_flow->dv.actions[actions_n++] =
9670                                         dev_flow->dv.encap_decap->action;
9671                         }
9672                         /* If decap is followed by encap, handle it at encap. */
9673                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9674                         break;
9675                 case RTE_FLOW_ACTION_TYPE_JUMP:
9676                         jump_group = ((const struct rte_flow_action_jump *)
9677                                                         action->conf)->group;
9678                         if (dev_flow->external && jump_group <
9679                                         MLX5_MAX_TABLES_EXTERNAL)
9680                                 jump_group *= MLX5_FLOW_TABLE_FACTOR;
9681                         ret = mlx5_flow_group_to_table(attr, dev_flow->external,
9682                                                        jump_group,
9683                                                        !!priv->fdb_def_rule,
9684                                                        &table, error);
9685                         if (ret)
9686                                 return ret;
9687                         tbl = flow_dv_tbl_resource_get(dev, table,
9688                                                        attr->egress,
9689                                                        attr->transfer, error);
9690                         if (!tbl)
9691                                 return rte_flow_error_set
9692                                                 (error, errno,
9693                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9694                                                  NULL,
9695                                                  "cannot create jump action.");
9696                         if (flow_dv_jump_tbl_resource_register
9697                             (dev, tbl, dev_flow, error)) {
9698                                 flow_dv_tbl_resource_release(dev, tbl);
9699                                 return rte_flow_error_set
9700                                                 (error, errno,
9701                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9702                                                  NULL,
9703                                                  "cannot create jump action.");
9704                         }
9705                         dev_flow->dv.actions[actions_n++] =
9706                                         dev_flow->dv.jump->action;
9707                         action_flags |= MLX5_FLOW_ACTION_JUMP;
9708                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
9709                         break;
9710                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
9711                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
9712                         if (flow_dv_convert_action_modify_mac
9713                                         (mhdr_res, actions, error))
9714                                 return -rte_errno;
9715                         action_flags |= actions->type ==
9716                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
9717                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
9718                                         MLX5_FLOW_ACTION_SET_MAC_DST;
9719                         break;
9720                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
9721                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
9722                         if (flow_dv_convert_action_modify_ipv4
9723                                         (mhdr_res, actions, error))
9724                                 return -rte_errno;
9725                         action_flags |= actions->type ==
9726                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
9727                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
9728                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
9729                         break;
9730                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
9731                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
9732                         if (flow_dv_convert_action_modify_ipv6
9733                                         (mhdr_res, actions, error))
9734                                 return -rte_errno;
9735                         action_flags |= actions->type ==
9736                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
9737                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
9738                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
9739                         break;
9740                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
9741                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
9742                         if (flow_dv_convert_action_modify_tp
9743                                         (mhdr_res, actions, items,
9744                                          &flow_attr, dev_flow, !!(action_flags &
9745                                          MLX5_FLOW_ACTION_DECAP), error))
9746                                 return -rte_errno;
9747                         action_flags |= actions->type ==
9748                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
9749                                         MLX5_FLOW_ACTION_SET_TP_SRC :
9750                                         MLX5_FLOW_ACTION_SET_TP_DST;
9751                         break;
9752                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
9753                         if (flow_dv_convert_action_modify_dec_ttl
9754                                         (mhdr_res, items, &flow_attr, dev_flow,
9755                                          !!(action_flags &
9756                                          MLX5_FLOW_ACTION_DECAP), error))
9757                                 return -rte_errno;
9758                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
9759                         break;
9760                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
9761                         if (flow_dv_convert_action_modify_ttl
9762                                         (mhdr_res, actions, items, &flow_attr,
9763                                          dev_flow, !!(action_flags &
9764                                          MLX5_FLOW_ACTION_DECAP), error))
9765                                 return -rte_errno;
9766                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
9767                         break;
9768                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
9769                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
9770                         if (flow_dv_convert_action_modify_tcp_seq
9771                                         (mhdr_res, actions, error))
9772                                 return -rte_errno;
9773                         action_flags |= actions->type ==
9774                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
9775                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
9776                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
9777                         break;
9778
9779                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
9780                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
9781                         if (flow_dv_convert_action_modify_tcp_ack
9782                                         (mhdr_res, actions, error))
9783                                 return -rte_errno;
9784                         action_flags |= actions->type ==
9785                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
9786                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
9787                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
9788                         break;
9789                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
9790                         if (flow_dv_convert_action_set_reg
9791                                         (mhdr_res, actions, error))
9792                                 return -rte_errno;
9793                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9794                         break;
9795                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
9796                         if (flow_dv_convert_action_copy_mreg
9797                                         (dev, mhdr_res, actions, error))
9798                                 return -rte_errno;
9799                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9800                         break;
9801                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
9802                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
9803                         dev_flow->handle->fate_action =
9804                                         MLX5_FLOW_FATE_DEFAULT_MISS;
9805                         break;
9806                 case RTE_FLOW_ACTION_TYPE_METER:
9807                         mtr = actions->conf;
9808                         if (!flow->meter) {
9809                                 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
9810                                                             attr, error);
9811                                 if (!fm)
9812                                         return rte_flow_error_set(error,
9813                                                 rte_errno,
9814                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9815                                                 NULL,
9816                                                 "meter not found "
9817                                                 "or invalid parameters");
9818                                 flow->meter = fm->idx;
9819                         }
9820                         /* Set the meter action. */
9821                         if (!fm) {
9822                                 fm = mlx5_ipool_get(priv->sh->ipool
9823                                                 [MLX5_IPOOL_MTR], flow->meter);
9824                                 if (!fm)
9825                                         return rte_flow_error_set(error,
9826                                                 rte_errno,
9827                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9828                                                 NULL,
9829                                                 "meter not found "
9830                                                 "or invalid parameters");
9831                         }
9832                         dev_flow->dv.actions[actions_n++] =
9833                                 fm->mfts->meter_action;
9834                         action_flags |= MLX5_FLOW_ACTION_METER;
9835                         break;
9836                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
9837                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
9838                                                               actions, error))
9839                                 return -rte_errno;
9840                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
9841                         break;
9842                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
9843                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
9844                                                               actions, error))
9845                                 return -rte_errno;
9846                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
9847                         break;
9848                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
9849                         sample_act_pos = actions_n;
9850                         ret = flow_dv_translate_action_sample(dev,
9851                                                               actions,
9852                                                               dev_flow, attr,
9853                                                               &num_of_dest,
9854                                                               sample_actions,
9855                                                               &sample_res,
9856                                                               error);
9857                         if (ret < 0)
9858                                 return ret;
9859                         actions_n++;
9860                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
9861                         /* put encap action into group if work with port id */
9862                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
9863                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
9864                                 sample_act->action_flags |=
9865                                                         MLX5_FLOW_ACTION_ENCAP;
9866                         break;
9867                 case RTE_FLOW_ACTION_TYPE_END:
9868                         actions_end = true;
9869                         if (mhdr_res->actions_num) {
9870                                 /* create modify action if needed. */
9871                                 if (flow_dv_modify_hdr_resource_register
9872                                         (dev, mhdr_res, dev_flow, error))
9873                                         return -rte_errno;
9874                                 dev_flow->dv.actions[modify_action_position] =
9875                                         handle->dvh.modify_hdr->action;
9876                         }
9877                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
9878                                 flow->counter =
9879                                         flow_dv_translate_create_counter(dev,
9880                                                 dev_flow, count, age);
9881
9882                                 if (!flow->counter)
9883                                         return rte_flow_error_set
9884                                                 (error, rte_errno,
9885                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9886                                                 NULL,
9887                                                 "cannot create counter"
9888                                                 " object.");
9889                                 dev_flow->dv.actions[actions_n] =
9890                                           (flow_dv_counter_get_by_idx(dev,
9891                                           flow->counter, NULL))->action;
9892                                 actions_n++;
9893                         }
9894                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
9895                                 ret = flow_dv_create_action_sample(dev,
9896                                                           dev_flow, attr,
9897                                                           num_of_dest,
9898                                                           &sample_res,
9899                                                           &mdest_res,
9900                                                           sample_actions,
9901                                                           action_flags,
9902                                                           error);
9903                                 if (ret < 0)
9904                                         return rte_flow_error_set
9905                                                 (error, rte_errno,
9906                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9907                                                 NULL,
9908                                                 "cannot create sample action");
9909                                 if (num_of_dest > 1) {
9910                                         dev_flow->dv.actions[sample_act_pos] =
9911                                         dev_flow->dv.dest_array_res->action;
9912                                 } else {
9913                                         dev_flow->dv.actions[sample_act_pos] =
9914                                         dev_flow->dv.sample_res->verbs_action;
9915                                 }
9916                         }
9917                         break;
9918                 default:
9919                         break;
9920                 }
9921                 if (mhdr_res->actions_num &&
9922                     modify_action_position == UINT32_MAX)
9923                         modify_action_position = actions_n++;
9924         }
9925         /*
9926          * For multiple destination (sample action with ratio=1), the encap
9927          * action and port id action will be combined into group action.
9928          * So need remove the original these actions in the flow and only
9929          * use the sample action instead of.
9930          */
9931         if (num_of_dest > 1 && sample_act->dr_port_id_action) {
9932                 int i;
9933                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
9934
9935                 for (i = 0; i < actions_n; i++) {
9936                         if ((sample_act->dr_encap_action &&
9937                                 sample_act->dr_encap_action ==
9938                                 dev_flow->dv.actions[i]) ||
9939                                 (sample_act->dr_port_id_action &&
9940                                 sample_act->dr_port_id_action ==
9941                                 dev_flow->dv.actions[i]))
9942                                 continue;
9943                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
9944                 }
9945                 memcpy((void *)dev_flow->dv.actions,
9946                                 (void *)temp_actions,
9947                                 tmp_actions_n * sizeof(void *));
9948                 actions_n = tmp_actions_n;
9949         }
9950         dev_flow->dv.actions_n = actions_n;
9951         dev_flow->act_flags = action_flags;
9952         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
9953                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
9954                 int item_type = items->type;
9955
9956                 if (!mlx5_flow_os_item_supported(item_type))
9957                         return rte_flow_error_set(error, ENOTSUP,
9958                                                   RTE_FLOW_ERROR_TYPE_ITEM,
9959                                                   NULL, "item not supported");
9960                 switch (item_type) {
9961                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
9962                         flow_dv_translate_item_port_id(dev, match_mask,
9963                                                        match_value, items);
9964                         last_item = MLX5_FLOW_ITEM_PORT_ID;
9965                         break;
9966                 case RTE_FLOW_ITEM_TYPE_ETH:
9967                         flow_dv_translate_item_eth(match_mask, match_value,
9968                                                    items, tunnel,
9969                                                    dev_flow->dv.group);
9970                         matcher.priority = action_flags &
9971                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
9972                                         !dev_flow->external ?
9973                                         MLX5_PRIORITY_MAP_L3 :
9974                                         MLX5_PRIORITY_MAP_L2;
9975                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
9976                                              MLX5_FLOW_LAYER_OUTER_L2;
9977                         break;
9978                 case RTE_FLOW_ITEM_TYPE_VLAN:
9979                         flow_dv_translate_item_vlan(dev_flow,
9980                                                     match_mask, match_value,
9981                                                     items, tunnel,
9982                                                     dev_flow->dv.group);
9983                         matcher.priority = MLX5_PRIORITY_MAP_L2;
9984                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
9985                                               MLX5_FLOW_LAYER_INNER_VLAN) :
9986                                              (MLX5_FLOW_LAYER_OUTER_L2 |
9987                                               MLX5_FLOW_LAYER_OUTER_VLAN);
9988                         break;
9989                 case RTE_FLOW_ITEM_TYPE_IPV4:
9990                         mlx5_flow_tunnel_ip_check(items, next_protocol,
9991                                                   &item_flags, &tunnel);
9992                         flow_dv_translate_item_ipv4(match_mask, match_value,
9993                                                     items, item_flags, tunnel,
9994                                                     dev_flow->dv.group);
9995                         matcher.priority = MLX5_PRIORITY_MAP_L3;
9996                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
9997                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
9998                         if (items->mask != NULL &&
9999                             ((const struct rte_flow_item_ipv4 *)
10000                              items->mask)->hdr.next_proto_id) {
10001                                 next_protocol =
10002                                         ((const struct rte_flow_item_ipv4 *)
10003                                          (items->spec))->hdr.next_proto_id;
10004                                 next_protocol &=
10005                                         ((const struct rte_flow_item_ipv4 *)
10006                                          (items->mask))->hdr.next_proto_id;
10007                         } else {
10008                                 /* Reset for inner layer. */
10009                                 next_protocol = 0xff;
10010                         }
10011                         break;
10012                 case RTE_FLOW_ITEM_TYPE_IPV6:
10013                         mlx5_flow_tunnel_ip_check(items, next_protocol,
10014                                                   &item_flags, &tunnel);
10015                         flow_dv_translate_item_ipv6(match_mask, match_value,
10016                                                     items, item_flags, tunnel,
10017                                                     dev_flow->dv.group);
10018                         matcher.priority = MLX5_PRIORITY_MAP_L3;
10019                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
10020                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
10021                         if (items->mask != NULL &&
10022                             ((const struct rte_flow_item_ipv6 *)
10023                              items->mask)->hdr.proto) {
10024                                 next_protocol =
10025                                         ((const struct rte_flow_item_ipv6 *)
10026                                          items->spec)->hdr.proto;
10027                                 next_protocol &=
10028                                         ((const struct rte_flow_item_ipv6 *)
10029                                          items->mask)->hdr.proto;
10030                         } else {
10031                                 /* Reset for inner layer. */
10032                                 next_protocol = 0xff;
10033                         }
10034                         break;
10035                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
10036                         flow_dv_translate_item_ipv6_frag_ext(match_mask,
10037                                                              match_value,
10038                                                              items, tunnel);
10039                         last_item = tunnel ?
10040                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
10041                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
10042                         if (items->mask != NULL &&
10043                             ((const struct rte_flow_item_ipv6_frag_ext *)
10044                              items->mask)->hdr.next_header) {
10045                                 next_protocol =
10046                                 ((const struct rte_flow_item_ipv6_frag_ext *)
10047                                  items->spec)->hdr.next_header;
10048                                 next_protocol &=
10049                                 ((const struct rte_flow_item_ipv6_frag_ext *)
10050                                  items->mask)->hdr.next_header;
10051                         } else {
10052                                 /* Reset for inner layer. */
10053                                 next_protocol = 0xff;
10054                         }
10055                         break;
10056                 case RTE_FLOW_ITEM_TYPE_TCP:
10057                         flow_dv_translate_item_tcp(match_mask, match_value,
10058                                                    items, tunnel);
10059                         matcher.priority = MLX5_PRIORITY_MAP_L4;
10060                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
10061                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
10062                         break;
10063                 case RTE_FLOW_ITEM_TYPE_UDP:
10064                         flow_dv_translate_item_udp(match_mask, match_value,
10065                                                    items, tunnel);
10066                         matcher.priority = MLX5_PRIORITY_MAP_L4;
10067                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
10068                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
10069                         break;
10070                 case RTE_FLOW_ITEM_TYPE_GRE:
10071                         flow_dv_translate_item_gre(match_mask, match_value,
10072                                                    items, tunnel);
10073                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10074                         last_item = MLX5_FLOW_LAYER_GRE;
10075                         break;
10076                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
10077                         flow_dv_translate_item_gre_key(match_mask,
10078                                                        match_value, items);
10079                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
10080                         break;
10081                 case RTE_FLOW_ITEM_TYPE_NVGRE:
10082                         flow_dv_translate_item_nvgre(match_mask, match_value,
10083                                                      items, tunnel);
10084                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10085                         last_item = MLX5_FLOW_LAYER_GRE;
10086                         break;
10087                 case RTE_FLOW_ITEM_TYPE_VXLAN:
10088                         flow_dv_translate_item_vxlan(match_mask, match_value,
10089                                                      items, tunnel);
10090                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10091                         last_item = MLX5_FLOW_LAYER_VXLAN;
10092                         break;
10093                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
10094                         flow_dv_translate_item_vxlan_gpe(match_mask,
10095                                                          match_value, items,
10096                                                          tunnel);
10097                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10098                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
10099                         break;
10100                 case RTE_FLOW_ITEM_TYPE_GENEVE:
10101                         flow_dv_translate_item_geneve(match_mask, match_value,
10102                                                       items, tunnel);
10103                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10104                         last_item = MLX5_FLOW_LAYER_GENEVE;
10105                         break;
10106                 case RTE_FLOW_ITEM_TYPE_MPLS:
10107                         flow_dv_translate_item_mpls(match_mask, match_value,
10108                                                     items, last_item, tunnel);
10109                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10110                         last_item = MLX5_FLOW_LAYER_MPLS;
10111                         break;
10112                 case RTE_FLOW_ITEM_TYPE_MARK:
10113                         flow_dv_translate_item_mark(dev, match_mask,
10114                                                     match_value, items);
10115                         last_item = MLX5_FLOW_ITEM_MARK;
10116                         break;
10117                 case RTE_FLOW_ITEM_TYPE_META:
10118                         flow_dv_translate_item_meta(dev, match_mask,
10119                                                     match_value, attr, items);
10120                         last_item = MLX5_FLOW_ITEM_METADATA;
10121                         break;
10122                 case RTE_FLOW_ITEM_TYPE_ICMP:
10123                         flow_dv_translate_item_icmp(match_mask, match_value,
10124                                                     items, tunnel);
10125                         last_item = MLX5_FLOW_LAYER_ICMP;
10126                         break;
10127                 case RTE_FLOW_ITEM_TYPE_ICMP6:
10128                         flow_dv_translate_item_icmp6(match_mask, match_value,
10129                                                       items, tunnel);
10130                         last_item = MLX5_FLOW_LAYER_ICMP6;
10131                         break;
10132                 case RTE_FLOW_ITEM_TYPE_TAG:
10133                         flow_dv_translate_item_tag(dev, match_mask,
10134                                                    match_value, items);
10135                         last_item = MLX5_FLOW_ITEM_TAG;
10136                         break;
10137                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
10138                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
10139                                                         match_value, items);
10140                         last_item = MLX5_FLOW_ITEM_TAG;
10141                         break;
10142                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
10143                         flow_dv_translate_item_tx_queue(dev, match_mask,
10144                                                         match_value,
10145                                                         items);
10146                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
10147                         break;
10148                 case RTE_FLOW_ITEM_TYPE_GTP:
10149                         flow_dv_translate_item_gtp(match_mask, match_value,
10150                                                    items, tunnel);
10151                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
10152                         last_item = MLX5_FLOW_LAYER_GTP;
10153                         break;
10154                 case RTE_FLOW_ITEM_TYPE_ECPRI:
10155                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
10156                                 /* Create it only the first time to be used. */
10157                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
10158                                 if (ret)
10159                                         return rte_flow_error_set
10160                                                 (error, -ret,
10161                                                 RTE_FLOW_ERROR_TYPE_ITEM,
10162                                                 NULL,
10163                                                 "cannot create eCPRI parser");
10164                         }
10165                         /* Adjust the length matcher and device flow value. */
10166                         matcher.mask.size = MLX5_ST_SZ_BYTES(fte_match_param);
10167                         dev_flow->dv.value.size =
10168                                         MLX5_ST_SZ_BYTES(fte_match_param);
10169                         flow_dv_translate_item_ecpri(dev, match_mask,
10170                                                      match_value, items);
10171                         /* No other protocol should follow eCPRI layer. */
10172                         last_item = MLX5_FLOW_LAYER_ECPRI;
10173                         break;
10174                 default:
10175                         break;
10176                 }
10177                 item_flags |= last_item;
10178         }
10179         /*
10180          * When E-Switch mode is enabled, we have two cases where we need to
10181          * set the source port manually.
10182          * The first one, is in case of Nic steering rule, and the second is
10183          * E-Switch rule where no port_id item was found. In both cases
10184          * the source port is set according the current port in use.
10185          */
10186         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
10187             (priv->representor || priv->master)) {
10188                 if (flow_dv_translate_item_port_id(dev, match_mask,
10189                                                    match_value, NULL))
10190                         return -rte_errno;
10191         }
10192 #ifdef RTE_LIBRTE_MLX5_DEBUG
10193         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
10194                                               dev_flow->dv.value.buf));
10195 #endif
10196         /*
10197          * Layers may be already initialized from prefix flow if this dev_flow
10198          * is the suffix flow.
10199          */
10200         handle->layers |= item_flags;
10201         if (action_flags & MLX5_FLOW_ACTION_RSS)
10202                 flow_dv_hashfields_set(dev_flow, rss_desc);
10203         /* Register matcher. */
10204         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
10205                                     matcher.mask.size);
10206         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
10207                                                      matcher.priority);
10208         /* reserved field no needs to be set to 0 here. */
10209         tbl_key.domain = attr->transfer;
10210         tbl_key.direction = attr->egress;
10211         tbl_key.table_id = dev_flow->dv.group;
10212         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
10213                 return -rte_errno;
10214         return 0;
10215 }
10216
10217 /**
10218  * Apply the flow to the NIC, lock free,
10219  * (mutex should be acquired by caller).
10220  *
10221  * @param[in] dev
10222  *   Pointer to the Ethernet device structure.
10223  * @param[in, out] flow
10224  *   Pointer to flow structure.
10225  * @param[out] error
10226  *   Pointer to error structure.
10227  *
10228  * @return
10229  *   0 on success, a negative errno value otherwise and rte_errno is set.
10230  */
10231 static int
10232 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
10233                 struct rte_flow_error *error)
10234 {
10235         struct mlx5_flow_dv_workspace *dv;
10236         struct mlx5_flow_handle *dh;
10237         struct mlx5_flow_handle_dv *dv_h;
10238         struct mlx5_flow *dev_flow;
10239         struct mlx5_priv *priv = dev->data->dev_private;
10240         uint32_t handle_idx;
10241         int n;
10242         int err;
10243         int idx;
10244
10245         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
10246                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
10247                 dv = &dev_flow->dv;
10248                 dh = dev_flow->handle;
10249                 dv_h = &dh->dvh;
10250                 n = dv->actions_n;
10251                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
10252                         if (dv->transfer) {
10253                                 dv->actions[n++] = priv->sh->esw_drop_action;
10254                         } else {
10255                                 struct mlx5_hrxq *drop_hrxq;
10256                                 drop_hrxq = mlx5_drop_action_create(dev);
10257                                 if (!drop_hrxq) {
10258                                         rte_flow_error_set
10259                                                 (error, errno,
10260                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10261                                                  NULL,
10262                                                  "cannot get drop hash queue");
10263                                         goto error;
10264                                 }
10265                                 /*
10266                                  * Drop queues will be released by the specify
10267                                  * mlx5_drop_action_destroy() function. Assign
10268                                  * the special index to hrxq to mark the queue
10269                                  * has been allocated.
10270                                  */
10271                                 dh->rix_hrxq = UINT32_MAX;
10272                                 dv->actions[n++] = drop_hrxq->action;
10273                         }
10274                 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
10275                            !dv_h->rix_sample && !dv_h->rix_dest_array) {
10276                         struct mlx5_hrxq *hrxq;
10277                         uint32_t hrxq_idx;
10278                         struct mlx5_flow_rss_desc *rss_desc =
10279                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
10280                                 [!!priv->flow_nested_idx];
10281
10282                         MLX5_ASSERT(rss_desc->queue_num);
10283                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
10284                                                  MLX5_RSS_HASH_KEY_LEN,
10285                                                  dev_flow->hash_fields,
10286                                                  rss_desc->queue,
10287                                                  rss_desc->queue_num);
10288                         if (!hrxq_idx) {
10289                                 hrxq_idx = mlx5_hrxq_new
10290                                                 (dev, rss_desc->key,
10291                                                  MLX5_RSS_HASH_KEY_LEN,
10292                                                  dev_flow->hash_fields,
10293                                                  rss_desc->queue,
10294                                                  rss_desc->queue_num,
10295                                                  !!(dh->layers &
10296                                                  MLX5_FLOW_LAYER_TUNNEL));
10297                         }
10298                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
10299                                               hrxq_idx);
10300                         if (!hrxq) {
10301                                 rte_flow_error_set
10302                                         (error, rte_errno,
10303                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10304                                          "cannot get hash queue");
10305                                 goto error;
10306                         }
10307                         dh->rix_hrxq = hrxq_idx;
10308                         dv->actions[n++] = hrxq->action;
10309                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
10310                         if (flow_dv_default_miss_resource_register
10311                                         (dev, error)) {
10312                                 rte_flow_error_set
10313                                         (error, rte_errno,
10314                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10315                                          "cannot create default miss resource");
10316                                 goto error_default_miss;
10317                         }
10318                         dh->rix_default_fate =  MLX5_FLOW_FATE_DEFAULT_MISS;
10319                         dv->actions[n++] = priv->sh->default_miss.action;
10320                 }
10321                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
10322                                                (void *)&dv->value, n,
10323                                                dv->actions, &dh->drv_flow);
10324                 if (err) {
10325                         rte_flow_error_set(error, errno,
10326                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10327                                            NULL,
10328                                            "hardware refuses to create flow");
10329                         goto error;
10330                 }
10331                 if (priv->vmwa_context &&
10332                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
10333                         /*
10334                          * The rule contains the VLAN pattern.
10335                          * For VF we are going to create VLAN
10336                          * interface to make hypervisor set correct
10337                          * e-Switch vport context.
10338                          */
10339                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
10340                 }
10341         }
10342         return 0;
10343 error:
10344         if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
10345                 flow_dv_default_miss_resource_release(dev);
10346 error_default_miss:
10347         err = rte_errno; /* Save rte_errno before cleanup. */
10348         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
10349                        handle_idx, dh, next) {
10350                 /* hrxq is union, don't clear it if the flag is not set. */
10351                 if (dh->rix_hrxq) {
10352                         if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
10353                                 mlx5_drop_action_destroy(dev);
10354                                 dh->rix_hrxq = 0;
10355                         } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
10356                                 mlx5_hrxq_release(dev, dh->rix_hrxq);
10357                                 dh->rix_hrxq = 0;
10358                         }
10359                 }
10360                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
10361                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
10362         }
10363         rte_errno = err; /* Restore rte_errno. */
10364         return -rte_errno;
10365 }
10366
10367 /**
10368  * Release the flow matcher.
10369  *
10370  * @param dev
10371  *   Pointer to Ethernet device.
10372  * @param handle
10373  *   Pointer to mlx5_flow_handle.
10374  *
10375  * @return
10376  *   1 while a reference on it exists, 0 when freed.
10377  */
10378 static int
10379 flow_dv_matcher_release(struct rte_eth_dev *dev,
10380                         struct mlx5_flow_handle *handle)
10381 {
10382         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
10383
10384         MLX5_ASSERT(matcher->matcher_object);
10385         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
10386                 dev->data->port_id, (void *)matcher,
10387                 rte_atomic32_read(&matcher->refcnt));
10388         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
10389                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10390                            (matcher->matcher_object));
10391                 LIST_REMOVE(matcher, next);
10392                 /* table ref-- in release interface. */
10393                 flow_dv_tbl_resource_release(dev, matcher->tbl);
10394                 mlx5_free(matcher);
10395                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
10396                         dev->data->port_id, (void *)matcher);
10397                 return 0;
10398         }
10399         return 1;
10400 }
10401
10402 /**
10403  * Release an encap/decap resource.
10404  *
10405  * @param dev
10406  *   Pointer to Ethernet device.
10407  * @param encap_decap_idx
10408  *   Index of encap decap resource.
10409  *
10410  * @return
10411  *   1 while a reference on it exists, 0 when freed.
10412  */
10413 static int
10414 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
10415                                      uint32_t encap_decap_idx)
10416 {
10417         struct mlx5_priv *priv = dev->data->dev_private;
10418         uint32_t idx = encap_decap_idx;
10419         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
10420
10421         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
10422                          idx);
10423         if (!cache_resource)
10424                 return 0;
10425         MLX5_ASSERT(cache_resource->action);
10426         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
10427                 (void *)cache_resource,
10428                 rte_atomic32_read(&cache_resource->refcnt));
10429         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10430                 claim_zero(mlx5_flow_os_destroy_flow_action
10431                                                 (cache_resource->action));
10432                 mlx5_hlist_remove(priv->sh->encaps_decaps,
10433                                   &cache_resource->entry);
10434                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
10435                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
10436                         (void *)cache_resource);
10437                 return 0;
10438         }
10439         return 1;
10440 }
10441
10442 /**
10443  * Release an jump to table action resource.
10444  *
10445  * @param dev
10446  *   Pointer to Ethernet device.
10447  * @param handle
10448  *   Pointer to mlx5_flow_handle.
10449  *
10450  * @return
10451  *   1 while a reference on it exists, 0 when freed.
10452  */
10453 static int
10454 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
10455                                   struct mlx5_flow_handle *handle)
10456 {
10457         struct mlx5_priv *priv = dev->data->dev_private;
10458         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
10459         struct mlx5_flow_tbl_data_entry *tbl_data;
10460
10461         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
10462                              handle->rix_jump);
10463         if (!tbl_data)
10464                 return 0;
10465         cache_resource = &tbl_data->jump;
10466         MLX5_ASSERT(cache_resource->action);
10467         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
10468                 (void *)cache_resource,
10469                 rte_atomic32_read(&cache_resource->refcnt));
10470         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10471                 claim_zero(mlx5_flow_os_destroy_flow_action
10472                                                 (cache_resource->action));
10473                 /* jump action memory free is inside the table release. */
10474                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
10475                 DRV_LOG(DEBUG, "jump table resource %p: removed",
10476                         (void *)cache_resource);
10477                 return 0;
10478         }
10479         return 1;
10480 }
10481
10482 /**
10483  * Release a default miss resource.
10484  *
10485  * @param dev
10486  *   Pointer to Ethernet device.
10487  * @return
10488  *   1 while a reference on it exists, 0 when freed.
10489  */
10490 static int
10491 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
10492 {
10493         struct mlx5_priv *priv = dev->data->dev_private;
10494         struct mlx5_dev_ctx_shared *sh = priv->sh;
10495         struct mlx5_flow_default_miss_resource *cache_resource =
10496                         &sh->default_miss;
10497
10498         MLX5_ASSERT(cache_resource->action);
10499         DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--",
10500                         (void *)cache_resource->action,
10501                         rte_atomic32_read(&cache_resource->refcnt));
10502         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10503                 claim_zero(mlx5_glue->destroy_flow_action
10504                                 (cache_resource->action));
10505                 DRV_LOG(DEBUG, "default miss resource %p: removed",
10506                                 (void *)cache_resource->action);
10507                 return 0;
10508         }
10509         return 1;
10510 }
10511
10512 /**
10513  * Release a modify-header resource.
10514  *
10515  * @param dev
10516  *   Pointer to Ethernet device.
10517  * @param handle
10518  *   Pointer to mlx5_flow_handle.
10519  *
10520  * @return
10521  *   1 while a reference on it exists, 0 when freed.
10522  */
10523 static int
10524 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
10525                                     struct mlx5_flow_handle *handle)
10526 {
10527         struct mlx5_priv *priv = dev->data->dev_private;
10528         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
10529                                                         handle->dvh.modify_hdr;
10530
10531         MLX5_ASSERT(cache_resource->action);
10532         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
10533                 (void *)cache_resource,
10534                 rte_atomic32_read(&cache_resource->refcnt));
10535         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10536                 claim_zero(mlx5_flow_os_destroy_flow_action
10537                                                 (cache_resource->action));
10538                 mlx5_hlist_remove(priv->sh->modify_cmds,
10539                                   &cache_resource->entry);
10540                 mlx5_free(cache_resource);
10541                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
10542                         (void *)cache_resource);
10543                 return 0;
10544         }
10545         return 1;
10546 }
10547
10548 /**
10549  * Release port ID action resource.
10550  *
10551  * @param dev
10552  *   Pointer to Ethernet device.
10553  * @param handle
10554  *   Pointer to mlx5_flow_handle.
10555  *
10556  * @return
10557  *   1 while a reference on it exists, 0 when freed.
10558  */
10559 static int
10560 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
10561                                         uint32_t port_id)
10562 {
10563         struct mlx5_priv *priv = dev->data->dev_private;
10564         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
10565         uint32_t idx = port_id;
10566
10567         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
10568                                         idx);
10569         if (!cache_resource)
10570                 return 0;
10571         MLX5_ASSERT(cache_resource->action);
10572         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
10573                 (void *)cache_resource,
10574                 rte_atomic32_read(&cache_resource->refcnt));
10575         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10576                 claim_zero(mlx5_flow_os_destroy_flow_action
10577                                                 (cache_resource->action));
10578                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
10579                              &priv->sh->port_id_action_list, idx,
10580                              cache_resource, next);
10581                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx);
10582                 DRV_LOG(DEBUG, "port id action resource %p: removed",
10583                         (void *)cache_resource);
10584                 return 0;
10585         }
10586         return 1;
10587 }
10588
10589 /**
10590  * Release push vlan action resource.
10591  *
10592  * @param dev
10593  *   Pointer to Ethernet device.
10594  * @param handle
10595  *   Pointer to mlx5_flow_handle.
10596  *
10597  * @return
10598  *   1 while a reference on it exists, 0 when freed.
10599  */
10600 static int
10601 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
10602                                           struct mlx5_flow_handle *handle)
10603 {
10604         struct mlx5_priv *priv = dev->data->dev_private;
10605         uint32_t idx = handle->dvh.rix_push_vlan;
10606         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
10607
10608         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
10609                                         idx);
10610         if (!cache_resource)
10611                 return 0;
10612         MLX5_ASSERT(cache_resource->action);
10613         DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
10614                 (void *)cache_resource,
10615                 rte_atomic32_read(&cache_resource->refcnt));
10616         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10617                 claim_zero(mlx5_flow_os_destroy_flow_action
10618                                                 (cache_resource->action));
10619                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
10620                              &priv->sh->push_vlan_action_list, idx,
10621                              cache_resource, next);
10622                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
10623                 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
10624                         (void *)cache_resource);
10625                 return 0;
10626         }
10627         return 1;
10628 }
10629
10630 /**
10631  * Release the fate resource.
10632  *
10633  * @param dev
10634  *   Pointer to Ethernet device.
10635  * @param handle
10636  *   Pointer to mlx5_flow_handle.
10637  */
10638 static void
10639 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
10640                                struct mlx5_flow_handle *handle)
10641 {
10642         if (!handle->rix_fate)
10643                 return;
10644         switch (handle->fate_action) {
10645         case MLX5_FLOW_FATE_DROP:
10646                 mlx5_drop_action_destroy(dev);
10647                 break;
10648         case MLX5_FLOW_FATE_QUEUE:
10649                 mlx5_hrxq_release(dev, handle->rix_hrxq);
10650                 break;
10651         case MLX5_FLOW_FATE_JUMP:
10652                 flow_dv_jump_tbl_resource_release(dev, handle);
10653                 break;
10654         case MLX5_FLOW_FATE_PORT_ID:
10655                 flow_dv_port_id_action_resource_release(dev,
10656                                 handle->rix_port_id_action);
10657                 break;
10658         case MLX5_FLOW_FATE_DEFAULT_MISS:
10659                 flow_dv_default_miss_resource_release(dev);
10660                 break;
10661         default:
10662                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
10663                 break;
10664         }
10665         handle->rix_fate = 0;
10666 }
10667
10668 /**
10669  * Release an sample resource.
10670  *
10671  * @param dev
10672  *   Pointer to Ethernet device.
10673  * @param handle
10674  *   Pointer to mlx5_flow_handle.
10675  *
10676  * @return
10677  *   1 while a reference on it exists, 0 when freed.
10678  */
10679 static int
10680 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
10681                                      struct mlx5_flow_handle *handle)
10682 {
10683         struct mlx5_priv *priv = dev->data->dev_private;
10684         uint32_t idx = handle->dvh.rix_sample;
10685         struct mlx5_flow_dv_sample_resource *cache_resource;
10686
10687         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
10688                          idx);
10689         if (!cache_resource)
10690                 return 0;
10691         MLX5_ASSERT(cache_resource->verbs_action);
10692         DRV_LOG(DEBUG, "sample resource %p: refcnt %d--",
10693                 (void *)cache_resource,
10694                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10695         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10696                                __ATOMIC_RELAXED) == 0) {
10697                 if (cache_resource->verbs_action)
10698                         claim_zero(mlx5_glue->destroy_flow_action
10699                                         (cache_resource->verbs_action));
10700                 if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
10701                         if (cache_resource->default_miss)
10702                                 claim_zero(mlx5_glue->destroy_flow_action
10703                                   (cache_resource->default_miss));
10704                 }
10705                 if (cache_resource->normal_path_tbl)
10706                         flow_dv_tbl_resource_release(dev,
10707                                 cache_resource->normal_path_tbl);
10708         }
10709         if (cache_resource->sample_idx.rix_hrxq &&
10710                 !mlx5_hrxq_release(dev,
10711                         cache_resource->sample_idx.rix_hrxq))
10712                 cache_resource->sample_idx.rix_hrxq = 0;
10713         if (cache_resource->sample_idx.rix_tag &&
10714                 !flow_dv_tag_release(dev,
10715                         cache_resource->sample_idx.rix_tag))
10716                 cache_resource->sample_idx.rix_tag = 0;
10717         if (cache_resource->sample_idx.cnt) {
10718                 flow_dv_counter_release(dev,
10719                         cache_resource->sample_idx.cnt);
10720                 cache_resource->sample_idx.cnt = 0;
10721         }
10722         if (!__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)) {
10723                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
10724                              &priv->sh->sample_action_list, idx,
10725                              cache_resource, next);
10726                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], idx);
10727                 DRV_LOG(DEBUG, "sample resource %p: removed",
10728                         (void *)cache_resource);
10729                 return 0;
10730         }
10731         return 1;
10732 }
10733
10734 /**
10735  * Release an destination array resource.
10736  *
10737  * @param dev
10738  *   Pointer to Ethernet device.
10739  * @param handle
10740  *   Pointer to mlx5_flow_handle.
10741  *
10742  * @return
10743  *   1 while a reference on it exists, 0 when freed.
10744  */
10745 static int
10746 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
10747                                      struct mlx5_flow_handle *handle)
10748 {
10749         struct mlx5_priv *priv = dev->data->dev_private;
10750         struct mlx5_flow_dv_dest_array_resource *cache_resource;
10751         struct mlx5_flow_sub_actions_idx *mdest_act_res;
10752         uint32_t idx = handle->dvh.rix_dest_array;
10753         uint32_t i = 0;
10754
10755         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10756                          idx);
10757         if (!cache_resource)
10758                 return 0;
10759         MLX5_ASSERT(cache_resource->action);
10760         DRV_LOG(DEBUG, "destination array resource %p: refcnt %d--",
10761                 (void *)cache_resource,
10762                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10763         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10764                                __ATOMIC_RELAXED) == 0) {
10765                 if (cache_resource->action)
10766                         claim_zero(mlx5_glue->destroy_flow_action
10767                                                 (cache_resource->action));
10768                 for (; i < cache_resource->num_of_dest; i++) {
10769                         mdest_act_res = &cache_resource->sample_idx[i];
10770                         if (mdest_act_res->rix_hrxq) {
10771                                 mlx5_hrxq_release(dev,
10772                                         mdest_act_res->rix_hrxq);
10773                                 mdest_act_res->rix_hrxq = 0;
10774                         }
10775                         if (mdest_act_res->rix_encap_decap) {
10776                                 flow_dv_encap_decap_resource_release(dev,
10777                                         mdest_act_res->rix_encap_decap);
10778                                 mdest_act_res->rix_encap_decap = 0;
10779                         }
10780                         if (mdest_act_res->rix_port_id_action) {
10781                                 flow_dv_port_id_action_resource_release(dev,
10782                                         mdest_act_res->rix_port_id_action);
10783                                 mdest_act_res->rix_port_id_action = 0;
10784                         }
10785                         if (mdest_act_res->rix_tag) {
10786                                 flow_dv_tag_release(dev,
10787                                         mdest_act_res->rix_tag);
10788                                 mdest_act_res->rix_tag = 0;
10789                         }
10790                 }
10791                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10792                              &priv->sh->dest_array_list, idx,
10793                              cache_resource, next);
10794                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], idx);
10795                 DRV_LOG(DEBUG, "destination array resource %p: removed",
10796                         (void *)cache_resource);
10797                 return 0;
10798         }
10799         return 1;
10800 }
10801
10802 /**
10803  * Remove the flow from the NIC but keeps it in memory.
10804  * Lock free, (mutex should be acquired by caller).
10805  *
10806  * @param[in] dev
10807  *   Pointer to Ethernet device.
10808  * @param[in, out] flow
10809  *   Pointer to flow structure.
10810  */
10811 static void
10812 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
10813 {
10814         struct mlx5_flow_handle *dh;
10815         uint32_t handle_idx;
10816         struct mlx5_priv *priv = dev->data->dev_private;
10817
10818         if (!flow)
10819                 return;
10820         handle_idx = flow->dev_handles;
10821         while (handle_idx) {
10822                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10823                                     handle_idx);
10824                 if (!dh)
10825                         return;
10826                 if (dh->drv_flow) {
10827                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
10828                         dh->drv_flow = NULL;
10829                 }
10830                 if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
10831                     dh->fate_action == MLX5_FLOW_FATE_QUEUE ||
10832                     dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
10833                         flow_dv_fate_resource_release(dev, dh);
10834                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
10835                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
10836                 handle_idx = dh->next.next;
10837         }
10838 }
10839
10840 /**
10841  * Remove the flow from the NIC and the memory.
10842  * Lock free, (mutex should be acquired by caller).
10843  *
10844  * @param[in] dev
10845  *   Pointer to the Ethernet device structure.
10846  * @param[in, out] flow
10847  *   Pointer to flow structure.
10848  */
10849 static void
10850 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
10851 {
10852         struct mlx5_flow_handle *dev_handle;
10853         struct mlx5_priv *priv = dev->data->dev_private;
10854
10855         if (!flow)
10856                 return;
10857         __flow_dv_remove(dev, flow);
10858         if (flow->counter) {
10859                 flow_dv_counter_release(dev, flow->counter);
10860                 flow->counter = 0;
10861         }
10862         if (flow->meter) {
10863                 struct mlx5_flow_meter *fm;
10864
10865                 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
10866                                     flow->meter);
10867                 if (fm)
10868                         mlx5_flow_meter_detach(fm);
10869                 flow->meter = 0;
10870         }
10871         while (flow->dev_handles) {
10872                 uint32_t tmp_idx = flow->dev_handles;
10873
10874                 dev_handle = mlx5_ipool_get(priv->sh->ipool
10875                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
10876                 if (!dev_handle)
10877                         return;
10878                 flow->dev_handles = dev_handle->next.next;
10879                 if (dev_handle->dvh.matcher)
10880                         flow_dv_matcher_release(dev, dev_handle);
10881                 if (dev_handle->dvh.rix_sample)
10882                         flow_dv_sample_resource_release(dev, dev_handle);
10883                 if (dev_handle->dvh.rix_dest_array)
10884                         flow_dv_dest_array_resource_release(dev, dev_handle);
10885                 if (dev_handle->dvh.rix_encap_decap)
10886                         flow_dv_encap_decap_resource_release(dev,
10887                                 dev_handle->dvh.rix_encap_decap);
10888                 if (dev_handle->dvh.modify_hdr)
10889                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
10890                 if (dev_handle->dvh.rix_push_vlan)
10891                         flow_dv_push_vlan_action_resource_release(dev,
10892                                                                   dev_handle);
10893                 if (dev_handle->dvh.rix_tag)
10894                         flow_dv_tag_release(dev,
10895                                             dev_handle->dvh.rix_tag);
10896                 flow_dv_fate_resource_release(dev, dev_handle);
10897                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10898                            tmp_idx);
10899         }
10900 }
10901
10902 /**
10903  * Query a dv flow  rule for its statistics via devx.
10904  *
10905  * @param[in] dev
10906  *   Pointer to Ethernet device.
10907  * @param[in] flow
10908  *   Pointer to the sub flow.
10909  * @param[out] data
10910  *   data retrieved by the query.
10911  * @param[out] error
10912  *   Perform verbose error reporting if not NULL.
10913  *
10914  * @return
10915  *   0 on success, a negative errno value otherwise and rte_errno is set.
10916  */
10917 static int
10918 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
10919                     void *data, struct rte_flow_error *error)
10920 {
10921         struct mlx5_priv *priv = dev->data->dev_private;
10922         struct rte_flow_query_count *qc = data;
10923
10924         if (!priv->config.devx)
10925                 return rte_flow_error_set(error, ENOTSUP,
10926                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10927                                           NULL,
10928                                           "counters are not supported");
10929         if (flow->counter) {
10930                 uint64_t pkts, bytes;
10931                 struct mlx5_flow_counter *cnt;
10932
10933                 cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
10934                                                  NULL);
10935                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
10936                                                &bytes);
10937
10938                 if (err)
10939                         return rte_flow_error_set(error, -err,
10940                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10941                                         NULL, "cannot read counters");
10942                 qc->hits_set = 1;
10943                 qc->bytes_set = 1;
10944                 qc->hits = pkts - cnt->hits;
10945                 qc->bytes = bytes - cnt->bytes;
10946                 if (qc->reset) {
10947                         cnt->hits = pkts;
10948                         cnt->bytes = bytes;
10949                 }
10950                 return 0;
10951         }
10952         return rte_flow_error_set(error, EINVAL,
10953                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10954                                   NULL,
10955                                   "counters are not available");
10956 }
10957
10958 /**
10959  * Query a flow.
10960  *
10961  * @see rte_flow_query()
10962  * @see rte_flow_ops
10963  */
10964 static int
10965 flow_dv_query(struct rte_eth_dev *dev,
10966               struct rte_flow *flow __rte_unused,
10967               const struct rte_flow_action *actions __rte_unused,
10968               void *data __rte_unused,
10969               struct rte_flow_error *error __rte_unused)
10970 {
10971         int ret = -EINVAL;
10972
10973         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
10974                 switch (actions->type) {
10975                 case RTE_FLOW_ACTION_TYPE_VOID:
10976                         break;
10977                 case RTE_FLOW_ACTION_TYPE_COUNT:
10978                         ret = flow_dv_query_count(dev, flow, data, error);
10979                         break;
10980                 default:
10981                         return rte_flow_error_set(error, ENOTSUP,
10982                                                   RTE_FLOW_ERROR_TYPE_ACTION,
10983                                                   actions,
10984                                                   "action not supported");
10985                 }
10986         }
10987         return ret;
10988 }
10989
10990 /**
10991  * Destroy the meter table set.
10992  * Lock free, (mutex should be acquired by caller).
10993  *
10994  * @param[in] dev
10995  *   Pointer to Ethernet device.
10996  * @param[in] tbl
10997  *   Pointer to the meter table set.
10998  *
10999  * @return
11000  *   Always 0.
11001  */
11002 static int
11003 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
11004                         struct mlx5_meter_domains_infos *tbl)
11005 {
11006         struct mlx5_priv *priv = dev->data->dev_private;
11007         struct mlx5_meter_domains_infos *mtd =
11008                                 (struct mlx5_meter_domains_infos *)tbl;
11009
11010         if (!mtd || !priv->config.dv_flow_en)
11011                 return 0;
11012         if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
11013                 claim_zero(mlx5_flow_os_destroy_flow
11014                            (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
11015         if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
11016                 claim_zero(mlx5_flow_os_destroy_flow
11017                            (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
11018         if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
11019                 claim_zero(mlx5_flow_os_destroy_flow
11020                            (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
11021         if (mtd->egress.color_matcher)
11022                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11023                            (mtd->egress.color_matcher));
11024         if (mtd->egress.any_matcher)
11025                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11026                            (mtd->egress.any_matcher));
11027         if (mtd->egress.tbl)
11028                 flow_dv_tbl_resource_release(dev, mtd->egress.tbl);
11029         if (mtd->egress.sfx_tbl)
11030                 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl);
11031         if (mtd->ingress.color_matcher)
11032                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11033                            (mtd->ingress.color_matcher));
11034         if (mtd->ingress.any_matcher)
11035                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11036                            (mtd->ingress.any_matcher));
11037         if (mtd->ingress.tbl)
11038                 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl);
11039         if (mtd->ingress.sfx_tbl)
11040                 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl);
11041         if (mtd->transfer.color_matcher)
11042                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11043                            (mtd->transfer.color_matcher));
11044         if (mtd->transfer.any_matcher)
11045                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11046                            (mtd->transfer.any_matcher));
11047         if (mtd->transfer.tbl)
11048                 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl);
11049         if (mtd->transfer.sfx_tbl)
11050                 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl);
11051         if (mtd->drop_actn)
11052                 claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn));
11053         mlx5_free(mtd);
11054         return 0;
11055 }
11056
11057 /* Number of meter flow actions, count and jump or count and drop. */
11058 #define METER_ACTIONS 2
11059
11060 /**
11061  * Create specify domain meter table and suffix table.
11062  *
11063  * @param[in] dev
11064  *   Pointer to Ethernet device.
11065  * @param[in,out] mtb
11066  *   Pointer to DV meter table set.
11067  * @param[in] egress
11068  *   Table attribute.
11069  * @param[in] transfer
11070  *   Table attribute.
11071  * @param[in] color_reg_c_idx
11072  *   Reg C index for color match.
11073  *
11074  * @return
11075  *   0 on success, -1 otherwise and rte_errno is set.
11076  */
11077 static int
11078 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
11079                            struct mlx5_meter_domains_infos *mtb,
11080                            uint8_t egress, uint8_t transfer,
11081                            uint32_t color_reg_c_idx)
11082 {
11083         struct mlx5_priv *priv = dev->data->dev_private;
11084         struct mlx5_dev_ctx_shared *sh = priv->sh;
11085         struct mlx5_flow_dv_match_params mask = {
11086                 .size = sizeof(mask.buf),
11087         };
11088         struct mlx5_flow_dv_match_params value = {
11089                 .size = sizeof(value.buf),
11090         };
11091         struct mlx5dv_flow_matcher_attr dv_attr = {
11092                 .type = IBV_FLOW_ATTR_NORMAL,
11093                 .priority = 0,
11094                 .match_criteria_enable = 0,
11095                 .match_mask = (void *)&mask,
11096         };
11097         void *actions[METER_ACTIONS];
11098         struct mlx5_meter_domain_info *dtb;
11099         struct rte_flow_error error;
11100         int i = 0;
11101         int ret;
11102
11103         if (transfer)
11104                 dtb = &mtb->transfer;
11105         else if (egress)
11106                 dtb = &mtb->egress;
11107         else
11108                 dtb = &mtb->ingress;
11109         /* Create the meter table with METER level. */
11110         dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
11111                                             egress, transfer, &error);
11112         if (!dtb->tbl) {
11113                 DRV_LOG(ERR, "Failed to create meter policer table.");
11114                 return -1;
11115         }
11116         /* Create the meter suffix table with SUFFIX level. */
11117         dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
11118                                             MLX5_FLOW_TABLE_LEVEL_SUFFIX,
11119                                             egress, transfer, &error);
11120         if (!dtb->sfx_tbl) {
11121                 DRV_LOG(ERR, "Failed to create meter suffix table.");
11122                 return -1;
11123         }
11124         /* Create matchers, Any and Color. */
11125         dv_attr.priority = 3;
11126         dv_attr.match_criteria_enable = 0;
11127         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
11128                                                &dtb->any_matcher);
11129         if (ret) {
11130                 DRV_LOG(ERR, "Failed to create meter"
11131                              " policer default matcher.");
11132                 goto error_exit;
11133         }
11134         dv_attr.priority = 0;
11135         dv_attr.match_criteria_enable =
11136                                 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
11137         flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
11138                                rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
11139         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
11140                                                &dtb->color_matcher);
11141         if (ret) {
11142                 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
11143                 goto error_exit;
11144         }
11145         if (mtb->count_actns[RTE_MTR_DROPPED])
11146                 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
11147         actions[i++] = mtb->drop_actn;
11148         /* Default rule: lowest priority, match any, actions: drop. */
11149         ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i,
11150                                        actions,
11151                                        &dtb->policer_rules[RTE_MTR_DROPPED]);
11152         if (ret) {
11153                 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
11154                 goto error_exit;
11155         }
11156         return 0;
11157 error_exit:
11158         return -1;
11159 }
11160
11161 /**
11162  * Create the needed meter and suffix tables.
11163  * Lock free, (mutex should be acquired by caller).
11164  *
11165  * @param[in] dev
11166  *   Pointer to Ethernet device.
11167  * @param[in] fm
11168  *   Pointer to the flow meter.
11169  *
11170  * @return
11171  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
11172  */
11173 static struct mlx5_meter_domains_infos *
11174 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
11175                        const struct mlx5_flow_meter *fm)
11176 {
11177         struct mlx5_priv *priv = dev->data->dev_private;
11178         struct mlx5_meter_domains_infos *mtb;
11179         int ret;
11180         int i;
11181
11182         if (!priv->mtr_en) {
11183                 rte_errno = ENOTSUP;
11184                 return NULL;
11185         }
11186         mtb = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*mtb), 0, SOCKET_ID_ANY);
11187         if (!mtb) {
11188                 DRV_LOG(ERR, "Failed to allocate memory for meter.");
11189                 return NULL;
11190         }
11191         /* Create meter count actions */
11192         for (i = 0; i <= RTE_MTR_DROPPED; i++) {
11193                 struct mlx5_flow_counter *cnt;
11194                 if (!fm->policer_stats.cnt[i])
11195                         continue;
11196                 cnt = flow_dv_counter_get_by_idx(dev,
11197                       fm->policer_stats.cnt[i], NULL);
11198                 mtb->count_actns[i] = cnt->action;
11199         }
11200         /* Create drop action. */
11201         ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn);
11202         if (ret) {
11203                 DRV_LOG(ERR, "Failed to create drop action.");
11204                 goto error_exit;
11205         }
11206         /* Egress meter table. */
11207         ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
11208         if (ret) {
11209                 DRV_LOG(ERR, "Failed to prepare egress meter table.");
11210                 goto error_exit;
11211         }
11212         /* Ingress meter table. */
11213         ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
11214         if (ret) {
11215                 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
11216                 goto error_exit;
11217         }
11218         /* FDB meter table. */
11219         if (priv->config.dv_esw_en) {
11220                 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
11221                                                  priv->mtr_color_reg);
11222                 if (ret) {
11223                         DRV_LOG(ERR, "Failed to prepare fdb meter table.");
11224                         goto error_exit;
11225                 }
11226         }
11227         return mtb;
11228 error_exit:
11229         flow_dv_destroy_mtr_tbl(dev, mtb);
11230         return NULL;
11231 }
11232
11233 /**
11234  * Destroy domain policer rule.
11235  *
11236  * @param[in] dt
11237  *   Pointer to domain table.
11238  */
11239 static void
11240 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
11241 {
11242         int i;
11243
11244         for (i = 0; i < RTE_MTR_DROPPED; i++) {
11245                 if (dt->policer_rules[i]) {
11246                         claim_zero(mlx5_flow_os_destroy_flow
11247                                    (dt->policer_rules[i]));
11248                         dt->policer_rules[i] = NULL;
11249                 }
11250         }
11251         if (dt->jump_actn) {
11252                 claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn));
11253                 dt->jump_actn = NULL;
11254         }
11255 }
11256
11257 /**
11258  * Destroy policer rules.
11259  *
11260  * @param[in] dev
11261  *   Pointer to Ethernet device.
11262  * @param[in] fm
11263  *   Pointer to flow meter structure.
11264  * @param[in] attr
11265  *   Pointer to flow attributes.
11266  *
11267  * @return
11268  *   Always 0.
11269  */
11270 static int
11271 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
11272                               const struct mlx5_flow_meter *fm,
11273                               const struct rte_flow_attr *attr)
11274 {
11275         struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
11276
11277         if (!mtb)
11278                 return 0;
11279         if (attr->egress)
11280                 flow_dv_destroy_domain_policer_rule(&mtb->egress);
11281         if (attr->ingress)
11282                 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
11283         if (attr->transfer)
11284                 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
11285         return 0;
11286 }
11287
11288 /**
11289  * Create specify domain meter policer rule.
11290  *
11291  * @param[in] fm
11292  *   Pointer to flow meter structure.
11293  * @param[in] mtb
11294  *   Pointer to DV meter table set.
11295  * @param[in] mtr_reg_c
11296  *   Color match REG_C.
11297  *
11298  * @return
11299  *   0 on success, -1 otherwise.
11300  */
11301 static int
11302 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
11303                                     struct mlx5_meter_domain_info *dtb,
11304                                     uint8_t mtr_reg_c)
11305 {
11306         struct mlx5_flow_dv_match_params matcher = {
11307                 .size = sizeof(matcher.buf),
11308         };
11309         struct mlx5_flow_dv_match_params value = {
11310                 .size = sizeof(value.buf),
11311         };
11312         struct mlx5_meter_domains_infos *mtb = fm->mfts;
11313         void *actions[METER_ACTIONS];
11314         int i;
11315         int ret = 0;
11316
11317         /* Create jump action. */
11318         if (!dtb->jump_actn)
11319                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
11320                                 (dtb->sfx_tbl->obj, &dtb->jump_actn);
11321         if (ret) {
11322                 DRV_LOG(ERR, "Failed to create policer jump action.");
11323                 goto error;
11324         }
11325         for (i = 0; i < RTE_MTR_DROPPED; i++) {
11326                 int j = 0;
11327
11328                 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
11329                                        rte_col_2_mlx5_col(i), UINT8_MAX);
11330                 if (mtb->count_actns[i])
11331                         actions[j++] = mtb->count_actns[i];
11332                 if (fm->action[i] == MTR_POLICER_ACTION_DROP)
11333                         actions[j++] = mtb->drop_actn;
11334                 else
11335                         actions[j++] = dtb->jump_actn;
11336                 ret = mlx5_flow_os_create_flow(dtb->color_matcher,
11337                                                (void *)&value, j, actions,
11338                                                &dtb->policer_rules[i]);
11339                 if (ret) {
11340                         DRV_LOG(ERR, "Failed to create policer rule.");
11341                         goto error;
11342                 }
11343         }
11344         return 0;
11345 error:
11346         rte_errno = errno;
11347         return -1;
11348 }
11349
11350 /**
11351  * Create policer rules.
11352  *
11353  * @param[in] dev
11354  *   Pointer to Ethernet device.
11355  * @param[in] fm
11356  *   Pointer to flow meter structure.
11357  * @param[in] attr
11358  *   Pointer to flow attributes.
11359  *
11360  * @return
11361  *   0 on success, -1 otherwise.
11362  */
11363 static int
11364 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
11365                              struct mlx5_flow_meter *fm,
11366                              const struct rte_flow_attr *attr)
11367 {
11368         struct mlx5_priv *priv = dev->data->dev_private;
11369         struct mlx5_meter_domains_infos *mtb = fm->mfts;
11370         int ret;
11371
11372         if (attr->egress) {
11373                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
11374                                                 priv->mtr_color_reg);
11375                 if (ret) {
11376                         DRV_LOG(ERR, "Failed to create egress policer.");
11377                         goto error;
11378                 }
11379         }
11380         if (attr->ingress) {
11381                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
11382                                                 priv->mtr_color_reg);
11383                 if (ret) {
11384                         DRV_LOG(ERR, "Failed to create ingress policer.");
11385                         goto error;
11386                 }
11387         }
11388         if (attr->transfer) {
11389                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
11390                                                 priv->mtr_color_reg);
11391                 if (ret) {
11392                         DRV_LOG(ERR, "Failed to create transfer policer.");
11393                         goto error;
11394                 }
11395         }
11396         return 0;
11397 error:
11398         flow_dv_destroy_policer_rules(dev, fm, attr);
11399         return -1;
11400 }
11401
11402 /**
11403  * Query a devx counter.
11404  *
11405  * @param[in] dev
11406  *   Pointer to the Ethernet device structure.
11407  * @param[in] cnt
11408  *   Index to the flow counter.
11409  * @param[in] clear
11410  *   Set to clear the counter statistics.
11411  * @param[out] pkts
11412  *   The statistics value of packets.
11413  * @param[out] bytes
11414  *   The statistics value of bytes.
11415  *
11416  * @return
11417  *   0 on success, otherwise return -1.
11418  */
11419 static int
11420 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
11421                       uint64_t *pkts, uint64_t *bytes)
11422 {
11423         struct mlx5_priv *priv = dev->data->dev_private;
11424         struct mlx5_flow_counter *cnt;
11425         uint64_t inn_pkts, inn_bytes;
11426         int ret;
11427
11428         if (!priv->config.devx)
11429                 return -1;
11430
11431         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
11432         if (ret)
11433                 return -1;
11434         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
11435         *pkts = inn_pkts - cnt->hits;
11436         *bytes = inn_bytes - cnt->bytes;
11437         if (clear) {
11438                 cnt->hits = inn_pkts;
11439                 cnt->bytes = inn_bytes;
11440         }
11441         return 0;
11442 }
11443
11444 /**
11445  * Get aged-out flows.
11446  *
11447  * @param[in] dev
11448  *   Pointer to the Ethernet device structure.
11449  * @param[in] context
11450  *   The address of an array of pointers to the aged-out flows contexts.
11451  * @param[in] nb_contexts
11452  *   The length of context array pointers.
11453  * @param[out] error
11454  *   Perform verbose error reporting if not NULL. Initialized in case of
11455  *   error only.
11456  *
11457  * @return
11458  *   how many contexts get in success, otherwise negative errno value.
11459  *   if nb_contexts is 0, return the amount of all aged contexts.
11460  *   if nb_contexts is not 0 , return the amount of aged flows reported
11461  *   in the context array.
11462  * @note: only stub for now
11463  */
11464 static int
11465 flow_get_aged_flows(struct rte_eth_dev *dev,
11466                     void **context,
11467                     uint32_t nb_contexts,
11468                     struct rte_flow_error *error)
11469 {
11470         struct mlx5_priv *priv = dev->data->dev_private;
11471         struct mlx5_age_info *age_info;
11472         struct mlx5_age_param *age_param;
11473         struct mlx5_flow_counter *counter;
11474         int nb_flows = 0;
11475
11476         if (nb_contexts && !context)
11477                 return rte_flow_error_set(error, EINVAL,
11478                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11479                                           NULL,
11480                                           "Should assign at least one flow or"
11481                                           " context to get if nb_contexts != 0");
11482         age_info = GET_PORT_AGE_INFO(priv);
11483         rte_spinlock_lock(&age_info->aged_sl);
11484         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
11485                 nb_flows++;
11486                 if (nb_contexts) {
11487                         age_param = MLX5_CNT_TO_AGE(counter);
11488                         context[nb_flows - 1] = age_param->context;
11489                         if (!(--nb_contexts))
11490                                 break;
11491                 }
11492         }
11493         rte_spinlock_unlock(&age_info->aged_sl);
11494         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
11495         return nb_flows;
11496 }
11497
11498 /*
11499  * Mutex-protected thunk to lock-free  __flow_dv_translate().
11500  */
11501 static int
11502 flow_dv_translate(struct rte_eth_dev *dev,
11503                   struct mlx5_flow *dev_flow,
11504                   const struct rte_flow_attr *attr,
11505                   const struct rte_flow_item items[],
11506                   const struct rte_flow_action actions[],
11507                   struct rte_flow_error *error)
11508 {
11509         int ret;
11510
11511         flow_dv_shared_lock(dev);
11512         ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
11513         flow_dv_shared_unlock(dev);
11514         return ret;
11515 }
11516
11517 /*
11518  * Mutex-protected thunk to lock-free  __flow_dv_apply().
11519  */
11520 static int
11521 flow_dv_apply(struct rte_eth_dev *dev,
11522               struct rte_flow *flow,
11523               struct rte_flow_error *error)
11524 {
11525         int ret;
11526
11527         flow_dv_shared_lock(dev);
11528         ret = __flow_dv_apply(dev, flow, error);
11529         flow_dv_shared_unlock(dev);
11530         return ret;
11531 }
11532
11533 /*
11534  * Mutex-protected thunk to lock-free __flow_dv_remove().
11535  */
11536 static void
11537 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
11538 {
11539         flow_dv_shared_lock(dev);
11540         __flow_dv_remove(dev, flow);
11541         flow_dv_shared_unlock(dev);
11542 }
11543
11544 /*
11545  * Mutex-protected thunk to lock-free __flow_dv_destroy().
11546  */
11547 static void
11548 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
11549 {
11550         flow_dv_shared_lock(dev);
11551         __flow_dv_destroy(dev, flow);
11552         flow_dv_shared_unlock(dev);
11553 }
11554
11555 /*
11556  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
11557  */
11558 static uint32_t
11559 flow_dv_counter_allocate(struct rte_eth_dev *dev)
11560 {
11561         uint32_t cnt;
11562
11563         flow_dv_shared_lock(dev);
11564         cnt = flow_dv_counter_alloc(dev, 0, 0, 1, 0);
11565         flow_dv_shared_unlock(dev);
11566         return cnt;
11567 }
11568
11569 /*
11570  * Mutex-protected thunk to lock-free flow_dv_counter_release().
11571  */
11572 static void
11573 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
11574 {
11575         flow_dv_shared_lock(dev);
11576         flow_dv_counter_release(dev, cnt);
11577         flow_dv_shared_unlock(dev);
11578 }
11579
11580 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
11581         .validate = flow_dv_validate,
11582         .prepare = flow_dv_prepare,
11583         .translate = flow_dv_translate,
11584         .apply = flow_dv_apply,
11585         .remove = flow_dv_remove,
11586         .destroy = flow_dv_destroy,
11587         .query = flow_dv_query,
11588         .create_mtr_tbls = flow_dv_create_mtr_tbl,
11589         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
11590         .create_policer_rules = flow_dv_create_policer_rules,
11591         .destroy_policer_rules = flow_dv_destroy_policer_rules,
11592         .counter_alloc = flow_dv_counter_allocate,
11593         .counter_free = flow_dv_counter_free,
11594         .counter_query = flow_dv_counter_query,
11595         .get_aged_flows = flow_get_aged_flows,
11596 };
11597
11598 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
11599