drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <rte_ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_ip.h>
  19 #include <rte_gre.h>
  20 #include <rte_vxlan.h>
  21 #include <rte_gtp.h>
  22 #include <rte_eal_paging.h>
  23 #include <rte_mpls.h>
  24
  25 #include <mlx5_glue.h>
  26 #include <mlx5_devx_cmds.h>
  27 #include <mlx5_prm.h>
  28 #include <mlx5_malloc.h>
  29
  30 #include "mlx5_defs.h"
  31 #include "mlx5.h"
  32 #include "mlx5_common_os.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_flow_os.h"
  35 #include "mlx5_rxtx.h"
  36
  37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  38
  39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  41 #endif
  42
  43 #ifndef HAVE_MLX5DV_DR_ESWITCH
  44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  46 #endif
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR
  50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  51 #endif
  52
  53 /* VLAN header definitions */
  54 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  55 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  56 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  57 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  58 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  59
  60 union flow_dv_attr {
  61         struct {
  62                 uint32_t valid:1;
  63                 uint32_t ipv4:1;
  64                 uint32_t ipv6:1;
  65                 uint32_t tcp:1;
  66                 uint32_t udp:1;
  67                 uint32_t reserved:27;
  68         };
  69         uint32_t attr;
  70 };
  71
  72 static int
  73 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
  74                              struct mlx5_flow_tbl_resource *tbl);
  75
  76 static int
  77 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev);
  78
  79 static int
  80 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  81                                       uint32_t encap_decap_idx);
  82
  83 static int
  84 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  85                                         uint32_t port_id);
  86
  87 /**
  88  * Initialize flow attributes structure according to flow items' types.
  89  *
  90  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  91  * mode. For tunnel mode, the items to be modified are the outermost ones.
  92  *
  93  * @param[in] item
  94  *   Pointer to item specification.
  95  * @param[out] attr
  96  *   Pointer to flow attributes structure.
  97  * @param[in] dev_flow
  98  *   Pointer to the sub flow.
  99  * @param[in] tunnel_decap
 100  *   Whether action is after tunnel decapsulation.
 101  */
 102 static void
 103 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 104                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 105 {
 106         uint64_t layers = dev_flow->handle->layers;
 107
 108         /*
 109          * If layers is already initialized, it means this dev_flow is the
 110          * suffix flow, the layers flags is set by the prefix flow. Need to
 111          * use the layer flags from prefix flow as the suffix flow may not
 112          * have the user defined items as the flow is split.
 113          */
 114         if (layers) {
 115                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 116                         attr->ipv4 = 1;
 117                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 118                         attr->ipv6 = 1;
 119                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 120                         attr->tcp = 1;
 121                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 122                         attr->udp = 1;
 123                 attr->valid = 1;
 124                 return;
 125         }
 126         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 127                 uint8_t next_protocol = 0xff;
 128                 switch (item->type) {
 129                 case RTE_FLOW_ITEM_TYPE_GRE:
 130                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 131                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 132                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 133                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 134                 case RTE_FLOW_ITEM_TYPE_MPLS:
 135                         if (tunnel_decap)
 136                                 attr->attr = 0;
 137                         break;
 138                 case RTE_FLOW_ITEM_TYPE_IPV4:
 139                         if (!attr->ipv6)
 140                                 attr->ipv4 = 1;
 141                         if (item->mask != NULL &&
 142                             ((const struct rte_flow_item_ipv4 *)
 143                             item->mask)->hdr.next_proto_id)
 144                                 next_protocol =
 145                                     ((const struct rte_flow_item_ipv4 *)
 146                                       (item->spec))->hdr.next_proto_id &
 147                                     ((const struct rte_flow_item_ipv4 *)
 148                                       (item->mask))->hdr.next_proto_id;
 149                         if ((next_protocol == IPPROTO_IPIP ||
 150                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 151                                 attr->attr = 0;
 152                         break;
 153                 case RTE_FLOW_ITEM_TYPE_IPV6:
 154                         if (!attr->ipv4)
 155                                 attr->ipv6 = 1;
 156                         if (item->mask != NULL &&
 157                             ((const struct rte_flow_item_ipv6 *)
 158                             item->mask)->hdr.proto)
 159                                 next_protocol =
 160                                     ((const struct rte_flow_item_ipv6 *)
 161                                       (item->spec))->hdr.proto &
 162                                     ((const struct rte_flow_item_ipv6 *)
 163                                       (item->mask))->hdr.proto;
 164                         if ((next_protocol == IPPROTO_IPIP ||
 165                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 166                                 attr->attr = 0;
 167                         break;
 168                 case RTE_FLOW_ITEM_TYPE_UDP:
 169                         if (!attr->tcp)
 170                                 attr->udp = 1;
 171                         break;
 172                 case RTE_FLOW_ITEM_TYPE_TCP:
 173                         if (!attr->udp)
 174                                 attr->tcp = 1;
 175                         break;
 176                 default:
 177                         break;
 178                 }
 179         }
 180         attr->valid = 1;
 181 }
 182
 183 /**
 184  * Convert rte_mtr_color to mlx5 color.
 185  *
 186  * @param[in] rcol
 187  *   rte_mtr_color.
 188  *
 189  * @return
 190  *   mlx5 color.
 191  */
 192 static int
 193 rte_col_2_mlx5_col(enum rte_color rcol)
 194 {
 195         switch (rcol) {
 196         case RTE_COLOR_GREEN:
 197                 return MLX5_FLOW_COLOR_GREEN;
 198         case RTE_COLOR_YELLOW:
 199                 return MLX5_FLOW_COLOR_YELLOW;
 200         case RTE_COLOR_RED:
 201                 return MLX5_FLOW_COLOR_RED;
 202         default:
 203                 break;
 204         }
 205         return MLX5_FLOW_COLOR_UNDEFINED;
 206 }
 207
 208 struct field_modify_info {
 209         uint32_t size; /* Size of field in protocol header, in bytes. */
 210         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 211         enum mlx5_modification_field id;
 212 };
 213
 214 struct field_modify_info modify_eth[] = {
 215         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 216         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 217         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 218         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 219         {0, 0, 0},
 220 };
 221
 222 struct field_modify_info modify_vlan_out_first_vid[] = {
 223         /* Size in bits !!! */
 224         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 225         {0, 0, 0},
 226 };
 227
 228 struct field_modify_info modify_ipv4[] = {
 229         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 230         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 231         {4, 12, MLX5_MODI_OUT_SIPV4},
 232         {4, 16, MLX5_MODI_OUT_DIPV4},
 233         {0, 0, 0},
 234 };
 235
 236 struct field_modify_info modify_ipv6[] = {
 237         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 238         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 239         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 240         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 241         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 242         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 243         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 244         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 245         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 246         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 247         {0, 0, 0},
 248 };
 249
 250 struct field_modify_info modify_udp[] = {
 251         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 252         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 253         {0, 0, 0},
 254 };
 255
 256 struct field_modify_info modify_tcp[] = {
 257         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 258         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 259         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 260         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 261         {0, 0, 0},
 262 };
 263
 264 static void
 265 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 266                           uint8_t next_protocol, uint64_t *item_flags,
 267                           int *tunnel)
 268 {
 269         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 270                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 271         if (next_protocol == IPPROTO_IPIP) {
 272                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 273                 *tunnel = 1;
 274         }
 275         if (next_protocol == IPPROTO_IPV6) {
 276                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 277                 *tunnel = 1;
 278         }
 279 }
 280
 281 /**
 282  * Acquire the synchronizing object to protect multithreaded access
 283  * to shared dv context. Lock occurs only if context is actually
 284  * shared, i.e. we have multiport IB device and representors are
 285  * created.
 286  *
 287  * @param[in] dev
 288  *   Pointer to the rte_eth_dev structure.
 289  */
 290 static void
 291 flow_dv_shared_lock(struct rte_eth_dev *dev)
 292 {
 293         struct mlx5_priv *priv = dev->data->dev_private;
 294         struct mlx5_dev_ctx_shared *sh = priv->sh;
 295
 296         if (sh->refcnt > 1) {
 297                 int ret;
 298
 299                 ret = pthread_mutex_lock(&sh->dv_mutex);
 300                 MLX5_ASSERT(!ret);
 301                 (void)ret;
 302         }
 303 }
 304
 305 static void
 306 flow_dv_shared_unlock(struct rte_eth_dev *dev)
 307 {
 308         struct mlx5_priv *priv = dev->data->dev_private;
 309         struct mlx5_dev_ctx_shared *sh = priv->sh;
 310
 311         if (sh->refcnt > 1) {
 312                 int ret;
 313
 314                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 315                 MLX5_ASSERT(!ret);
 316                 (void)ret;
 317         }
 318 }
 319
 320 /* Update VLAN's VID/PCP based on input rte_flow_action.
 321  *
 322  * @param[in] action
 323  *   Pointer to struct rte_flow_action.
 324  * @param[out] vlan
 325  *   Pointer to struct rte_vlan_hdr.
 326  */
 327 static void
 328 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 329                          struct rte_vlan_hdr *vlan)
 330 {
 331         uint16_t vlan_tci;
 332         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 333                 vlan_tci =
 334                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 335                                                action->conf)->vlan_pcp;
 336                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 337                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 338                 vlan->vlan_tci |= vlan_tci;
 339         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 340                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 341                 vlan->vlan_tci |= rte_be_to_cpu_16
 342                     (((const struct rte_flow_action_of_set_vlan_vid *)
 343                                              action->conf)->vlan_vid);
 344         }
 345 }
 346
 347 /**
 348  * Fetch 1, 2, 3 or 4 byte field from the byte array
 349  * and return as unsigned integer in host-endian format.
 350  *
 351  * @param[in] data
 352  *   Pointer to data array.
 353  * @param[in] size
 354  *   Size of field to extract.
 355  *
 356  * @return
 357  *   converted field in host endian format.
 358  */
 359 static inline uint32_t
 360 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 361 {
 362         uint32_t ret;
 363
 364         switch (size) {
 365         case 1:
 366                 ret = *data;
 367                 break;
 368         case 2:
 369                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 370                 break;
 371         case 3:
 372                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 373                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 374                 break;
 375         case 4:
 376                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 377                 break;
 378         default:
 379                 MLX5_ASSERT(false);
 380                 ret = 0;
 381                 break;
 382         }
 383         return ret;
 384 }
 385
 386 /**
 387  * Convert modify-header action to DV specification.
 388  *
 389  * Data length of each action is determined by provided field description
 390  * and the item mask. Data bit offset and width of each action is determined
 391  * by provided item mask.
 392  *
 393  * @param[in] item
 394  *   Pointer to item specification.
 395  * @param[in] field
 396  *   Pointer to field modification information.
 397  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 398  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 399  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 400  * @param[in] dcopy
 401  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 402  *   Negative offset value sets the same offset as source offset.
 403  *   size field is ignored, value is taken from source field.
 404  * @param[in,out] resource
 405  *   Pointer to the modify-header resource.
 406  * @param[in] type
 407  *   Type of modification.
 408  * @param[out] error
 409  *   Pointer to the error structure.
 410  *
 411  * @return
 412  *   0 on success, a negative errno value otherwise and rte_errno is set.
 413  */
 414 static int
 415 flow_dv_convert_modify_action(struct rte_flow_item *item,
 416                               struct field_modify_info *field,
 417                               struct field_modify_info *dcopy,
 418                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 419                               uint32_t type, struct rte_flow_error *error)
 420 {
 421         uint32_t i = resource->actions_num;
 422         struct mlx5_modification_cmd *actions = resource->actions;
 423
 424         /*
 425          * The item and mask are provided in big-endian format.
 426          * The fields should be presented as in big-endian format either.
 427          * Mask must be always present, it defines the actual field width.
 428          */
 429         MLX5_ASSERT(item->mask);
 430         MLX5_ASSERT(field->size);
 431         do {
 432                 unsigned int size_b;
 433                 unsigned int off_b;
 434                 uint32_t mask;
 435                 uint32_t data;
 436
 437                 if (i >= MLX5_MAX_MODIFY_NUM)
 438                         return rte_flow_error_set(error, EINVAL,
 439                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 440                                  "too many items to modify");
 441                 /* Fetch variable byte size mask from the array. */
 442                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 443                                            field->offset, field->size);
 444                 if (!mask) {
 445                         ++field;
 446                         continue;
 447                 }
 448                 /* Deduce actual data width in bits from mask value. */
 449                 off_b = rte_bsf32(mask);
 450                 size_b = sizeof(uint32_t) * CHAR_BIT -
 451                          off_b - __builtin_clz(mask);
 452                 MLX5_ASSERT(size_b);
 453                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 454                 actions[i] = (struct mlx5_modification_cmd) {
 455                         .action_type = type,
 456                         .field = field->id,
 457                         .offset = off_b,
 458                         .length = size_b,
 459                 };
 460                 /* Convert entire record to expected big-endian format. */
 461                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 462                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 463                         MLX5_ASSERT(dcopy);
 464                         actions[i].dst_field = dcopy->id;
 465                         actions[i].dst_offset =
 466                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 467                         /* Convert entire record to big-endian format. */
 468                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 469                 } else {
 470                         MLX5_ASSERT(item->spec);
 471                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 472                                                    field->offset, field->size);
 473                         /* Shift out the trailing masked bits from data. */
 474                         data = (data & mask) >> off_b;
 475                         actions[i].data1 = rte_cpu_to_be_32(data);
 476                 }
 477                 ++i;
 478                 ++field;
 479         } while (field->size);
 480         if (resource->actions_num == i)
 481                 return rte_flow_error_set(error, EINVAL,
 482                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 483                                           "invalid modification flow item");
 484         resource->actions_num = i;
 485         return 0;
 486 }
 487
 488 /**
 489  * Convert modify-header set IPv4 address action to DV specification.
 490  *
 491  * @param[in,out] resource
 492  *   Pointer to the modify-header resource.
 493  * @param[in] action
 494  *   Pointer to action specification.
 495  * @param[out] error
 496  *   Pointer to the error structure.
 497  *
 498  * @return
 499  *   0 on success, a negative errno value otherwise and rte_errno is set.
 500  */
 501 static int
 502 flow_dv_convert_action_modify_ipv4
 503                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 504                          const struct rte_flow_action *action,
 505                          struct rte_flow_error *error)
 506 {
 507         const struct rte_flow_action_set_ipv4 *conf =
 508                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 509         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 510         struct rte_flow_item_ipv4 ipv4;
 511         struct rte_flow_item_ipv4 ipv4_mask;
 512
 513         memset(&ipv4, 0, sizeof(ipv4));
 514         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 515         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 516                 ipv4.hdr.src_addr = conf->ipv4_addr;
 517                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 518         } else {
 519                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 520                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 521         }
 522         item.spec = &ipv4;
 523         item.mask = &ipv4_mask;
 524         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 525                                              MLX5_MODIFICATION_TYPE_SET, error);
 526 }
 527
 528 /**
 529  * Convert modify-header set IPv6 address action to DV specification.
 530  *
 531  * @param[in,out] resource
 532  *   Pointer to the modify-header resource.
 533  * @param[in] action
 534  *   Pointer to action specification.
 535  * @param[out] error
 536  *   Pointer to the error structure.
 537  *
 538  * @return
 539  *   0 on success, a negative errno value otherwise and rte_errno is set.
 540  */
 541 static int
 542 flow_dv_convert_action_modify_ipv6
 543                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 544                          const struct rte_flow_action *action,
 545                          struct rte_flow_error *error)
 546 {
 547         const struct rte_flow_action_set_ipv6 *conf =
 548                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 549         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 550         struct rte_flow_item_ipv6 ipv6;
 551         struct rte_flow_item_ipv6 ipv6_mask;
 552
 553         memset(&ipv6, 0, sizeof(ipv6));
 554         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 555         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 556                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 557                        sizeof(ipv6.hdr.src_addr));
 558                 memcpy(&ipv6_mask.hdr.src_addr,
 559                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 560                        sizeof(ipv6.hdr.src_addr));
 561         } else {
 562                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 563                        sizeof(ipv6.hdr.dst_addr));
 564                 memcpy(&ipv6_mask.hdr.dst_addr,
 565                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 566                        sizeof(ipv6.hdr.dst_addr));
 567         }
 568         item.spec = &ipv6;
 569         item.mask = &ipv6_mask;
 570         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 571                                              MLX5_MODIFICATION_TYPE_SET, error);
 572 }
 573
 574 /**
 575  * Convert modify-header set MAC address action to DV specification.
 576  *
 577  * @param[in,out] resource
 578  *   Pointer to the modify-header resource.
 579  * @param[in] action
 580  *   Pointer to action specification.
 581  * @param[out] error
 582  *   Pointer to the error structure.
 583  *
 584  * @return
 585  *   0 on success, a negative errno value otherwise and rte_errno is set.
 586  */
 587 static int
 588 flow_dv_convert_action_modify_mac
 589                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 590                          const struct rte_flow_action *action,
 591                          struct rte_flow_error *error)
 592 {
 593         const struct rte_flow_action_set_mac *conf =
 594                 (const struct rte_flow_action_set_mac *)(action->conf);
 595         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 596         struct rte_flow_item_eth eth;
 597         struct rte_flow_item_eth eth_mask;
 598
 599         memset(&eth, 0, sizeof(eth));
 600         memset(&eth_mask, 0, sizeof(eth_mask));
 601         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 602                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 603                        sizeof(eth.src.addr_bytes));
 604                 memcpy(&eth_mask.src.addr_bytes,
 605                        &rte_flow_item_eth_mask.src.addr_bytes,
 606                        sizeof(eth_mask.src.addr_bytes));
 607         } else {
 608                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 609                        sizeof(eth.dst.addr_bytes));
 610                 memcpy(&eth_mask.dst.addr_bytes,
 611                        &rte_flow_item_eth_mask.dst.addr_bytes,
 612                        sizeof(eth_mask.dst.addr_bytes));
 613         }
 614         item.spec = &eth;
 615         item.mask = &eth_mask;
 616         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 617                                              MLX5_MODIFICATION_TYPE_SET, error);
 618 }
 619
 620 /**
 621  * Convert modify-header set VLAN VID action to DV specification.
 622  *
 623  * @param[in,out] resource
 624  *   Pointer to the modify-header resource.
 625  * @param[in] action
 626  *   Pointer to action specification.
 627  * @param[out] error
 628  *   Pointer to the error structure.
 629  *
 630  * @return
 631  *   0 on success, a negative errno value otherwise and rte_errno is set.
 632  */
 633 static int
 634 flow_dv_convert_action_modify_vlan_vid
 635                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 636                          const struct rte_flow_action *action,
 637                          struct rte_flow_error *error)
 638 {
 639         const struct rte_flow_action_of_set_vlan_vid *conf =
 640                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 641         int i = resource->actions_num;
 642         struct mlx5_modification_cmd *actions = resource->actions;
 643         struct field_modify_info *field = modify_vlan_out_first_vid;
 644
 645         if (i >= MLX5_MAX_MODIFY_NUM)
 646                 return rte_flow_error_set(error, EINVAL,
 647                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 648                          "too many items to modify");
 649         actions[i] = (struct mlx5_modification_cmd) {
 650                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 651                 .field = field->id,
 652                 .length = field->size,
 653                 .offset = field->offset,
 654         };
 655         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 656         actions[i].data1 = conf->vlan_vid;
 657         actions[i].data1 = actions[i].data1 << 16;
 658         resource->actions_num = ++i;
 659         return 0;
 660 }
 661
 662 /**
 663  * Convert modify-header set TP action to DV specification.
 664  *
 665  * @param[in,out] resource
 666  *   Pointer to the modify-header resource.
 667  * @param[in] action
 668  *   Pointer to action specification.
 669  * @param[in] items
 670  *   Pointer to rte_flow_item objects list.
 671  * @param[in] attr
 672  *   Pointer to flow attributes structure.
 673  * @param[in] dev_flow
 674  *   Pointer to the sub flow.
 675  * @param[in] tunnel_decap
 676  *   Whether action is after tunnel decapsulation.
 677  * @param[out] error
 678  *   Pointer to the error structure.
 679  *
 680  * @return
 681  *   0 on success, a negative errno value otherwise and rte_errno is set.
 682  */
 683 static int
 684 flow_dv_convert_action_modify_tp
 685                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 686                          const struct rte_flow_action *action,
 687                          const struct rte_flow_item *items,
 688                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 689                          bool tunnel_decap, struct rte_flow_error *error)
 690 {
 691         const struct rte_flow_action_set_tp *conf =
 692                 (const struct rte_flow_action_set_tp *)(action->conf);
 693         struct rte_flow_item item;
 694         struct rte_flow_item_udp udp;
 695         struct rte_flow_item_udp udp_mask;
 696         struct rte_flow_item_tcp tcp;
 697         struct rte_flow_item_tcp tcp_mask;
 698         struct field_modify_info *field;
 699
 700         if (!attr->valid)
 701                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 702         if (attr->udp) {
 703                 memset(&udp, 0, sizeof(udp));
 704                 memset(&udp_mask, 0, sizeof(udp_mask));
 705                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 706                         udp.hdr.src_port = conf->port;
 707                         udp_mask.hdr.src_port =
 708                                         rte_flow_item_udp_mask.hdr.src_port;
 709                 } else {
 710                         udp.hdr.dst_port = conf->port;
 711                         udp_mask.hdr.dst_port =
 712                                         rte_flow_item_udp_mask.hdr.dst_port;
 713                 }
 714                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 715                 item.spec = &udp;
 716                 item.mask = &udp_mask;
 717                 field = modify_udp;
 718         } else {
 719                 MLX5_ASSERT(attr->tcp);
 720                 memset(&tcp, 0, sizeof(tcp));
 721                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 722                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 723                         tcp.hdr.src_port = conf->port;
 724                         tcp_mask.hdr.src_port =
 725                                         rte_flow_item_tcp_mask.hdr.src_port;
 726                 } else {
 727                         tcp.hdr.dst_port = conf->port;
 728                         tcp_mask.hdr.dst_port =
 729                                         rte_flow_item_tcp_mask.hdr.dst_port;
 730                 }
 731                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 732                 item.spec = &tcp;
 733                 item.mask = &tcp_mask;
 734                 field = modify_tcp;
 735         }
 736         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 737                                              MLX5_MODIFICATION_TYPE_SET, error);
 738 }
 739
 740 /**
 741  * Convert modify-header set TTL action to DV specification.
 742  *
 743  * @param[in,out] resource
 744  *   Pointer to the modify-header resource.
 745  * @param[in] action
 746  *   Pointer to action specification.
 747  * @param[in] items
 748  *   Pointer to rte_flow_item objects list.
 749  * @param[in] attr
 750  *   Pointer to flow attributes structure.
 751  * @param[in] dev_flow
 752  *   Pointer to the sub flow.
 753  * @param[in] tunnel_decap
 754  *   Whether action is after tunnel decapsulation.
 755  * @param[out] error
 756  *   Pointer to the error structure.
 757  *
 758  * @return
 759  *   0 on success, a negative errno value otherwise and rte_errno is set.
 760  */
 761 static int
 762 flow_dv_convert_action_modify_ttl
 763                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 764                          const struct rte_flow_action *action,
 765                          const struct rte_flow_item *items,
 766                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 767                          bool tunnel_decap, struct rte_flow_error *error)
 768 {
 769         const struct rte_flow_action_set_ttl *conf =
 770                 (const struct rte_flow_action_set_ttl *)(action->conf);
 771         struct rte_flow_item item;
 772         struct rte_flow_item_ipv4 ipv4;
 773         struct rte_flow_item_ipv4 ipv4_mask;
 774         struct rte_flow_item_ipv6 ipv6;
 775         struct rte_flow_item_ipv6 ipv6_mask;
 776         struct field_modify_info *field;
 777
 778         if (!attr->valid)
 779                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 780         if (attr->ipv4) {
 781                 memset(&ipv4, 0, sizeof(ipv4));
 782                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 783                 ipv4.hdr.time_to_live = conf->ttl_value;
 784                 ipv4_mask.hdr.time_to_live = 0xFF;
 785                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 786                 item.spec = &ipv4;
 787                 item.mask = &ipv4_mask;
 788                 field = modify_ipv4;
 789         } else {
 790                 MLX5_ASSERT(attr->ipv6);
 791                 memset(&ipv6, 0, sizeof(ipv6));
 792                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 793                 ipv6.hdr.hop_limits = conf->ttl_value;
 794                 ipv6_mask.hdr.hop_limits = 0xFF;
 795                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 796                 item.spec = &ipv6;
 797                 item.mask = &ipv6_mask;
 798                 field = modify_ipv6;
 799         }
 800         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 801                                              MLX5_MODIFICATION_TYPE_SET, error);
 802 }
 803
 804 /**
 805  * Convert modify-header decrement TTL action to DV specification.
 806  *
 807  * @param[in,out] resource
 808  *   Pointer to the modify-header resource.
 809  * @param[in] action
 810  *   Pointer to action specification.
 811  * @param[in] items
 812  *   Pointer to rte_flow_item objects list.
 813  * @param[in] attr
 814  *   Pointer to flow attributes structure.
 815  * @param[in] dev_flow
 816  *   Pointer to the sub flow.
 817  * @param[in] tunnel_decap
 818  *   Whether action is after tunnel decapsulation.
 819  * @param[out] error
 820  *   Pointer to the error structure.
 821  *
 822  * @return
 823  *   0 on success, a negative errno value otherwise and rte_errno is set.
 824  */
 825 static int
 826 flow_dv_convert_action_modify_dec_ttl
 827                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 828                          const struct rte_flow_item *items,
 829                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 830                          bool tunnel_decap, struct rte_flow_error *error)
 831 {
 832         struct rte_flow_item item;
 833         struct rte_flow_item_ipv4 ipv4;
 834         struct rte_flow_item_ipv4 ipv4_mask;
 835         struct rte_flow_item_ipv6 ipv6;
 836         struct rte_flow_item_ipv6 ipv6_mask;
 837         struct field_modify_info *field;
 838
 839         if (!attr->valid)
 840                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 841         if (attr->ipv4) {
 842                 memset(&ipv4, 0, sizeof(ipv4));
 843                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 844                 ipv4.hdr.time_to_live = 0xFF;
 845                 ipv4_mask.hdr.time_to_live = 0xFF;
 846                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 847                 item.spec = &ipv4;
 848                 item.mask = &ipv4_mask;
 849                 field = modify_ipv4;
 850         } else {
 851                 MLX5_ASSERT(attr->ipv6);
 852                 memset(&ipv6, 0, sizeof(ipv6));
 853                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 854                 ipv6.hdr.hop_limits = 0xFF;
 855                 ipv6_mask.hdr.hop_limits = 0xFF;
 856                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 857                 item.spec = &ipv6;
 858                 item.mask = &ipv6_mask;
 859                 field = modify_ipv6;
 860         }
 861         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 862                                              MLX5_MODIFICATION_TYPE_ADD, error);
 863 }
 864
 865 /**
 866  * Convert modify-header increment/decrement TCP Sequence number
 867  * to DV specification.
 868  *
 869  * @param[in,out] resource
 870  *   Pointer to the modify-header resource.
 871  * @param[in] action
 872  *   Pointer to action specification.
 873  * @param[out] error
 874  *   Pointer to the error structure.
 875  *
 876  * @return
 877  *   0 on success, a negative errno value otherwise and rte_errno is set.
 878  */
 879 static int
 880 flow_dv_convert_action_modify_tcp_seq
 881                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 882                          const struct rte_flow_action *action,
 883                          struct rte_flow_error *error)
 884 {
 885         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 886         uint64_t value = rte_be_to_cpu_32(*conf);
 887         struct rte_flow_item item;
 888         struct rte_flow_item_tcp tcp;
 889         struct rte_flow_item_tcp tcp_mask;
 890
 891         memset(&tcp, 0, sizeof(tcp));
 892         memset(&tcp_mask, 0, sizeof(tcp_mask));
 893         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 894                 /*
 895                  * The HW has no decrement operation, only increment operation.
 896                  * To simulate decrement X from Y using increment operation
 897                  * we need to add UINT32_MAX X times to Y.
 898                  * Each adding of UINT32_MAX decrements Y by 1.
 899                  */
 900                 value *= UINT32_MAX;
 901         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 902         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 903         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 904         item.spec = &tcp;
 905         item.mask = &tcp_mask;
 906         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 907                                              MLX5_MODIFICATION_TYPE_ADD, error);
 908 }
 909
 910 /**
 911  * Convert modify-header increment/decrement TCP Acknowledgment number
 912  * to DV specification.
 913  *
 914  * @param[in,out] resource
 915  *   Pointer to the modify-header resource.
 916  * @param[in] action
 917  *   Pointer to action specification.
 918  * @param[out] error
 919  *   Pointer to the error structure.
 920  *
 921  * @return
 922  *   0 on success, a negative errno value otherwise and rte_errno is set.
 923  */
 924 static int
 925 flow_dv_convert_action_modify_tcp_ack
 926                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 927                          const struct rte_flow_action *action,
 928                          struct rte_flow_error *error)
 929 {
 930         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 931         uint64_t value = rte_be_to_cpu_32(*conf);
 932         struct rte_flow_item item;
 933         struct rte_flow_item_tcp tcp;
 934         struct rte_flow_item_tcp tcp_mask;
 935
 936         memset(&tcp, 0, sizeof(tcp));
 937         memset(&tcp_mask, 0, sizeof(tcp_mask));
 938         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 939                 /*
 940                  * The HW has no decrement operation, only increment operation.
 941                  * To simulate decrement X from Y using increment operation
 942                  * we need to add UINT32_MAX X times to Y.
 943                  * Each adding of UINT32_MAX decrements Y by 1.
 944                  */
 945                 value *= UINT32_MAX;
 946         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 947         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 948         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 949         item.spec = &tcp;
 950         item.mask = &tcp_mask;
 951         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 952                                              MLX5_MODIFICATION_TYPE_ADD, error);
 953 }
 954
 955 static enum mlx5_modification_field reg_to_field[] = {
 956         [REG_NON] = MLX5_MODI_OUT_NONE,
 957         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 958         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 959         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 960         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 961         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 962         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 963         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 964         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 965         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 966         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 967 };
 968
 969 /**
 970  * Convert register set to DV specification.
 971  *
 972  * @param[in,out] resource
 973  *   Pointer to the modify-header resource.
 974  * @param[in] action
 975  *   Pointer to action specification.
 976  * @param[out] error
 977  *   Pointer to the error structure.
 978  *
 979  * @return
 980  *   0 on success, a negative errno value otherwise and rte_errno is set.
 981  */
 982 static int
 983 flow_dv_convert_action_set_reg
 984                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 985                          const struct rte_flow_action *action,
 986                          struct rte_flow_error *error)
 987 {
 988         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 989         struct mlx5_modification_cmd *actions = resource->actions;
 990         uint32_t i = resource->actions_num;
 991
 992         if (i >= MLX5_MAX_MODIFY_NUM)
 993                 return rte_flow_error_set(error, EINVAL,
 994                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 995                                           "too many items to modify");
 996         MLX5_ASSERT(conf->id != REG_NON);
 997         MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field));
 998         actions[i] = (struct mlx5_modification_cmd) {
 999                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1000                 .field = reg_to_field[conf->id],
1001         };
1002         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1003         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1004         ++i;
1005         resource->actions_num = i;
1006         return 0;
1007 }
1008
1009 /**
1010  * Convert SET_TAG action to DV specification.
1011  *
1012  * @param[in] dev
1013  *   Pointer to the rte_eth_dev structure.
1014  * @param[in,out] resource
1015  *   Pointer to the modify-header resource.
1016  * @param[in] conf
1017  *   Pointer to action specification.
1018  * @param[out] error
1019  *   Pointer to the error structure.
1020  *
1021  * @return
1022  *   0 on success, a negative errno value otherwise and rte_errno is set.
1023  */
1024 static int
1025 flow_dv_convert_action_set_tag
1026                         (struct rte_eth_dev *dev,
1027                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1028                          const struct rte_flow_action_set_tag *conf,
1029                          struct rte_flow_error *error)
1030 {
1031         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1032         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1033         struct rte_flow_item item = {
1034                 .spec = &data,
1035                 .mask = &mask,
1036         };
1037         struct field_modify_info reg_c_x[] = {
1038                 [1] = {0, 0, 0},
1039         };
1040         enum mlx5_modification_field reg_type;
1041         int ret;
1042
1043         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1044         if (ret < 0)
1045                 return ret;
1046         MLX5_ASSERT(ret != REG_NON);
1047         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1048         reg_type = reg_to_field[ret];
1049         MLX5_ASSERT(reg_type > 0);
1050         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1051         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1052                                              MLX5_MODIFICATION_TYPE_SET, error);
1053 }
1054
1055 /**
1056  * Convert internal COPY_REG action to DV specification.
1057  *
1058  * @param[in] dev
1059  *   Pointer to the rte_eth_dev structure.
1060  * @param[in,out] res
1061  *   Pointer to the modify-header resource.
1062  * @param[in] action
1063  *   Pointer to action specification.
1064  * @param[out] error
1065  *   Pointer to the error structure.
1066  *
1067  * @return
1068  *   0 on success, a negative errno value otherwise and rte_errno is set.
1069  */
1070 static int
1071 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1072                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1073                                  const struct rte_flow_action *action,
1074                                  struct rte_flow_error *error)
1075 {
1076         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1077         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1078         struct rte_flow_item item = {
1079                 .spec = NULL,
1080                 .mask = &mask,
1081         };
1082         struct field_modify_info reg_src[] = {
1083                 {4, 0, reg_to_field[conf->src]},
1084                 {0, 0, 0},
1085         };
1086         struct field_modify_info reg_dst = {
1087                 .offset = 0,
1088                 .id = reg_to_field[conf->dst],
1089         };
1090         /* Adjust reg_c[0] usage according to reported mask. */
1091         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1092                 struct mlx5_priv *priv = dev->data->dev_private;
1093                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1094
1095                 MLX5_ASSERT(reg_c0);
1096                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1097                 if (conf->dst == REG_C_0) {
1098                         /* Copy to reg_c[0], within mask only. */
1099                         reg_dst.offset = rte_bsf32(reg_c0);
1100                         /*
1101                          * Mask is ignoring the enianness, because
1102                          * there is no conversion in datapath.
1103                          */
1104 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1105                         /* Copy from destination lower bits to reg_c[0]. */
1106                         mask = reg_c0 >> reg_dst.offset;
1107 #else
1108                         /* Copy from destination upper bits to reg_c[0]. */
1109                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1110                                           rte_fls_u32(reg_c0));
1111 #endif
1112                 } else {
1113                         mask = rte_cpu_to_be_32(reg_c0);
1114 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1115                         /* Copy from reg_c[0] to destination lower bits. */
1116                         reg_dst.offset = 0;
1117 #else
1118                         /* Copy from reg_c[0] to destination upper bits. */
1119                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1120                                          (rte_fls_u32(reg_c0) -
1121                                           rte_bsf32(reg_c0));
1122 #endif
1123                 }
1124         }
1125         return flow_dv_convert_modify_action(&item,
1126                                              reg_src, &reg_dst, res,
1127                                              MLX5_MODIFICATION_TYPE_COPY,
1128                                              error);
1129 }
1130
1131 /**
1132  * Convert MARK action to DV specification. This routine is used
1133  * in extensive metadata only and requires metadata register to be
1134  * handled. In legacy mode hardware tag resource is engaged.
1135  *
1136  * @param[in] dev
1137  *   Pointer to the rte_eth_dev structure.
1138  * @param[in] conf
1139  *   Pointer to MARK action specification.
1140  * @param[in,out] resource
1141  *   Pointer to the modify-header resource.
1142  * @param[out] error
1143  *   Pointer to the error structure.
1144  *
1145  * @return
1146  *   0 on success, a negative errno value otherwise and rte_errno is set.
1147  */
1148 static int
1149 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1150                             const struct rte_flow_action_mark *conf,
1151                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1152                             struct rte_flow_error *error)
1153 {
1154         struct mlx5_priv *priv = dev->data->dev_private;
1155         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1156                                            priv->sh->dv_mark_mask);
1157         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1158         struct rte_flow_item item = {
1159                 .spec = &data,
1160                 .mask = &mask,
1161         };
1162         struct field_modify_info reg_c_x[] = {
1163                 [1] = {0, 0, 0},
1164         };
1165         int reg;
1166
1167         if (!mask)
1168                 return rte_flow_error_set(error, EINVAL,
1169                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1170                                           NULL, "zero mark action mask");
1171         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1172         if (reg < 0)
1173                 return reg;
1174         MLX5_ASSERT(reg > 0);
1175         if (reg == REG_C_0) {
1176                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1177                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1178
1179                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1180                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1181                 mask = rte_cpu_to_be_32(mask << shl_c0);
1182         }
1183         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1184         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1185                                              MLX5_MODIFICATION_TYPE_SET, error);
1186 }
1187
1188 /**
1189  * Get metadata register index for specified steering domain.
1190  *
1191  * @param[in] dev
1192  *   Pointer to the rte_eth_dev structure.
1193  * @param[in] attr
1194  *   Attributes of flow to determine steering domain.
1195  * @param[out] error
1196  *   Pointer to the error structure.
1197  *
1198  * @return
1199  *   positive index on success, a negative errno value otherwise
1200  *   and rte_errno is set.
1201  */
1202 static enum modify_reg
1203 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1204                          const struct rte_flow_attr *attr,
1205                          struct rte_flow_error *error)
1206 {
1207         int reg =
1208                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1209                                           MLX5_METADATA_FDB :
1210                                             attr->egress ?
1211                                             MLX5_METADATA_TX :
1212                                             MLX5_METADATA_RX, 0, error);
1213         if (reg < 0)
1214                 return rte_flow_error_set(error,
1215                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1216                                           NULL, "unavailable "
1217                                           "metadata register");
1218         return reg;
1219 }
1220
1221 /**
1222  * Convert SET_META action to DV specification.
1223  *
1224  * @param[in] dev
1225  *   Pointer to the rte_eth_dev structure.
1226  * @param[in,out] resource
1227  *   Pointer to the modify-header resource.
1228  * @param[in] attr
1229  *   Attributes of flow that includes this item.
1230  * @param[in] conf
1231  *   Pointer to action specification.
1232  * @param[out] error
1233  *   Pointer to the error structure.
1234  *
1235  * @return
1236  *   0 on success, a negative errno value otherwise and rte_errno is set.
1237  */
1238 static int
1239 flow_dv_convert_action_set_meta
1240                         (struct rte_eth_dev *dev,
1241                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1242                          const struct rte_flow_attr *attr,
1243                          const struct rte_flow_action_set_meta *conf,
1244                          struct rte_flow_error *error)
1245 {
1246         uint32_t data = conf->data;
1247         uint32_t mask = conf->mask;
1248         struct rte_flow_item item = {
1249                 .spec = &data,
1250                 .mask = &mask,
1251         };
1252         struct field_modify_info reg_c_x[] = {
1253                 [1] = {0, 0, 0},
1254         };
1255         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1256
1257         if (reg < 0)
1258                 return reg;
1259         /*
1260          * In datapath code there is no endianness
1261          * coversions for perfromance reasons, all
1262          * pattern conversions are done in rte_flow.
1263          */
1264         if (reg == REG_C_0) {
1265                 struct mlx5_priv *priv = dev->data->dev_private;
1266                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1267                 uint32_t shl_c0;
1268
1269                 MLX5_ASSERT(msk_c0);
1270 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1271                 shl_c0 = rte_bsf32(msk_c0);
1272 #else
1273                 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1274 #endif
1275                 mask <<= shl_c0;
1276                 data <<= shl_c0;
1277                 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1278         }
1279         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1280         /* The routine expects parameters in memory as big-endian ones. */
1281         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1282                                              MLX5_MODIFICATION_TYPE_SET, error);
1283 }
1284
1285 /**
1286  * Convert modify-header set IPv4 DSCP action to DV specification.
1287  *
1288  * @param[in,out] resource
1289  *   Pointer to the modify-header resource.
1290  * @param[in] action
1291  *   Pointer to action specification.
1292  * @param[out] error
1293  *   Pointer to the error structure.
1294  *
1295  * @return
1296  *   0 on success, a negative errno value otherwise and rte_errno is set.
1297  */
1298 static int
1299 flow_dv_convert_action_modify_ipv4_dscp
1300                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1301                          const struct rte_flow_action *action,
1302                          struct rte_flow_error *error)
1303 {
1304         const struct rte_flow_action_set_dscp *conf =
1305                 (const struct rte_flow_action_set_dscp *)(action->conf);
1306         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1307         struct rte_flow_item_ipv4 ipv4;
1308         struct rte_flow_item_ipv4 ipv4_mask;
1309
1310         memset(&ipv4, 0, sizeof(ipv4));
1311         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1312         ipv4.hdr.type_of_service = conf->dscp;
1313         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1314         item.spec = &ipv4;
1315         item.mask = &ipv4_mask;
1316         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1317                                              MLX5_MODIFICATION_TYPE_SET, error);
1318 }
1319
1320 /**
1321  * Convert modify-header set IPv6 DSCP action to DV specification.
1322  *
1323  * @param[in,out] resource
1324  *   Pointer to the modify-header resource.
1325  * @param[in] action
1326  *   Pointer to action specification.
1327  * @param[out] error
1328  *   Pointer to the error structure.
1329  *
1330  * @return
1331  *   0 on success, a negative errno value otherwise and rte_errno is set.
1332  */
1333 static int
1334 flow_dv_convert_action_modify_ipv6_dscp
1335                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1336                          const struct rte_flow_action *action,
1337                          struct rte_flow_error *error)
1338 {
1339         const struct rte_flow_action_set_dscp *conf =
1340                 (const struct rte_flow_action_set_dscp *)(action->conf);
1341         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1342         struct rte_flow_item_ipv6 ipv6;
1343         struct rte_flow_item_ipv6 ipv6_mask;
1344
1345         memset(&ipv6, 0, sizeof(ipv6));
1346         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1347         /*
1348          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1349          * rdma-core only accept the DSCP bits byte aligned start from
1350          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1351          * bits in IPv6 case as rdma-core requires byte aligned value.
1352          */
1353         ipv6.hdr.vtc_flow = conf->dscp;
1354         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1355         item.spec = &ipv6;
1356         item.mask = &ipv6_mask;
1357         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1358                                              MLX5_MODIFICATION_TYPE_SET, error);
1359 }
1360
1361 /**
1362  * Validate MARK item.
1363  *
1364  * @param[in] dev
1365  *   Pointer to the rte_eth_dev structure.
1366  * @param[in] item
1367  *   Item specification.
1368  * @param[in] attr
1369  *   Attributes of flow that includes this item.
1370  * @param[out] error
1371  *   Pointer to error structure.
1372  *
1373  * @return
1374  *   0 on success, a negative errno value otherwise and rte_errno is set.
1375  */
1376 static int
1377 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1378                            const struct rte_flow_item *item,
1379                            const struct rte_flow_attr *attr __rte_unused,
1380                            struct rte_flow_error *error)
1381 {
1382         struct mlx5_priv *priv = dev->data->dev_private;
1383         struct mlx5_dev_config *config = &priv->config;
1384         const struct rte_flow_item_mark *spec = item->spec;
1385         const struct rte_flow_item_mark *mask = item->mask;
1386         const struct rte_flow_item_mark nic_mask = {
1387                 .id = priv->sh->dv_mark_mask,
1388         };
1389         int ret;
1390
1391         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1392                 return rte_flow_error_set(error, ENOTSUP,
1393                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1394                                           "extended metadata feature"
1395                                           " isn't enabled");
1396         if (!mlx5_flow_ext_mreg_supported(dev))
1397                 return rte_flow_error_set(error, ENOTSUP,
1398                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1399                                           "extended metadata register"
1400                                           " isn't supported");
1401         if (!nic_mask.id)
1402                 return rte_flow_error_set(error, ENOTSUP,
1403                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1404                                           "extended metadata register"
1405                                           " isn't available");
1406         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1407         if (ret < 0)
1408                 return ret;
1409         if (!spec)
1410                 return rte_flow_error_set(error, EINVAL,
1411                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1412                                           item->spec,
1413                                           "data cannot be empty");
1414         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1415                 return rte_flow_error_set(error, EINVAL,
1416                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1417                                           &spec->id,
1418                                           "mark id exceeds the limit");
1419         if (!mask)
1420                 mask = &nic_mask;
1421         if (!mask->id)
1422                 return rte_flow_error_set(error, EINVAL,
1423                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1424                                         "mask cannot be zero");
1425
1426         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1427                                         (const uint8_t *)&nic_mask,
1428                                         sizeof(struct rte_flow_item_mark),
1429                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1430         if (ret < 0)
1431                 return ret;
1432         return 0;
1433 }
1434
1435 /**
1436  * Validate META item.
1437  *
1438  * @param[in] dev
1439  *   Pointer to the rte_eth_dev structure.
1440  * @param[in] item
1441  *   Item specification.
1442  * @param[in] attr
1443  *   Attributes of flow that includes this item.
1444  * @param[out] error
1445  *   Pointer to error structure.
1446  *
1447  * @return
1448  *   0 on success, a negative errno value otherwise and rte_errno is set.
1449  */
1450 static int
1451 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1452                            const struct rte_flow_item *item,
1453                            const struct rte_flow_attr *attr,
1454                            struct rte_flow_error *error)
1455 {
1456         struct mlx5_priv *priv = dev->data->dev_private;
1457         struct mlx5_dev_config *config = &priv->config;
1458         const struct rte_flow_item_meta *spec = item->spec;
1459         const struct rte_flow_item_meta *mask = item->mask;
1460         struct rte_flow_item_meta nic_mask = {
1461                 .data = UINT32_MAX
1462         };
1463         int reg;
1464         int ret;
1465
1466         if (!spec)
1467                 return rte_flow_error_set(error, EINVAL,
1468                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1469                                           item->spec,
1470                                           "data cannot be empty");
1471         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1472                 if (!mlx5_flow_ext_mreg_supported(dev))
1473                         return rte_flow_error_set(error, ENOTSUP,
1474                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1475                                           "extended metadata register"
1476                                           " isn't supported");
1477                 reg = flow_dv_get_metadata_reg(dev, attr, error);
1478                 if (reg < 0)
1479                         return reg;
1480                 if (reg == REG_B)
1481                         return rte_flow_error_set(error, ENOTSUP,
1482                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1483                                           "match on reg_b "
1484                                           "isn't supported");
1485                 if (reg != REG_A)
1486                         nic_mask.data = priv->sh->dv_meta_mask;
1487         } else if (attr->transfer) {
1488                 return rte_flow_error_set(error, ENOTSUP,
1489                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1490                                         "extended metadata feature "
1491                                         "should be enabled when "
1492                                         "meta item is requested "
1493                                         "with e-switch mode ");
1494         }
1495         if (!mask)
1496                 mask = &rte_flow_item_meta_mask;
1497         if (!mask->data)
1498                 return rte_flow_error_set(error, EINVAL,
1499                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1500                                         "mask cannot be zero");
1501
1502         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1503                                         (const uint8_t *)&nic_mask,
1504                                         sizeof(struct rte_flow_item_meta),
1505                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1506         return ret;
1507 }
1508
1509 /**
1510  * Validate TAG item.
1511  *
1512  * @param[in] dev
1513  *   Pointer to the rte_eth_dev structure.
1514  * @param[in] item
1515  *   Item specification.
1516  * @param[in] attr
1517  *   Attributes of flow that includes this item.
1518  * @param[out] error
1519  *   Pointer to error structure.
1520  *
1521  * @return
1522  *   0 on success, a negative errno value otherwise and rte_errno is set.
1523  */
1524 static int
1525 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1526                           const struct rte_flow_item *item,
1527                           const struct rte_flow_attr *attr __rte_unused,
1528                           struct rte_flow_error *error)
1529 {
1530         const struct rte_flow_item_tag *spec = item->spec;
1531         const struct rte_flow_item_tag *mask = item->mask;
1532         const struct rte_flow_item_tag nic_mask = {
1533                 .data = RTE_BE32(UINT32_MAX),
1534                 .index = 0xff,
1535         };
1536         int ret;
1537
1538         if (!mlx5_flow_ext_mreg_supported(dev))
1539                 return rte_flow_error_set(error, ENOTSUP,
1540                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1541                                           "extensive metadata register"
1542                                           " isn't supported");
1543         if (!spec)
1544                 return rte_flow_error_set(error, EINVAL,
1545                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1546                                           item->spec,
1547                                           "data cannot be empty");
1548         if (!mask)
1549                 mask = &rte_flow_item_tag_mask;
1550         if (!mask->data)
1551                 return rte_flow_error_set(error, EINVAL,
1552                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1553                                         "mask cannot be zero");
1554
1555         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1556                                         (const uint8_t *)&nic_mask,
1557                                         sizeof(struct rte_flow_item_tag),
1558                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1559         if (ret < 0)
1560                 return ret;
1561         if (mask->index != 0xff)
1562                 return rte_flow_error_set(error, EINVAL,
1563                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1564                                           "partial mask for tag index"
1565                                           " is not supported");
1566         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1567         if (ret < 0)
1568                 return ret;
1569         MLX5_ASSERT(ret != REG_NON);
1570         return 0;
1571 }
1572
1573 /**
1574  * Validate vport item.
1575  *
1576  * @param[in] dev
1577  *   Pointer to the rte_eth_dev structure.
1578  * @param[in] item
1579  *   Item specification.
1580  * @param[in] attr
1581  *   Attributes of flow that includes this item.
1582  * @param[in] item_flags
1583  *   Bit-fields that holds the items detected until now.
1584  * @param[out] error
1585  *   Pointer to error structure.
1586  *
1587  * @return
1588  *   0 on success, a negative errno value otherwise and rte_errno is set.
1589  */
1590 static int
1591 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1592                               const struct rte_flow_item *item,
1593                               const struct rte_flow_attr *attr,
1594                               uint64_t item_flags,
1595                               struct rte_flow_error *error)
1596 {
1597         const struct rte_flow_item_port_id *spec = item->spec;
1598         const struct rte_flow_item_port_id *mask = item->mask;
1599         const struct rte_flow_item_port_id switch_mask = {
1600                         .id = 0xffffffff,
1601         };
1602         struct mlx5_priv *esw_priv;
1603         struct mlx5_priv *dev_priv;
1604         int ret;
1605
1606         if (!attr->transfer)
1607                 return rte_flow_error_set(error, EINVAL,
1608                                           RTE_FLOW_ERROR_TYPE_ITEM,
1609                                           NULL,
1610                                           "match on port id is valid only"
1611                                           " when transfer flag is enabled");
1612         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1613                 return rte_flow_error_set(error, ENOTSUP,
1614                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1615                                           "multiple source ports are not"
1616                                           " supported");
1617         if (!mask)
1618                 mask = &switch_mask;
1619         if (mask->id != 0xffffffff)
1620                 return rte_flow_error_set(error, ENOTSUP,
1621                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1622                                            mask,
1623                                            "no support for partial mask on"
1624                                            " \"id\" field");
1625         ret = mlx5_flow_item_acceptable
1626                                 (item, (const uint8_t *)mask,
1627                                  (const uint8_t *)&rte_flow_item_port_id_mask,
1628                                  sizeof(struct rte_flow_item_port_id),
1629                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1630         if (ret)
1631                 return ret;
1632         if (!spec)
1633                 return 0;
1634         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1635         if (!esw_priv)
1636                 return rte_flow_error_set(error, rte_errno,
1637                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1638                                           "failed to obtain E-Switch info for"
1639                                           " port");
1640         dev_priv = mlx5_dev_to_eswitch_info(dev);
1641         if (!dev_priv)
1642                 return rte_flow_error_set(error, rte_errno,
1643                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1644                                           NULL,
1645                                           "failed to obtain E-Switch info");
1646         if (esw_priv->domain_id != dev_priv->domain_id)
1647                 return rte_flow_error_set(error, EINVAL,
1648                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1649                                           "cannot match on a port from a"
1650                                           " different E-Switch");
1651         return 0;
1652 }
1653
1654 /**
1655  * Validate VLAN item.
1656  *
1657  * @param[in] item
1658  *   Item specification.
1659  * @param[in] item_flags
1660  *   Bit-fields that holds the items detected until now.
1661  * @param[in] dev
1662  *   Ethernet device flow is being created on.
1663  * @param[out] error
1664  *   Pointer to error structure.
1665  *
1666  * @return
1667  *   0 on success, a negative errno value otherwise and rte_errno is set.
1668  */
1669 static int
1670 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
1671                            uint64_t item_flags,
1672                            struct rte_eth_dev *dev,
1673                            struct rte_flow_error *error)
1674 {
1675         const struct rte_flow_item_vlan *mask = item->mask;
1676         const struct rte_flow_item_vlan nic_mask = {
1677                 .tci = RTE_BE16(UINT16_MAX),
1678                 .inner_type = RTE_BE16(UINT16_MAX),
1679         };
1680         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1681         int ret;
1682         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1683                                         MLX5_FLOW_LAYER_INNER_L4) :
1684                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1685                                         MLX5_FLOW_LAYER_OUTER_L4);
1686         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1687                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1688
1689         if (item_flags & vlanm)
1690                 return rte_flow_error_set(error, EINVAL,
1691                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1692                                           "multiple VLAN layers not supported");
1693         else if ((item_flags & l34m) != 0)
1694                 return rte_flow_error_set(error, EINVAL,
1695                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1696                                           "VLAN cannot follow L3/L4 layer");
1697         if (!mask)
1698                 mask = &rte_flow_item_vlan_mask;
1699         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1700                                         (const uint8_t *)&nic_mask,
1701                                         sizeof(struct rte_flow_item_vlan),
1702                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1703         if (ret)
1704                 return ret;
1705         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1706                 struct mlx5_priv *priv = dev->data->dev_private;
1707
1708                 if (priv->vmwa_context) {
1709                         /*
1710                          * Non-NULL context means we have a virtual machine
1711                          * and SR-IOV enabled, we have to create VLAN interface
1712                          * to make hypervisor to setup E-Switch vport
1713                          * context correctly. We avoid creating the multiple
1714                          * VLAN interfaces, so we cannot support VLAN tag mask.
1715                          */
1716                         return rte_flow_error_set(error, EINVAL,
1717                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1718                                                   item,
1719                                                   "VLAN tag mask is not"
1720                                                   " supported in virtual"
1721                                                   " environment");
1722                 }
1723         }
1724         return 0;
1725 }
1726
1727 /*
1728  * GTP flags are contained in 1 byte of the format:
1729  * -------------------------------------------
1730  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
1731  * |-----------------------------------------|
1732  * | value | Version | PT | Res | E | S | PN |
1733  * -------------------------------------------
1734  *
1735  * Matching is supported only for GTP flags E, S, PN.
1736  */
1737 #define MLX5_GTP_FLAGS_MASK     0x07
1738
1739 /**
1740  * Validate GTP item.
1741  *
1742  * @param[in] dev
1743  *   Pointer to the rte_eth_dev structure.
1744  * @param[in] item
1745  *   Item specification.
1746  * @param[in] item_flags
1747  *   Bit-fields that holds the items detected until now.
1748  * @param[out] error
1749  *   Pointer to error structure.
1750  *
1751  * @return
1752  *   0 on success, a negative errno value otherwise and rte_errno is set.
1753  */
1754 static int
1755 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
1756                           const struct rte_flow_item *item,
1757                           uint64_t item_flags,
1758                           struct rte_flow_error *error)
1759 {
1760         struct mlx5_priv *priv = dev->data->dev_private;
1761         const struct rte_flow_item_gtp *spec = item->spec;
1762         const struct rte_flow_item_gtp *mask = item->mask;
1763         const struct rte_flow_item_gtp nic_mask = {
1764                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
1765                 .msg_type = 0xff,
1766                 .teid = RTE_BE32(0xffffffff),
1767         };
1768
1769         if (!priv->config.hca_attr.tunnel_stateless_gtp)
1770                 return rte_flow_error_set(error, ENOTSUP,
1771                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1772                                           "GTP support is not enabled");
1773         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1774                 return rte_flow_error_set(error, ENOTSUP,
1775                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1776                                           "multiple tunnel layers not"
1777                                           " supported");
1778         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1779                 return rte_flow_error_set(error, EINVAL,
1780                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1781                                           "no outer UDP layer found");
1782         if (!mask)
1783                 mask = &rte_flow_item_gtp_mask;
1784         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
1785                 return rte_flow_error_set(error, ENOTSUP,
1786                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1787                                           "Match is supported for GTP"
1788                                           " flags only");
1789         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1790                                          (const uint8_t *)&nic_mask,
1791                                          sizeof(struct rte_flow_item_gtp),
1792                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1793 }
1794
1795 /**
1796  * Validate IPV4 item.
1797  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
1798  * add specific validation of fragment_offset field,
1799  *
1800  * @param[in] item
1801  *   Item specification.
1802  * @param[in] item_flags
1803  *   Bit-fields that holds the items detected until now.
1804  * @param[out] error
1805  *   Pointer to error structure.
1806  *
1807  * @return
1808  *   0 on success, a negative errno value otherwise and rte_errno is set.
1809  */
1810 static int
1811 flow_dv_validate_item_ipv4(const struct rte_flow_item *item,
1812                            uint64_t item_flags,
1813                            uint64_t last_item,
1814                            uint16_t ether_type,
1815                            struct rte_flow_error *error)
1816 {
1817         int ret;
1818         const struct rte_flow_item_ipv4 *spec = item->spec;
1819         const struct rte_flow_item_ipv4 *last = item->last;
1820         const struct rte_flow_item_ipv4 *mask = item->mask;
1821         rte_be16_t fragment_offset_spec = 0;
1822         rte_be16_t fragment_offset_last = 0;
1823         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
1824                 .hdr = {
1825                         .src_addr = RTE_BE32(0xffffffff),
1826                         .dst_addr = RTE_BE32(0xffffffff),
1827                         .type_of_service = 0xff,
1828                         .fragment_offset = RTE_BE16(0xffff),
1829                         .next_proto_id = 0xff,
1830                         .time_to_live = 0xff,
1831                 },
1832         };
1833
1834         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
1835                                            ether_type, &nic_ipv4_mask,
1836                                            MLX5_ITEM_RANGE_ACCEPTED, error);
1837         if (ret < 0)
1838                 return ret;
1839         if (spec && mask)
1840                 fragment_offset_spec = spec->hdr.fragment_offset &
1841                                        mask->hdr.fragment_offset;
1842         if (!fragment_offset_spec)
1843                 return 0;
1844         /*
1845          * spec and mask are valid, enforce using full mask to make sure the
1846          * complete value is used correctly.
1847          */
1848         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1849                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1850                 return rte_flow_error_set(error, EINVAL,
1851                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1852                                           item, "must use full mask for"
1853                                           " fragment_offset");
1854         /*
1855          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
1856          * indicating this is 1st fragment of fragmented packet.
1857          * This is not yet supported in MLX5, return appropriate error message.
1858          */
1859         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
1860                 return rte_flow_error_set(error, ENOTSUP,
1861                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1862                                           "match on first fragment not "
1863                                           "supported");
1864         if (fragment_offset_spec && !last)
1865                 return rte_flow_error_set(error, ENOTSUP,
1866                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1867                                           "specified value not supported");
1868         /* spec and last are valid, validate the specified range. */
1869         fragment_offset_last = last->hdr.fragment_offset &
1870                                mask->hdr.fragment_offset;
1871         /*
1872          * Match on fragment_offset spec 0x2001 and last 0x3fff
1873          * means MF is 1 and frag-offset is > 0.
1874          * This packet is fragment 2nd and onward, excluding last.
1875          * This is not yet supported in MLX5, return appropriate
1876          * error message.
1877          */
1878         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
1879             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1880                 return rte_flow_error_set(error, ENOTSUP,
1881                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1882                                           last, "match on following "
1883                                           "fragments not supported");
1884         /*
1885          * Match on fragment_offset spec 0x0001 and last 0x1fff
1886          * means MF is 0 and frag-offset is > 0.
1887          * This packet is last fragment of fragmented packet.
1888          * This is not yet supported in MLX5, return appropriate
1889          * error message.
1890          */
1891         if (fragment_offset_spec == RTE_BE16(1) &&
1892             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
1893                 return rte_flow_error_set(error, ENOTSUP,
1894                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1895                                           last, "match on last "
1896                                           "fragment not supported");
1897         /*
1898          * Match on fragment_offset spec 0x0001 and last 0x3fff
1899          * means MF and/or frag-offset is not 0.
1900          * This is a fragmented packet.
1901          * Other range values are invalid and rejected.
1902          */
1903         if (!(fragment_offset_spec == RTE_BE16(1) &&
1904               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
1905                 return rte_flow_error_set(error, ENOTSUP,
1906                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
1907                                           "specified range not supported");
1908         return 0;
1909 }
1910
1911 /**
1912  * Validate IPV6 fragment extension item.
1913  *
1914  * @param[in] item
1915  *   Item specification.
1916  * @param[in] item_flags
1917  *   Bit-fields that holds the items detected until now.
1918  * @param[out] error
1919  *   Pointer to error structure.
1920  *
1921  * @return
1922  *   0 on success, a negative errno value otherwise and rte_errno is set.
1923  */
1924 static int
1925 flow_dv_validate_item_ipv6_frag_ext(const struct rte_flow_item *item,
1926                                     uint64_t item_flags,
1927                                     struct rte_flow_error *error)
1928 {
1929         const struct rte_flow_item_ipv6_frag_ext *spec = item->spec;
1930         const struct rte_flow_item_ipv6_frag_ext *last = item->last;
1931         const struct rte_flow_item_ipv6_frag_ext *mask = item->mask;
1932         rte_be16_t frag_data_spec = 0;
1933         rte_be16_t frag_data_last = 0;
1934         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1935         const uint64_t l4m = tunnel ? MLX5_FLOW_LAYER_INNER_L4 :
1936                                       MLX5_FLOW_LAYER_OUTER_L4;
1937         int ret = 0;
1938         struct rte_flow_item_ipv6_frag_ext nic_mask = {
1939                 .hdr = {
1940                         .next_header = 0xff,
1941                         .frag_data = RTE_BE16(0xffff),
1942                 },
1943         };
1944
1945         if (item_flags & l4m)
1946                 return rte_flow_error_set(error, EINVAL,
1947                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1948                                           "ipv6 fragment extension item cannot "
1949                                           "follow L4 item.");
1950         if ((tunnel && !(item_flags & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
1951             (!tunnel && !(item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV6)))
1952                 return rte_flow_error_set(error, EINVAL,
1953                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1954                                           "ipv6 fragment extension item must "
1955                                           "follow ipv6 item");
1956         if (spec && mask)
1957                 frag_data_spec = spec->hdr.frag_data & mask->hdr.frag_data;
1958         if (!frag_data_spec)
1959                 return 0;
1960         /*
1961          * spec and mask are valid, enforce using full mask to make sure the
1962          * complete value is used correctly.
1963          */
1964         if ((mask->hdr.frag_data & RTE_BE16(RTE_IPV6_FRAG_USED_MASK)) !=
1965                                 RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
1966                 return rte_flow_error_set(error, EINVAL,
1967                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1968                                           item, "must use full mask for"
1969                                           " frag_data");
1970         /*
1971          * Match on frag_data 0x00001 means M is 1 and frag-offset is 0.
1972          * This is 1st fragment of fragmented packet.
1973          */
1974         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_MF_MASK))
1975                 return rte_flow_error_set(error, ENOTSUP,
1976                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1977                                           "match on first fragment not "
1978                                           "supported");
1979         if (frag_data_spec && !last)
1980                 return rte_flow_error_set(error, EINVAL,
1981                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1982                                           "specified value not supported");
1983         ret = mlx5_flow_item_acceptable
1984                                 (item, (const uint8_t *)mask,
1985                                  (const uint8_t *)&nic_mask,
1986                                  sizeof(struct rte_flow_item_ipv6_frag_ext),
1987                                  MLX5_ITEM_RANGE_ACCEPTED, error);
1988         if (ret)
1989                 return ret;
1990         /* spec and last are valid, validate the specified range. */
1991         frag_data_last = last->hdr.frag_data & mask->hdr.frag_data;
1992         /*
1993          * Match on frag_data spec 0x0009 and last 0xfff9
1994          * means M is 1 and frag-offset is > 0.
1995          * This packet is fragment 2nd and onward, excluding last.
1996          * This is not yet supported in MLX5, return appropriate
1997          * error message.
1998          */
1999         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN |
2000                                        RTE_IPV6_EHDR_MF_MASK) &&
2001             frag_data_last == RTE_BE16(RTE_IPV6_FRAG_USED_MASK))
2002                 return rte_flow_error_set(error, ENOTSUP,
2003                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2004                                           last, "match on following "
2005                                           "fragments not supported");
2006         /*
2007          * Match on frag_data spec 0x0008 and last 0xfff8
2008          * means M is 0 and frag-offset is > 0.
2009          * This packet is last fragment of fragmented packet.
2010          * This is not yet supported in MLX5, return appropriate
2011          * error message.
2012          */
2013         if (frag_data_spec == RTE_BE16(RTE_IPV6_EHDR_FO_ALIGN) &&
2014             frag_data_last == RTE_BE16(RTE_IPV6_EHDR_FO_MASK))
2015                 return rte_flow_error_set(error, ENOTSUP,
2016                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
2017                                           last, "match on last "
2018                                           "fragment not supported");
2019         /* Other range values are invalid and rejected. */
2020         return rte_flow_error_set(error, EINVAL,
2021                                   RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
2022                                   "specified range not supported");
2023 }
2024
2025 /**
2026  * Validate the pop VLAN action.
2027  *
2028  * @param[in] dev
2029  *   Pointer to the rte_eth_dev structure.
2030  * @param[in] action_flags
2031  *   Holds the actions detected until now.
2032  * @param[in] action
2033  *   Pointer to the pop vlan action.
2034  * @param[in] item_flags
2035  *   The items found in this flow rule.
2036  * @param[in] attr
2037  *   Pointer to flow attributes.
2038  * @param[out] error
2039  *   Pointer to error structure.
2040  *
2041  * @return
2042  *   0 on success, a negative errno value otherwise and rte_errno is set.
2043  */
2044 static int
2045 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
2046                                  uint64_t action_flags,
2047                                  const struct rte_flow_action *action,
2048                                  uint64_t item_flags,
2049                                  const struct rte_flow_attr *attr,
2050                                  struct rte_flow_error *error)
2051 {
2052         const struct mlx5_priv *priv = dev->data->dev_private;
2053
2054         (void)action;
2055         (void)attr;
2056         if (!priv->sh->pop_vlan_action)
2057                 return rte_flow_error_set(error, ENOTSUP,
2058                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2059                                           NULL,
2060                                           "pop vlan action is not supported");
2061         if (attr->egress)
2062                 return rte_flow_error_set(error, ENOTSUP,
2063                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2064                                           NULL,
2065                                           "pop vlan action not supported for "
2066                                           "egress");
2067         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
2068                 return rte_flow_error_set(error, ENOTSUP,
2069                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2070                                           "no support for multiple VLAN "
2071                                           "actions");
2072         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
2073         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
2074             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
2075                 return rte_flow_error_set(error, ENOTSUP,
2076                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2077                                           NULL,
2078                                           "cannot pop vlan after decap without "
2079                                           "match on inner vlan in the flow");
2080         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
2081         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
2082             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2083                 return rte_flow_error_set(error, ENOTSUP,
2084                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2085                                           NULL,
2086                                           "cannot pop vlan without a "
2087                                           "match on (outer) vlan in the flow");
2088         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2089                 return rte_flow_error_set(error, EINVAL,
2090                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2091                                           "wrong action order, port_id should "
2092                                           "be after pop VLAN action");
2093         if (!attr->transfer && priv->representor)
2094                 return rte_flow_error_set(error, ENOTSUP,
2095                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2096                                           "pop vlan action for VF representor "
2097                                           "not supported on NIC table");
2098         return 0;
2099 }
2100
2101 /**
2102  * Get VLAN default info from vlan match info.
2103  *
2104  * @param[in] items
2105  *   the list of item specifications.
2106  * @param[out] vlan
2107  *   pointer VLAN info to fill to.
2108  *
2109  * @return
2110  *   0 on success, a negative errno value otherwise and rte_errno is set.
2111  */
2112 static void
2113 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2114                                   struct rte_vlan_hdr *vlan)
2115 {
2116         const struct rte_flow_item_vlan nic_mask = {
2117                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2118                                 MLX5DV_FLOW_VLAN_VID_MASK),
2119                 .inner_type = RTE_BE16(0xffff),
2120         };
2121
2122         if (items == NULL)
2123                 return;
2124         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2125                 int type = items->type;
2126
2127                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2128                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2129                         break;
2130         }
2131         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2132                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2133                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2134
2135                 /* If VLAN item in pattern doesn't contain data, return here. */
2136                 if (!vlan_v)
2137                         return;
2138                 if (!vlan_m)
2139                         vlan_m = &nic_mask;
2140                 /* Only full match values are accepted */
2141                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2142                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2143                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2144                         vlan->vlan_tci |=
2145                                 rte_be_to_cpu_16(vlan_v->tci &
2146                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2147                 }
2148                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2149                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2150                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2151                         vlan->vlan_tci |=
2152                                 rte_be_to_cpu_16(vlan_v->tci &
2153                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2154                 }
2155                 if (vlan_m->inner_type == nic_mask.inner_type)
2156                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2157                                                            vlan_m->inner_type);
2158         }
2159 }
2160
2161 /**
2162  * Validate the push VLAN action.
2163  *
2164  * @param[in] dev
2165  *   Pointer to the rte_eth_dev structure.
2166  * @param[in] action_flags
2167  *   Holds the actions detected until now.
2168  * @param[in] item_flags
2169  *   The items found in this flow rule.
2170  * @param[in] action
2171  *   Pointer to the action structure.
2172  * @param[in] attr
2173  *   Pointer to flow attributes
2174  * @param[out] error
2175  *   Pointer to error structure.
2176  *
2177  * @return
2178  *   0 on success, a negative errno value otherwise and rte_errno is set.
2179  */
2180 static int
2181 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2182                                   uint64_t action_flags,
2183                                   const struct rte_flow_item_vlan *vlan_m,
2184                                   const struct rte_flow_action *action,
2185                                   const struct rte_flow_attr *attr,
2186                                   struct rte_flow_error *error)
2187 {
2188         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2189         const struct mlx5_priv *priv = dev->data->dev_private;
2190
2191         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2192             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2193                 return rte_flow_error_set(error, EINVAL,
2194                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2195                                           "invalid vlan ethertype");
2196         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2197                 return rte_flow_error_set(error, EINVAL,
2198                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2199                                           "wrong action order, port_id should "
2200                                           "be after push VLAN");
2201         if (!attr->transfer && priv->representor)
2202                 return rte_flow_error_set(error, ENOTSUP,
2203                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2204                                           "push vlan action for VF representor "
2205                                           "not supported on NIC table");
2206         if (vlan_m &&
2207             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2208             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2209                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2210             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2211             !(mlx5_flow_find_action
2212                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2213                 return rte_flow_error_set(error, EINVAL,
2214                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2215                                           "not full match mask on VLAN PCP and "
2216                                           "there is no of_set_vlan_pcp action, "
2217                                           "push VLAN action cannot figure out "
2218                                           "PCP value");
2219         if (vlan_m &&
2220             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2221             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2222                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2223             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2224             !(mlx5_flow_find_action
2225                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2226                 return rte_flow_error_set(error, EINVAL,
2227                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2228                                           "not full match mask on VLAN VID and "
2229                                           "there is no of_set_vlan_vid action, "
2230                                           "push VLAN action cannot figure out "
2231                                           "VID value");
2232         (void)attr;
2233         return 0;
2234 }
2235
2236 /**
2237  * Validate the set VLAN PCP.
2238  *
2239  * @param[in] action_flags
2240  *   Holds the actions detected until now.
2241  * @param[in] actions
2242  *   Pointer to the list of actions remaining in the flow rule.
2243  * @param[out] error
2244  *   Pointer to error structure.
2245  *
2246  * @return
2247  *   0 on success, a negative errno value otherwise and rte_errno is set.
2248  */
2249 static int
2250 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2251                                      const struct rte_flow_action actions[],
2252                                      struct rte_flow_error *error)
2253 {
2254         const struct rte_flow_action *action = actions;
2255         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2256
2257         if (conf->vlan_pcp > 7)
2258                 return rte_flow_error_set(error, EINVAL,
2259                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2260                                           "VLAN PCP value is too big");
2261         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2262                 return rte_flow_error_set(error, ENOTSUP,
2263                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2264                                           "set VLAN PCP action must follow "
2265                                           "the push VLAN action");
2266         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2267                 return rte_flow_error_set(error, ENOTSUP,
2268                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2269                                           "Multiple VLAN PCP modification are "
2270                                           "not supported");
2271         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2272                 return rte_flow_error_set(error, EINVAL,
2273                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2274                                           "wrong action order, port_id should "
2275                                           "be after set VLAN PCP");
2276         return 0;
2277 }
2278
2279 /**
2280  * Validate the set VLAN VID.
2281  *
2282  * @param[in] item_flags
2283  *   Holds the items detected in this rule.
2284  * @param[in] action_flags
2285  *   Holds the actions detected until now.
2286  * @param[in] actions
2287  *   Pointer to the list of actions remaining in the flow rule.
2288  * @param[out] error
2289  *   Pointer to error structure.
2290  *
2291  * @return
2292  *   0 on success, a negative errno value otherwise and rte_errno is set.
2293  */
2294 static int
2295 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2296                                      uint64_t action_flags,
2297                                      const struct rte_flow_action actions[],
2298                                      struct rte_flow_error *error)
2299 {
2300         const struct rte_flow_action *action = actions;
2301         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2302
2303         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2304                 return rte_flow_error_set(error, EINVAL,
2305                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2306                                           "VLAN VID value is too big");
2307         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2308             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2309                 return rte_flow_error_set(error, ENOTSUP,
2310                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2311                                           "set VLAN VID action must follow push"
2312                                           " VLAN action or match on VLAN item");
2313         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2314                 return rte_flow_error_set(error, ENOTSUP,
2315                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2316                                           "Multiple VLAN VID modifications are "
2317                                           "not supported");
2318         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2319                 return rte_flow_error_set(error, EINVAL,
2320                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2321                                           "wrong action order, port_id should "
2322                                           "be after set VLAN VID");
2323         return 0;
2324 }
2325
2326 /*
2327  * Validate the FLAG action.
2328  *
2329  * @param[in] dev
2330  *   Pointer to the rte_eth_dev structure.
2331  * @param[in] action_flags
2332  *   Holds the actions detected until now.
2333  * @param[in] attr
2334  *   Pointer to flow attributes
2335  * @param[out] error
2336  *   Pointer to error structure.
2337  *
2338  * @return
2339  *   0 on success, a negative errno value otherwise and rte_errno is set.
2340  */
2341 static int
2342 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
2343                              uint64_t action_flags,
2344                              const struct rte_flow_attr *attr,
2345                              struct rte_flow_error *error)
2346 {
2347         struct mlx5_priv *priv = dev->data->dev_private;
2348         struct mlx5_dev_config *config = &priv->config;
2349         int ret;
2350
2351         /* Fall back if no extended metadata register support. */
2352         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2353                 return mlx5_flow_validate_action_flag(action_flags, attr,
2354                                                       error);
2355         /* Extensive metadata mode requires registers. */
2356         if (!mlx5_flow_ext_mreg_supported(dev))
2357                 return rte_flow_error_set(error, ENOTSUP,
2358                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2359                                           "no metadata registers "
2360                                           "to support flag action");
2361         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
2362                 return rte_flow_error_set(error, ENOTSUP,
2363                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2364                                           "extended metadata register"
2365                                           " isn't available");
2366         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2367         if (ret < 0)
2368                 return ret;
2369         MLX5_ASSERT(ret > 0);
2370         if (action_flags & MLX5_FLOW_ACTION_MARK)
2371                 return rte_flow_error_set(error, EINVAL,
2372                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2373                                           "can't mark and flag in same flow");
2374         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2375                 return rte_flow_error_set(error, EINVAL,
2376                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2377                                           "can't have 2 flag"
2378                                           " actions in same flow");
2379         return 0;
2380 }
2381
2382 /**
2383  * Validate MARK action.
2384  *
2385  * @param[in] dev
2386  *   Pointer to the rte_eth_dev structure.
2387  * @param[in] action
2388  *   Pointer to action.
2389  * @param[in] action_flags
2390  *   Holds the actions detected until now.
2391  * @param[in] attr
2392  *   Pointer to flow attributes
2393  * @param[out] error
2394  *   Pointer to error structure.
2395  *
2396  * @return
2397  *   0 on success, a negative errno value otherwise and rte_errno is set.
2398  */
2399 static int
2400 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
2401                              const struct rte_flow_action *action,
2402                              uint64_t action_flags,
2403                              const struct rte_flow_attr *attr,
2404                              struct rte_flow_error *error)
2405 {
2406         struct mlx5_priv *priv = dev->data->dev_private;
2407         struct mlx5_dev_config *config = &priv->config;
2408         const struct rte_flow_action_mark *mark = action->conf;
2409         int ret;
2410
2411         /* Fall back if no extended metadata register support. */
2412         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2413                 return mlx5_flow_validate_action_mark(action, action_flags,
2414                                                       attr, error);
2415         /* Extensive metadata mode requires registers. */
2416         if (!mlx5_flow_ext_mreg_supported(dev))
2417                 return rte_flow_error_set(error, ENOTSUP,
2418                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2419                                           "no metadata registers "
2420                                           "to support mark action");
2421         if (!priv->sh->dv_mark_mask)
2422                 return rte_flow_error_set(error, ENOTSUP,
2423                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2424                                           "extended metadata register"
2425                                           " isn't available");
2426         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2427         if (ret < 0)
2428                 return ret;
2429         MLX5_ASSERT(ret > 0);
2430         if (!mark)
2431                 return rte_flow_error_set(error, EINVAL,
2432                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2433                                           "configuration cannot be null");
2434         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
2435                 return rte_flow_error_set(error, EINVAL,
2436                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2437                                           &mark->id,
2438                                           "mark id exceeds the limit");
2439         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2440                 return rte_flow_error_set(error, EINVAL,
2441                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2442                                           "can't flag and mark in same flow");
2443         if (action_flags & MLX5_FLOW_ACTION_MARK)
2444                 return rte_flow_error_set(error, EINVAL,
2445                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2446                                           "can't have 2 mark actions in same"
2447                                           " flow");
2448         return 0;
2449 }
2450
2451 /**
2452  * Validate SET_META action.
2453  *
2454  * @param[in] dev
2455  *   Pointer to the rte_eth_dev structure.
2456  * @param[in] action
2457  *   Pointer to the action structure.
2458  * @param[in] action_flags
2459  *   Holds the actions detected until now.
2460  * @param[in] attr
2461  *   Pointer to flow attributes
2462  * @param[out] error
2463  *   Pointer to error structure.
2464  *
2465  * @return
2466  *   0 on success, a negative errno value otherwise and rte_errno is set.
2467  */
2468 static int
2469 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
2470                                  const struct rte_flow_action *action,
2471                                  uint64_t action_flags __rte_unused,
2472                                  const struct rte_flow_attr *attr,
2473                                  struct rte_flow_error *error)
2474 {
2475         const struct rte_flow_action_set_meta *conf;
2476         uint32_t nic_mask = UINT32_MAX;
2477         int reg;
2478
2479         if (!mlx5_flow_ext_mreg_supported(dev))
2480                 return rte_flow_error_set(error, ENOTSUP,
2481                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2482                                           "extended metadata register"
2483                                           " isn't supported");
2484         reg = flow_dv_get_metadata_reg(dev, attr, error);
2485         if (reg < 0)
2486                 return reg;
2487         if (reg != REG_A && reg != REG_B) {
2488                 struct mlx5_priv *priv = dev->data->dev_private;
2489
2490                 nic_mask = priv->sh->dv_meta_mask;
2491         }
2492         if (!(action->conf))
2493                 return rte_flow_error_set(error, EINVAL,
2494                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2495                                           "configuration cannot be null");
2496         conf = (const struct rte_flow_action_set_meta *)action->conf;
2497         if (!conf->mask)
2498                 return rte_flow_error_set(error, EINVAL,
2499                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2500                                           "zero mask doesn't have any effect");
2501         if (conf->mask & ~nic_mask)
2502                 return rte_flow_error_set(error, EINVAL,
2503                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2504                                           "meta data must be within reg C0");
2505         return 0;
2506 }
2507
2508 /**
2509  * Validate SET_TAG action.
2510  *
2511  * @param[in] dev
2512  *   Pointer to the rte_eth_dev structure.
2513  * @param[in] action
2514  *   Pointer to the action structure.
2515  * @param[in] action_flags
2516  *   Holds the actions detected until now.
2517  * @param[in] attr
2518  *   Pointer to flow attributes
2519  * @param[out] error
2520  *   Pointer to error structure.
2521  *
2522  * @return
2523  *   0 on success, a negative errno value otherwise and rte_errno is set.
2524  */
2525 static int
2526 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
2527                                 const struct rte_flow_action *action,
2528                                 uint64_t action_flags,
2529                                 const struct rte_flow_attr *attr,
2530                                 struct rte_flow_error *error)
2531 {
2532         const struct rte_flow_action_set_tag *conf;
2533         const uint64_t terminal_action_flags =
2534                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
2535                 MLX5_FLOW_ACTION_RSS;
2536         int ret;
2537
2538         if (!mlx5_flow_ext_mreg_supported(dev))
2539                 return rte_flow_error_set(error, ENOTSUP,
2540                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2541                                           "extensive metadata register"
2542                                           " isn't supported");
2543         if (!(action->conf))
2544                 return rte_flow_error_set(error, EINVAL,
2545                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2546                                           "configuration cannot be null");
2547         conf = (const struct rte_flow_action_set_tag *)action->conf;
2548         if (!conf->mask)
2549                 return rte_flow_error_set(error, EINVAL,
2550                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2551                                           "zero mask doesn't have any effect");
2552         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
2553         if (ret < 0)
2554                 return ret;
2555         if (!attr->transfer && attr->ingress &&
2556             (action_flags & terminal_action_flags))
2557                 return rte_flow_error_set(error, EINVAL,
2558                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2559                                           "set_tag has no effect"
2560                                           " with terminal actions");
2561         return 0;
2562 }
2563
2564 /**
2565  * Validate count action.
2566  *
2567  * @param[in] dev
2568  *   Pointer to rte_eth_dev structure.
2569  * @param[out] error
2570  *   Pointer to error structure.
2571  *
2572  * @return
2573  *   0 on success, a negative errno value otherwise and rte_errno is set.
2574  */
2575 static int
2576 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2577                               struct rte_flow_error *error)
2578 {
2579         struct mlx5_priv *priv = dev->data->dev_private;
2580
2581         if (!priv->config.devx)
2582                 goto notsup_err;
2583 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2584         return 0;
2585 #endif
2586 notsup_err:
2587         return rte_flow_error_set
2588                       (error, ENOTSUP,
2589                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2590                        NULL,
2591                        "count action not supported");
2592 }
2593
2594 /**
2595  * Validate the L2 encap action.
2596  *
2597  * @param[in] dev
2598  *   Pointer to the rte_eth_dev structure.
2599  * @param[in] action_flags
2600  *   Holds the actions detected until now.
2601  * @param[in] action
2602  *   Pointer to the action structure.
2603  * @param[in] attr
2604  *   Pointer to flow attributes.
2605  * @param[out] error
2606  *   Pointer to error structure.
2607  *
2608  * @return
2609  *   0 on success, a negative errno value otherwise and rte_errno is set.
2610  */
2611 static int
2612 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
2613                                  uint64_t action_flags,
2614                                  const struct rte_flow_action *action,
2615                                  const struct rte_flow_attr *attr,
2616                                  struct rte_flow_error *error)
2617 {
2618         const struct mlx5_priv *priv = dev->data->dev_private;
2619
2620         if (!(action->conf))
2621                 return rte_flow_error_set(error, EINVAL,
2622                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2623                                           "configuration cannot be null");
2624         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
2625                 return rte_flow_error_set(error, EINVAL,
2626                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2627                                           "can only have a single encap action "
2628                                           "in a flow");
2629         if (!attr->transfer && priv->representor)
2630                 return rte_flow_error_set(error, ENOTSUP,
2631                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2632                                           "encap action for VF representor "
2633                                           "not supported on NIC table");
2634         return 0;
2635 }
2636
2637 /**
2638  * Validate a decap action.
2639  *
2640  * @param[in] dev
2641  *   Pointer to the rte_eth_dev structure.
2642  * @param[in] action_flags
2643  *   Holds the actions detected until now.
2644  * @param[in] attr
2645  *   Pointer to flow attributes
2646  * @param[out] error
2647  *   Pointer to error structure.
2648  *
2649  * @return
2650  *   0 on success, a negative errno value otherwise and rte_errno is set.
2651  */
2652 static int
2653 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
2654                               uint64_t action_flags,
2655                               const struct rte_flow_attr *attr,
2656                               struct rte_flow_error *error)
2657 {
2658         const struct mlx5_priv *priv = dev->data->dev_private;
2659
2660         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
2661             !priv->config.decap_en)
2662                 return rte_flow_error_set(error, ENOTSUP,
2663                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2664                                           "decap is not enabled");
2665         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
2666                 return rte_flow_error_set(error, ENOTSUP,
2667                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2668                                           action_flags &
2669                                           MLX5_FLOW_ACTION_DECAP ? "can only "
2670                                           "have a single decap action" : "decap "
2671                                           "after encap is not supported");
2672         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2673                 return rte_flow_error_set(error, EINVAL,
2674                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2675                                           "can't have decap action after"
2676                                           " modify action");
2677         if (attr->egress)
2678                 return rte_flow_error_set(error, ENOTSUP,
2679                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2680                                           NULL,
2681                                           "decap action not supported for "
2682                                           "egress");
2683         if (!attr->transfer && priv->representor)
2684                 return rte_flow_error_set(error, ENOTSUP,
2685                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2686                                           "decap action for VF representor "
2687                                           "not supported on NIC table");
2688         return 0;
2689 }
2690
2691 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
2692
2693 /**
2694  * Validate the raw encap and decap actions.
2695  *
2696  * @param[in] dev
2697  *   Pointer to the rte_eth_dev structure.
2698  * @param[in] decap
2699  *   Pointer to the decap action.
2700  * @param[in] encap
2701  *   Pointer to the encap action.
2702  * @param[in] attr
2703  *   Pointer to flow attributes
2704  * @param[in/out] action_flags
2705  *   Holds the actions detected until now.
2706  * @param[out] actions_n
2707  *   pointer to the number of actions counter.
2708  * @param[out] error
2709  *   Pointer to error structure.
2710  *
2711  * @return
2712  *   0 on success, a negative errno value otherwise and rte_errno is set.
2713  */
2714 static int
2715 flow_dv_validate_action_raw_encap_decap
2716         (struct rte_eth_dev *dev,
2717          const struct rte_flow_action_raw_decap *decap,
2718          const struct rte_flow_action_raw_encap *encap,
2719          const struct rte_flow_attr *attr, uint64_t *action_flags,
2720          int *actions_n, struct rte_flow_error *error)
2721 {
2722         const struct mlx5_priv *priv = dev->data->dev_private;
2723         int ret;
2724
2725         if (encap && (!encap->size || !encap->data))
2726                 return rte_flow_error_set(error, EINVAL,
2727                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2728                                           "raw encap data cannot be empty");
2729         if (decap && encap) {
2730                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
2731                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2732                         /* L3 encap. */
2733                         decap = NULL;
2734                 else if (encap->size <=
2735                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2736                            decap->size >
2737                            MLX5_ENCAPSULATION_DECISION_SIZE)
2738                         /* L3 decap. */
2739                         encap = NULL;
2740                 else if (encap->size >
2741                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2742                            decap->size >
2743                            MLX5_ENCAPSULATION_DECISION_SIZE)
2744                         /* 2 L2 actions: encap and decap. */
2745                         ;
2746                 else
2747                         return rte_flow_error_set(error,
2748                                 ENOTSUP,
2749                                 RTE_FLOW_ERROR_TYPE_ACTION,
2750                                 NULL, "unsupported too small "
2751                                 "raw decap and too small raw "
2752                                 "encap combination");
2753         }
2754         if (decap) {
2755                 ret = flow_dv_validate_action_decap(dev, *action_flags, attr,
2756                                                     error);
2757                 if (ret < 0)
2758                         return ret;
2759                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
2760                 ++(*actions_n);
2761         }
2762         if (encap) {
2763                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
2764                         return rte_flow_error_set(error, ENOTSUP,
2765                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2766                                                   NULL,
2767                                                   "small raw encap size");
2768                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
2769                         return rte_flow_error_set(error, EINVAL,
2770                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2771                                                   NULL,
2772                                                   "more than one encap action");
2773                 if (!attr->transfer && priv->representor)
2774                         return rte_flow_error_set
2775                                         (error, ENOTSUP,
2776                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2777                                          "encap action for VF representor "
2778                                          "not supported on NIC table");
2779                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
2780                 ++(*actions_n);
2781         }
2782         return 0;
2783 }
2784
2785 /**
2786  * Match encap_decap resource.
2787  *
2788  * @param entry
2789  *   Pointer to exist resource entry object.
2790  * @param ctx
2791  *   Pointer to new encap_decap resource.
2792  *
2793  * @return
2794  *   0 on matching, -1 otherwise.
2795  */
2796 static int
2797 flow_dv_encap_decap_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
2798 {
2799         struct mlx5_flow_dv_encap_decap_resource *resource;
2800         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2801
2802         resource = (struct mlx5_flow_dv_encap_decap_resource *)ctx;
2803         cache_resource = container_of(entry,
2804                                       struct mlx5_flow_dv_encap_decap_resource,
2805                                       entry);
2806         if (resource->entry.key == cache_resource->entry.key &&
2807             resource->reformat_type == cache_resource->reformat_type &&
2808             resource->ft_type == cache_resource->ft_type &&
2809             resource->flags == cache_resource->flags &&
2810             resource->size == cache_resource->size &&
2811             !memcmp((const void *)resource->buf,
2812                     (const void *)cache_resource->buf,
2813                     resource->size))
2814                 return 0;
2815         return -1;
2816 }
2817
2818 /**
2819  * Find existing encap/decap resource or create and register a new one.
2820  *
2821  * @param[in, out] dev
2822  *   Pointer to rte_eth_dev structure.
2823  * @param[in, out] resource
2824  *   Pointer to encap/decap resource.
2825  * @parm[in, out] dev_flow
2826  *   Pointer to the dev_flow.
2827  * @param[out] error
2828  *   pointer to error structure.
2829  *
2830  * @return
2831  *   0 on success otherwise -errno and errno is set.
2832  */
2833 static int
2834 flow_dv_encap_decap_resource_register
2835                         (struct rte_eth_dev *dev,
2836                          struct mlx5_flow_dv_encap_decap_resource *resource,
2837                          struct mlx5_flow *dev_flow,
2838                          struct rte_flow_error *error)
2839 {
2840         struct mlx5_priv *priv = dev->data->dev_private;
2841         struct mlx5_dev_ctx_shared *sh = priv->sh;
2842         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2843         struct mlx5dv_dr_domain *domain;
2844         struct mlx5_hlist_entry *entry;
2845         union mlx5_flow_encap_decap_key encap_decap_key = {
2846                 {
2847                         .ft_type = resource->ft_type,
2848                         .refmt_type = resource->reformat_type,
2849                         .buf_size = resource->size,
2850                         .table_level = !!dev_flow->dv.group,
2851                         .cksum = 0,
2852                 }
2853         };
2854         int ret;
2855
2856         resource->flags = dev_flow->dv.group ? 0 : 1;
2857         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2858                 domain = sh->fdb_domain;
2859         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2860                 domain = sh->rx_domain;
2861         else
2862                 domain = sh->tx_domain;
2863         encap_decap_key.cksum = __rte_raw_cksum(resource->buf,
2864                                                 resource->size, 0);
2865         resource->entry.key = encap_decap_key.v64;
2866         /* Lookup a matching resource from cache. */
2867         entry = mlx5_hlist_lookup_ex(sh->encaps_decaps, resource->entry.key,
2868                                      flow_dv_encap_decap_resource_match,
2869                                      (void *)resource);
2870         if (entry) {
2871                 cache_resource = container_of(entry,
2872                         struct mlx5_flow_dv_encap_decap_resource, entry);
2873                 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2874                         (void *)cache_resource,
2875                         rte_atomic32_read(&cache_resource->refcnt));
2876                 rte_atomic32_inc(&cache_resource->refcnt);
2877                 dev_flow->handle->dvh.rix_encap_decap = cache_resource->idx;
2878                 dev_flow->dv.encap_decap = cache_resource;
2879                 return 0;
2880         }
2881         /* Register new encap/decap resource. */
2882         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2883                                        &dev_flow->handle->dvh.rix_encap_decap);
2884         if (!cache_resource)
2885                 return rte_flow_error_set(error, ENOMEM,
2886                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2887                                           "cannot allocate resource memory");
2888         *cache_resource = *resource;
2889         cache_resource->idx = dev_flow->handle->dvh.rix_encap_decap;
2890         ret = mlx5_flow_os_create_flow_action_packet_reformat
2891                                         (sh->ctx, domain, cache_resource,
2892                                          &cache_resource->action);
2893         if (ret) {
2894                 mlx5_free(cache_resource);
2895                 return rte_flow_error_set(error, ENOMEM,
2896                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2897                                           NULL, "cannot create action");
2898         }
2899         rte_atomic32_init(&cache_resource->refcnt);
2900         rte_atomic32_inc(&cache_resource->refcnt);
2901         if (mlx5_hlist_insert_ex(sh->encaps_decaps, &cache_resource->entry,
2902                                  flow_dv_encap_decap_resource_match,
2903                                  (void *)cache_resource)) {
2904                 claim_zero(mlx5_flow_os_destroy_flow_action
2905                                                 (cache_resource->action));
2906                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2907                                 cache_resource->idx);
2908                 return rte_flow_error_set(error, EEXIST,
2909                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2910                                           NULL, "action exist");
2911         }
2912         dev_flow->dv.encap_decap = cache_resource;
2913         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2914                 (void *)cache_resource,
2915                 rte_atomic32_read(&cache_resource->refcnt));
2916         return 0;
2917 }
2918
2919 /**
2920  * Find existing table jump resource or create and register a new one.
2921  *
2922  * @param[in, out] dev
2923  *   Pointer to rte_eth_dev structure.
2924  * @param[in, out] tbl
2925  *   Pointer to flow table resource.
2926  * @parm[in, out] dev_flow
2927  *   Pointer to the dev_flow.
2928  * @param[out] error
2929  *   pointer to error structure.
2930  *
2931  * @return
2932  *   0 on success otherwise -errno and errno is set.
2933  */
2934 static int
2935 flow_dv_jump_tbl_resource_register
2936                         (struct rte_eth_dev *dev __rte_unused,
2937                          struct mlx5_flow_tbl_resource *tbl,
2938                          struct mlx5_flow *dev_flow,
2939                          struct rte_flow_error *error)
2940 {
2941         struct mlx5_flow_tbl_data_entry *tbl_data =
2942                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2943         int cnt, ret;
2944
2945         MLX5_ASSERT(tbl);
2946         cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
2947         if (!cnt) {
2948                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
2949                                 (tbl->obj, &tbl_data->jump.action);
2950                 if (ret)
2951                         return rte_flow_error_set(error, ENOMEM,
2952                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2953                                         NULL, "cannot create jump action");
2954                 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2955                         (void *)&tbl_data->jump, cnt);
2956         } else {
2957                 /* old jump should not make the table ref++. */
2958                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
2959                 MLX5_ASSERT(tbl_data->jump.action);
2960                 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2961                         (void *)&tbl_data->jump, cnt);
2962         }
2963         rte_atomic32_inc(&tbl_data->jump.refcnt);
2964         dev_flow->handle->rix_jump = tbl_data->idx;
2965         dev_flow->dv.jump = &tbl_data->jump;
2966         return 0;
2967 }
2968
2969 /**
2970  * Find existing default miss resource or create and register a new one.
2971  *
2972  * @param[in, out] dev
2973  *   Pointer to rte_eth_dev structure.
2974  * @param[out] error
2975  *   pointer to error structure.
2976  *
2977  * @return
2978  *   0 on success otherwise -errno and errno is set.
2979  */
2980 static int
2981 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
2982                 struct rte_flow_error *error)
2983 {
2984         struct mlx5_priv *priv = dev->data->dev_private;
2985         struct mlx5_dev_ctx_shared *sh = priv->sh;
2986         struct mlx5_flow_default_miss_resource *cache_resource =
2987                         &sh->default_miss;
2988         int cnt = rte_atomic32_read(&cache_resource->refcnt);
2989
2990         if (!cnt) {
2991                 MLX5_ASSERT(cache_resource->action);
2992                 cache_resource->action =
2993                 mlx5_glue->dr_create_flow_action_default_miss();
2994                 if (!cache_resource->action)
2995                         return rte_flow_error_set(error, ENOMEM,
2996                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2997                                         "cannot create default miss action");
2998                 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++",
2999                                 (void *)cache_resource->action, cnt);
3000         }
3001         rte_atomic32_inc(&cache_resource->refcnt);
3002         return 0;
3003 }
3004
3005 /**
3006  * Find existing table port ID resource or create and register a new one.
3007  *
3008  * @param[in, out] dev
3009  *   Pointer to rte_eth_dev structure.
3010  * @param[in, out] resource
3011  *   Pointer to port ID action resource.
3012  * @parm[in, out] dev_flow
3013  *   Pointer to the dev_flow.
3014  * @param[out] error
3015  *   pointer to error structure.
3016  *
3017  * @return
3018  *   0 on success otherwise -errno and errno is set.
3019  */
3020 static int
3021 flow_dv_port_id_action_resource_register
3022                         (struct rte_eth_dev *dev,
3023                          struct mlx5_flow_dv_port_id_action_resource *resource,
3024                          struct mlx5_flow *dev_flow,
3025                          struct rte_flow_error *error)
3026 {
3027         struct mlx5_priv *priv = dev->data->dev_private;
3028         struct mlx5_dev_ctx_shared *sh = priv->sh;
3029         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
3030         uint32_t idx = 0;
3031         int ret;
3032
3033         /* Lookup a matching resource from cache. */
3034         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list,
3035                       idx, cache_resource, next) {
3036                 if (resource->port_id == cache_resource->port_id) {
3037                         DRV_LOG(DEBUG, "port id action resource resource %p: "
3038                                 "refcnt %d++",
3039                                 (void *)cache_resource,
3040                                 rte_atomic32_read(&cache_resource->refcnt));
3041                         rte_atomic32_inc(&cache_resource->refcnt);
3042                         dev_flow->handle->rix_port_id_action = idx;
3043                         dev_flow->dv.port_id_action = cache_resource;
3044                         return 0;
3045                 }
3046         }
3047         /* Register new port id action resource. */
3048         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID],
3049                                        &dev_flow->handle->rix_port_id_action);
3050         if (!cache_resource)
3051                 return rte_flow_error_set(error, ENOMEM,
3052                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3053                                           "cannot allocate resource memory");
3054         *cache_resource = *resource;
3055         ret = mlx5_flow_os_create_flow_action_dest_port
3056                                 (priv->sh->fdb_domain, resource->port_id,
3057                                  &cache_resource->action);
3058         if (ret) {
3059                 mlx5_free(cache_resource);
3060                 return rte_flow_error_set(error, ENOMEM,
3061                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3062                                           NULL, "cannot create action");
3063         }
3064         rte_atomic32_init(&cache_resource->refcnt);
3065         rte_atomic32_inc(&cache_resource->refcnt);
3066         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
3067                      dev_flow->handle->rix_port_id_action, cache_resource,
3068                      next);
3069         dev_flow->dv.port_id_action = cache_resource;
3070         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
3071                 (void *)cache_resource,
3072                 rte_atomic32_read(&cache_resource->refcnt));
3073         return 0;
3074 }
3075
3076 /**
3077  * Find existing push vlan resource or create and register a new one.
3078  *
3079  * @param [in, out] dev
3080  *   Pointer to rte_eth_dev structure.
3081  * @param[in, out] resource
3082  *   Pointer to port ID action resource.
3083  * @parm[in, out] dev_flow
3084  *   Pointer to the dev_flow.
3085  * @param[out] error
3086  *   pointer to error structure.
3087  *
3088  * @return
3089  *   0 on success otherwise -errno and errno is set.
3090  */
3091 static int
3092 flow_dv_push_vlan_action_resource_register
3093                        (struct rte_eth_dev *dev,
3094                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
3095                         struct mlx5_flow *dev_flow,
3096                         struct rte_flow_error *error)
3097 {
3098         struct mlx5_priv *priv = dev->data->dev_private;
3099         struct mlx5_dev_ctx_shared *sh = priv->sh;
3100         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
3101         struct mlx5dv_dr_domain *domain;
3102         uint32_t idx = 0;
3103         int ret;
3104
3105         /* Lookup a matching resource from cache. */
3106         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3107                       sh->push_vlan_action_list, idx, cache_resource, next) {
3108                 if (resource->vlan_tag == cache_resource->vlan_tag &&
3109                     resource->ft_type == cache_resource->ft_type) {
3110                         DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
3111                                 "refcnt %d++",
3112                                 (void *)cache_resource,
3113                                 rte_atomic32_read(&cache_resource->refcnt));
3114                         rte_atomic32_inc(&cache_resource->refcnt);
3115                         dev_flow->handle->dvh.rix_push_vlan = idx;
3116                         dev_flow->dv.push_vlan_res = cache_resource;
3117                         return 0;
3118                 }
3119         }
3120         /* Register new push_vlan action resource. */
3121         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3122                                        &dev_flow->handle->dvh.rix_push_vlan);
3123         if (!cache_resource)
3124                 return rte_flow_error_set(error, ENOMEM,
3125                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3126                                           "cannot allocate resource memory");
3127         *cache_resource = *resource;
3128         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3129                 domain = sh->fdb_domain;
3130         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3131                 domain = sh->rx_domain;
3132         else
3133                 domain = sh->tx_domain;
3134         ret = mlx5_flow_os_create_flow_action_push_vlan
3135                                         (domain, resource->vlan_tag,
3136                                          &cache_resource->action);
3137         if (ret) {
3138                 mlx5_free(cache_resource);
3139                 return rte_flow_error_set(error, ENOMEM,
3140                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3141                                           NULL, "cannot create action");
3142         }
3143         rte_atomic32_init(&cache_resource->refcnt);
3144         rte_atomic32_inc(&cache_resource->refcnt);
3145         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3146                      &sh->push_vlan_action_list,
3147                      dev_flow->handle->dvh.rix_push_vlan,
3148                      cache_resource, next);
3149         dev_flow->dv.push_vlan_res = cache_resource;
3150         DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
3151                 (void *)cache_resource,
3152                 rte_atomic32_read(&cache_resource->refcnt));
3153         return 0;
3154 }
3155 /**
3156  * Get the size of specific rte_flow_item_type hdr size
3157  *
3158  * @param[in] item_type
3159  *   Tested rte_flow_item_type.
3160  *
3161  * @return
3162  *   sizeof struct item_type, 0 if void or irrelevant.
3163  */
3164 static size_t
3165 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
3166 {
3167         size_t retval;
3168
3169         switch (item_type) {
3170         case RTE_FLOW_ITEM_TYPE_ETH:
3171                 retval = sizeof(struct rte_ether_hdr);
3172                 break;
3173         case RTE_FLOW_ITEM_TYPE_VLAN:
3174                 retval = sizeof(struct rte_vlan_hdr);
3175                 break;
3176         case RTE_FLOW_ITEM_TYPE_IPV4:
3177                 retval = sizeof(struct rte_ipv4_hdr);
3178                 break;
3179         case RTE_FLOW_ITEM_TYPE_IPV6:
3180                 retval = sizeof(struct rte_ipv6_hdr);
3181                 break;
3182         case RTE_FLOW_ITEM_TYPE_UDP:
3183                 retval = sizeof(struct rte_udp_hdr);
3184                 break;
3185         case RTE_FLOW_ITEM_TYPE_TCP:
3186                 retval = sizeof(struct rte_tcp_hdr);
3187                 break;
3188         case RTE_FLOW_ITEM_TYPE_VXLAN:
3189         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3190                 retval = sizeof(struct rte_vxlan_hdr);
3191                 break;
3192         case RTE_FLOW_ITEM_TYPE_GRE:
3193         case RTE_FLOW_ITEM_TYPE_NVGRE:
3194                 retval = sizeof(struct rte_gre_hdr);
3195                 break;
3196         case RTE_FLOW_ITEM_TYPE_MPLS:
3197                 retval = sizeof(struct rte_mpls_hdr);
3198                 break;
3199         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
3200         default:
3201                 retval = 0;
3202                 break;
3203         }
3204         return retval;
3205 }
3206
3207 #define MLX5_ENCAP_IPV4_VERSION         0x40
3208 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
3209 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
3210 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
3211 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
3212 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
3213 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
3214
3215 /**
3216  * Convert the encap action data from list of rte_flow_item to raw buffer
3217  *
3218  * @param[in] items
3219  *   Pointer to rte_flow_item objects list.
3220  * @param[out] buf
3221  *   Pointer to the output buffer.
3222  * @param[out] size
3223  *   Pointer to the output buffer size.
3224  * @param[out] error
3225  *   Pointer to the error structure.
3226  *
3227  * @return
3228  *   0 on success, a negative errno value otherwise and rte_errno is set.
3229  */
3230 static int
3231 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
3232                            size_t *size, struct rte_flow_error *error)
3233 {
3234         struct rte_ether_hdr *eth = NULL;
3235         struct rte_vlan_hdr *vlan = NULL;
3236         struct rte_ipv4_hdr *ipv4 = NULL;
3237         struct rte_ipv6_hdr *ipv6 = NULL;
3238         struct rte_udp_hdr *udp = NULL;
3239         struct rte_vxlan_hdr *vxlan = NULL;
3240         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
3241         struct rte_gre_hdr *gre = NULL;
3242         size_t len;
3243         size_t temp_size = 0;
3244
3245         if (!items)
3246                 return rte_flow_error_set(error, EINVAL,
3247                                           RTE_FLOW_ERROR_TYPE_ACTION,
3248                                           NULL, "invalid empty data");
3249         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3250                 len = flow_dv_get_item_hdr_len(items->type);
3251                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
3252                         return rte_flow_error_set(error, EINVAL,
3253                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3254                                                   (void *)items->type,
3255                                                   "items total size is too big"
3256                                                   " for encap action");
3257                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
3258                 switch (items->type) {
3259                 case RTE_FLOW_ITEM_TYPE_ETH:
3260                         eth = (struct rte_ether_hdr *)&buf[temp_size];
3261                         break;
3262                 case RTE_FLOW_ITEM_TYPE_VLAN:
3263                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
3264                         if (!eth)
3265                                 return rte_flow_error_set(error, EINVAL,
3266                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3267                                                 (void *)items->type,
3268                                                 "eth header not found");
3269                         if (!eth->ether_type)
3270                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
3271                         break;
3272                 case RTE_FLOW_ITEM_TYPE_IPV4:
3273                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
3274                         if (!vlan && !eth)
3275                                 return rte_flow_error_set(error, EINVAL,
3276                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3277                                                 (void *)items->type,
3278                                                 "neither eth nor vlan"
3279                                                 " header found");
3280                         if (vlan && !vlan->eth_proto)
3281                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3282                         else if (eth && !eth->ether_type)
3283                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3284                         if (!ipv4->version_ihl)
3285                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
3286                                                     MLX5_ENCAP_IPV4_IHL_MIN;
3287                         if (!ipv4->time_to_live)
3288                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
3289                         break;
3290                 case RTE_FLOW_ITEM_TYPE_IPV6:
3291                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
3292                         if (!vlan && !eth)
3293                                 return rte_flow_error_set(error, EINVAL,
3294                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3295                                                 (void *)items->type,
3296                                                 "neither eth nor vlan"
3297                                                 " header found");
3298                         if (vlan && !vlan->eth_proto)
3299                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3300                         else if (eth && !eth->ether_type)
3301                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3302                         if (!ipv6->vtc_flow)
3303                                 ipv6->vtc_flow =
3304                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
3305                         if (!ipv6->hop_limits)
3306                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
3307                         break;
3308                 case RTE_FLOW_ITEM_TYPE_UDP:
3309                         udp = (struct rte_udp_hdr *)&buf[temp_size];
3310                         if (!ipv4 && !ipv6)
3311                                 return rte_flow_error_set(error, EINVAL,
3312                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3313                                                 (void *)items->type,
3314                                                 "ip header not found");
3315                         if (ipv4 && !ipv4->next_proto_id)
3316                                 ipv4->next_proto_id = IPPROTO_UDP;
3317                         else if (ipv6 && !ipv6->proto)
3318                                 ipv6->proto = IPPROTO_UDP;
3319                         break;
3320                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3321                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
3322                         if (!udp)
3323                                 return rte_flow_error_set(error, EINVAL,
3324                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3325                                                 (void *)items->type,
3326                                                 "udp header not found");
3327                         if (!udp->dst_port)
3328                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
3329                         if (!vxlan->vx_flags)
3330                                 vxlan->vx_flags =
3331                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
3332                         break;
3333                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3334                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
3335                         if (!udp)
3336                                 return rte_flow_error_set(error, EINVAL,
3337                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3338                                                 (void *)items->type,
3339                                                 "udp header not found");
3340                         if (!vxlan_gpe->proto)
3341                                 return rte_flow_error_set(error, EINVAL,
3342                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3343                                                 (void *)items->type,
3344                                                 "next protocol not found");
3345                         if (!udp->dst_port)
3346                                 udp->dst_port =
3347                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
3348                         if (!vxlan_gpe->vx_flags)
3349                                 vxlan_gpe->vx_flags =
3350                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
3351                         break;
3352                 case RTE_FLOW_ITEM_TYPE_GRE:
3353                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3354                         gre = (struct rte_gre_hdr *)&buf[temp_size];
3355                         if (!gre->proto)
3356                                 return rte_flow_error_set(error, EINVAL,
3357                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3358                                                 (void *)items->type,
3359                                                 "next protocol not found");
3360                         if (!ipv4 && !ipv6)
3361                                 return rte_flow_error_set(error, EINVAL,
3362                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3363                                                 (void *)items->type,
3364                                                 "ip header not found");
3365                         if (ipv4 && !ipv4->next_proto_id)
3366                                 ipv4->next_proto_id = IPPROTO_GRE;
3367                         else if (ipv6 && !ipv6->proto)
3368                                 ipv6->proto = IPPROTO_GRE;
3369                         break;
3370                 case RTE_FLOW_ITEM_TYPE_VOID:
3371                         break;
3372                 default:
3373                         return rte_flow_error_set(error, EINVAL,
3374                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3375                                                   (void *)items->type,
3376                                                   "unsupported item type");
3377                         break;
3378                 }
3379                 temp_size += len;
3380         }
3381         *size = temp_size;
3382         return 0;
3383 }
3384
3385 static int
3386 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
3387 {
3388         struct rte_ether_hdr *eth = NULL;
3389         struct rte_vlan_hdr *vlan = NULL;
3390         struct rte_ipv6_hdr *ipv6 = NULL;
3391         struct rte_udp_hdr *udp = NULL;
3392         char *next_hdr;
3393         uint16_t proto;
3394
3395         eth = (struct rte_ether_hdr *)data;
3396         next_hdr = (char *)(eth + 1);
3397         proto = RTE_BE16(eth->ether_type);
3398
3399         /* VLAN skipping */
3400         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
3401                 vlan = (struct rte_vlan_hdr *)next_hdr;
3402                 proto = RTE_BE16(vlan->eth_proto);
3403                 next_hdr += sizeof(struct rte_vlan_hdr);
3404         }
3405
3406         /* HW calculates IPv4 csum. no need to proceed */
3407         if (proto == RTE_ETHER_TYPE_IPV4)
3408                 return 0;
3409
3410         /* non IPv4/IPv6 header. not supported */
3411         if (proto != RTE_ETHER_TYPE_IPV6) {
3412                 return rte_flow_error_set(error, ENOTSUP,
3413                                           RTE_FLOW_ERROR_TYPE_ACTION,
3414                                           NULL, "Cannot offload non IPv4/IPv6");
3415         }
3416
3417         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
3418
3419         /* ignore non UDP */
3420         if (ipv6->proto != IPPROTO_UDP)
3421                 return 0;
3422
3423         udp = (struct rte_udp_hdr *)(ipv6 + 1);
3424         udp->dgram_cksum = 0;
3425
3426         return 0;
3427 }
3428
3429 /**
3430  * Convert L2 encap action to DV specification.
3431  *
3432  * @param[in] dev
3433  *   Pointer to rte_eth_dev structure.
3434  * @param[in] action
3435  *   Pointer to action structure.
3436  * @param[in, out] dev_flow
3437  *   Pointer to the mlx5_flow.
3438  * @param[in] transfer
3439  *   Mark if the flow is E-Switch flow.
3440  * @param[out] error
3441  *   Pointer to the error structure.
3442  *
3443  * @return
3444  *   0 on success, a negative errno value otherwise and rte_errno is set.
3445  */
3446 static int
3447 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
3448                                const struct rte_flow_action *action,
3449                                struct mlx5_flow *dev_flow,
3450                                uint8_t transfer,
3451                                struct rte_flow_error *error)
3452 {
3453         const struct rte_flow_item *encap_data;
3454         const struct rte_flow_action_raw_encap *raw_encap_data;
3455         struct mlx5_flow_dv_encap_decap_resource res = {
3456                 .reformat_type =
3457                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
3458                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3459                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
3460         };
3461
3462         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3463                 raw_encap_data =
3464                         (const struct rte_flow_action_raw_encap *)action->conf;
3465                 res.size = raw_encap_data->size;
3466                 memcpy(res.buf, raw_encap_data->data, res.size);
3467         } else {
3468                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
3469                         encap_data =
3470                                 ((const struct rte_flow_action_vxlan_encap *)
3471                                                 action->conf)->definition;
3472                 else
3473                         encap_data =
3474                                 ((const struct rte_flow_action_nvgre_encap *)
3475                                                 action->conf)->definition;
3476                 if (flow_dv_convert_encap_data(encap_data, res.buf,
3477                                                &res.size, error))
3478                         return -rte_errno;
3479         }
3480         if (flow_dv_zero_encap_udp_csum(res.buf, error))
3481                 return -rte_errno;
3482         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3483                 return rte_flow_error_set(error, EINVAL,
3484                                           RTE_FLOW_ERROR_TYPE_ACTION,
3485                                           NULL, "can't create L2 encap action");
3486         return 0;
3487 }
3488
3489 /**
3490  * Convert L2 decap action to DV specification.
3491  *
3492  * @param[in] dev
3493  *   Pointer to rte_eth_dev structure.
3494  * @param[in, out] dev_flow
3495  *   Pointer to the mlx5_flow.
3496  * @param[in] transfer
3497  *   Mark if the flow is E-Switch flow.
3498  * @param[out] error
3499  *   Pointer to the error structure.
3500  *
3501  * @return
3502  *   0 on success, a negative errno value otherwise and rte_errno is set.
3503  */
3504 static int
3505 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
3506                                struct mlx5_flow *dev_flow,
3507                                uint8_t transfer,
3508                                struct rte_flow_error *error)
3509 {
3510         struct mlx5_flow_dv_encap_decap_resource res = {
3511                 .size = 0,
3512                 .reformat_type =
3513                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
3514                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3515                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
3516         };
3517
3518         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3519                 return rte_flow_error_set(error, EINVAL,
3520                                           RTE_FLOW_ERROR_TYPE_ACTION,
3521                                           NULL, "can't create L2 decap action");
3522         return 0;
3523 }
3524
3525 /**
3526  * Convert raw decap/encap (L3 tunnel) action to DV specification.
3527  *
3528  * @param[in] dev
3529  *   Pointer to rte_eth_dev structure.
3530  * @param[in] action
3531  *   Pointer to action structure.
3532  * @param[in, out] dev_flow
3533  *   Pointer to the mlx5_flow.
3534  * @param[in] attr
3535  *   Pointer to the flow attributes.
3536  * @param[out] error
3537  *   Pointer to the error structure.
3538  *
3539  * @return
3540  *   0 on success, a negative errno value otherwise and rte_errno is set.
3541  */
3542 static int
3543 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
3544                                 const struct rte_flow_action *action,
3545                                 struct mlx5_flow *dev_flow,
3546                                 const struct rte_flow_attr *attr,
3547                                 struct rte_flow_error *error)
3548 {
3549         const struct rte_flow_action_raw_encap *encap_data;
3550         struct mlx5_flow_dv_encap_decap_resource res;
3551
3552         memset(&res, 0, sizeof(res));
3553         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
3554         res.size = encap_data->size;
3555         memcpy(res.buf, encap_data->data, res.size);
3556         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
3557                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
3558                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3559         if (attr->transfer)
3560                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3561         else
3562                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3563                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3564         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3565                 return rte_flow_error_set(error, EINVAL,
3566                                           RTE_FLOW_ERROR_TYPE_ACTION,
3567                                           NULL, "can't create encap action");
3568         return 0;
3569 }
3570
3571 /**
3572  * Create action push VLAN.
3573  *
3574  * @param[in] dev
3575  *   Pointer to rte_eth_dev structure.
3576  * @param[in] attr
3577  *   Pointer to the flow attributes.
3578  * @param[in] vlan
3579  *   Pointer to the vlan to push to the Ethernet header.
3580  * @param[in, out] dev_flow
3581  *   Pointer to the mlx5_flow.
3582  * @param[out] error
3583  *   Pointer to the error structure.
3584  *
3585  * @return
3586  *   0 on success, a negative errno value otherwise and rte_errno is set.
3587  */
3588 static int
3589 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
3590                                 const struct rte_flow_attr *attr,
3591                                 const struct rte_vlan_hdr *vlan,
3592                                 struct mlx5_flow *dev_flow,
3593                                 struct rte_flow_error *error)
3594 {
3595         struct mlx5_flow_dv_push_vlan_action_resource res;
3596
3597         memset(&res, 0, sizeof(res));
3598         res.vlan_tag =
3599                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
3600                                  vlan->vlan_tci);
3601         if (attr->transfer)
3602                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3603         else
3604                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3605                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3606         return flow_dv_push_vlan_action_resource_register
3607                                             (dev, &res, dev_flow, error);
3608 }
3609
3610 static int fdb_mirror;
3611
3612 /**
3613  * Validate the modify-header actions.
3614  *
3615  * @param[in] action_flags
3616  *   Holds the actions detected until now.
3617  * @param[in] action
3618  *   Pointer to the modify action.
3619  * @param[out] error
3620  *   Pointer to error structure.
3621  *
3622  * @return
3623  *   0 on success, a negative errno value otherwise and rte_errno is set.
3624  */
3625 static int
3626 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
3627                                    const struct rte_flow_action *action,
3628                                    struct rte_flow_error *error)
3629 {
3630         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
3631                 return rte_flow_error_set(error, EINVAL,
3632                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3633                                           NULL, "action configuration not set");
3634         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3635                 return rte_flow_error_set(error, EINVAL,
3636                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3637                                           "can't have encap action before"
3638                                           " modify action");
3639         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror)
3640                 return rte_flow_error_set(error, EINVAL,
3641                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3642                                           "can't support sample action before"
3643                                           " modify action for E-Switch"
3644                                           " mirroring");
3645         return 0;
3646 }
3647
3648 /**
3649  * Validate the modify-header MAC address actions.
3650  *
3651  * @param[in] action_flags
3652  *   Holds the actions detected until now.
3653  * @param[in] action
3654  *   Pointer to the modify action.
3655  * @param[in] item_flags
3656  *   Holds the items detected.
3657  * @param[out] error
3658  *   Pointer to error structure.
3659  *
3660  * @return
3661  *   0 on success, a negative errno value otherwise and rte_errno is set.
3662  */
3663 static int
3664 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
3665                                    const struct rte_flow_action *action,
3666                                    const uint64_t item_flags,
3667                                    struct rte_flow_error *error)
3668 {
3669         int ret = 0;
3670
3671         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3672         if (!ret) {
3673                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
3674                         return rte_flow_error_set(error, EINVAL,
3675                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3676                                                   NULL,
3677                                                   "no L2 item in pattern");
3678         }
3679         return ret;
3680 }
3681
3682 /**
3683  * Validate the modify-header IPv4 address actions.
3684  *
3685  * @param[in] action_flags
3686  *   Holds the actions detected until now.
3687  * @param[in] action
3688  *   Pointer to the modify action.
3689  * @param[in] item_flags
3690  *   Holds the items detected.
3691  * @param[out] error
3692  *   Pointer to error structure.
3693  *
3694  * @return
3695  *   0 on success, a negative errno value otherwise and rte_errno is set.
3696  */
3697 static int
3698 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3699                                     const struct rte_flow_action *action,
3700                                     const uint64_t item_flags,
3701                                     struct rte_flow_error *error)
3702 {
3703         int ret = 0;
3704         uint64_t layer;
3705
3706         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3707         if (!ret) {
3708                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3709                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3710                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3711                 if (!(item_flags & layer))
3712                         return rte_flow_error_set(error, EINVAL,
3713                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3714                                                   NULL,
3715                                                   "no ipv4 item in pattern");
3716         }
3717         return ret;
3718 }
3719
3720 /**
3721  * Validate the modify-header IPv6 address actions.
3722  *
3723  * @param[in] action_flags
3724  *   Holds the actions detected until now.
3725  * @param[in] action
3726  *   Pointer to the modify action.
3727  * @param[in] item_flags
3728  *   Holds the items detected.
3729  * @param[out] error
3730  *   Pointer to error structure.
3731  *
3732  * @return
3733  *   0 on success, a negative errno value otherwise and rte_errno is set.
3734  */
3735 static int
3736 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3737                                     const struct rte_flow_action *action,
3738                                     const uint64_t item_flags,
3739                                     struct rte_flow_error *error)
3740 {
3741         int ret = 0;
3742         uint64_t layer;
3743
3744         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3745         if (!ret) {
3746                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3747                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3748                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3749                 if (!(item_flags & layer))
3750                         return rte_flow_error_set(error, EINVAL,
3751                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3752                                                   NULL,
3753                                                   "no ipv6 item in pattern");
3754         }
3755         return ret;
3756 }
3757
3758 /**
3759  * Validate the modify-header TP actions.
3760  *
3761  * @param[in] action_flags
3762  *   Holds the actions detected until now.
3763  * @param[in] action
3764  *   Pointer to the modify action.
3765  * @param[in] item_flags
3766  *   Holds the items detected.
3767  * @param[out] error
3768  *   Pointer to error structure.
3769  *
3770  * @return
3771  *   0 on success, a negative errno value otherwise and rte_errno is set.
3772  */
3773 static int
3774 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3775                                   const struct rte_flow_action *action,
3776                                   const uint64_t item_flags,
3777                                   struct rte_flow_error *error)
3778 {
3779         int ret = 0;
3780         uint64_t layer;
3781
3782         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3783         if (!ret) {
3784                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3785                                  MLX5_FLOW_LAYER_INNER_L4 :
3786                                  MLX5_FLOW_LAYER_OUTER_L4;
3787                 if (!(item_flags & layer))
3788                         return rte_flow_error_set(error, EINVAL,
3789                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3790                                                   NULL, "no transport layer "
3791                                                   "in pattern");
3792         }
3793         return ret;
3794 }
3795
3796 /**
3797  * Validate the modify-header actions of increment/decrement
3798  * TCP Sequence-number.
3799  *
3800  * @param[in] action_flags
3801  *   Holds the actions detected until now.
3802  * @param[in] action
3803  *   Pointer to the modify action.
3804  * @param[in] item_flags
3805  *   Holds the items detected.
3806  * @param[out] error
3807  *   Pointer to error structure.
3808  *
3809  * @return
3810  *   0 on success, a negative errno value otherwise and rte_errno is set.
3811  */
3812 static int
3813 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3814                                        const struct rte_flow_action *action,
3815                                        const uint64_t item_flags,
3816                                        struct rte_flow_error *error)
3817 {
3818         int ret = 0;
3819         uint64_t layer;
3820
3821         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3822         if (!ret) {
3823                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3824                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3825                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3826                 if (!(item_flags & layer))
3827                         return rte_flow_error_set(error, EINVAL,
3828                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3829                                                   NULL, "no TCP item in"
3830                                                   " pattern");
3831                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3832                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3833                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3834                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3835                         return rte_flow_error_set(error, EINVAL,
3836                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3837                                                   NULL,
3838                                                   "cannot decrease and increase"
3839                                                   " TCP sequence number"
3840                                                   " at the same time");
3841         }
3842         return ret;
3843 }
3844
3845 /**
3846  * Validate the modify-header actions of increment/decrement
3847  * TCP Acknowledgment number.
3848  *
3849  * @param[in] action_flags
3850  *   Holds the actions detected until now.
3851  * @param[in] action
3852  *   Pointer to the modify action.
3853  * @param[in] item_flags
3854  *   Holds the items detected.
3855  * @param[out] error
3856  *   Pointer to error structure.
3857  *
3858  * @return
3859  *   0 on success, a negative errno value otherwise and rte_errno is set.
3860  */
3861 static int
3862 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3863                                        const struct rte_flow_action *action,
3864                                        const uint64_t item_flags,
3865                                        struct rte_flow_error *error)
3866 {
3867         int ret = 0;
3868         uint64_t layer;
3869
3870         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3871         if (!ret) {
3872                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3873                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3874                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3875                 if (!(item_flags & layer))
3876                         return rte_flow_error_set(error, EINVAL,
3877                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3878                                                   NULL, "no TCP item in"
3879                                                   " pattern");
3880                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3881                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3882                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3883                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3884                         return rte_flow_error_set(error, EINVAL,
3885                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3886                                                   NULL,
3887                                                   "cannot decrease and increase"
3888                                                   " TCP acknowledgment number"
3889                                                   " at the same time");
3890         }
3891         return ret;
3892 }
3893
3894 /**
3895  * Validate the modify-header TTL actions.
3896  *
3897  * @param[in] action_flags
3898  *   Holds the actions detected until now.
3899  * @param[in] action
3900  *   Pointer to the modify action.
3901  * @param[in] item_flags
3902  *   Holds the items detected.
3903  * @param[out] error
3904  *   Pointer to error structure.
3905  *
3906  * @return
3907  *   0 on success, a negative errno value otherwise and rte_errno is set.
3908  */
3909 static int
3910 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3911                                    const struct rte_flow_action *action,
3912                                    const uint64_t item_flags,
3913                                    struct rte_flow_error *error)
3914 {
3915         int ret = 0;
3916         uint64_t layer;
3917
3918         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3919         if (!ret) {
3920                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3921                                  MLX5_FLOW_LAYER_INNER_L3 :
3922                                  MLX5_FLOW_LAYER_OUTER_L3;
3923                 if (!(item_flags & layer))
3924                         return rte_flow_error_set(error, EINVAL,
3925                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3926                                                   NULL,
3927                                                   "no IP protocol in pattern");
3928         }
3929         return ret;
3930 }
3931
3932 /**
3933  * Validate jump action.
3934  *
3935  * @param[in] action
3936  *   Pointer to the jump action.
3937  * @param[in] action_flags
3938  *   Holds the actions detected until now.
3939  * @param[in] attributes
3940  *   Pointer to flow attributes
3941  * @param[in] external
3942  *   Action belongs to flow rule created by request external to PMD.
3943  * @param[out] error
3944  *   Pointer to error structure.
3945  *
3946  * @return
3947  *   0 on success, a negative errno value otherwise and rte_errno is set.
3948  */
3949 static int
3950 flow_dv_validate_action_jump(const struct rte_flow_action *action,
3951                              uint64_t action_flags,
3952                              const struct rte_flow_attr *attributes,
3953                              bool external, struct rte_flow_error *error)
3954 {
3955         uint32_t target_group, table;
3956         int ret = 0;
3957
3958         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3959                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3960                 return rte_flow_error_set(error, EINVAL,
3961                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3962                                           "can't have 2 fate actions in"
3963                                           " same flow");
3964         if (action_flags & MLX5_FLOW_ACTION_METER)
3965                 return rte_flow_error_set(error, ENOTSUP,
3966                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3967                                           "jump with meter not support");
3968         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror)
3969                 return rte_flow_error_set(error, EINVAL,
3970                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3971                                           "E-Switch mirroring can't support"
3972                                           " Sample action and jump action in"
3973                                           " same flow now");
3974         if (!action->conf)
3975                 return rte_flow_error_set(error, EINVAL,
3976                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3977                                           NULL, "action configuration not set");
3978         target_group =
3979                 ((const struct rte_flow_action_jump *)action->conf)->group;
3980         ret = mlx5_flow_group_to_table(attributes, external, target_group,
3981                                        true, &table, error);
3982         if (ret)
3983                 return ret;
3984         if (attributes->group == target_group)
3985                 return rte_flow_error_set(error, EINVAL,
3986                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3987                                           "target group must be other than"
3988                                           " the current flow group");
3989         return 0;
3990 }
3991
3992 /*
3993  * Validate the port_id action.
3994  *
3995  * @param[in] dev
3996  *   Pointer to rte_eth_dev structure.
3997  * @param[in] action_flags
3998  *   Bit-fields that holds the actions detected until now.
3999  * @param[in] action
4000  *   Port_id RTE action structure.
4001  * @param[in] attr
4002  *   Attributes of flow that includes this action.
4003  * @param[out] error
4004  *   Pointer to error structure.
4005  *
4006  * @return
4007  *   0 on success, a negative errno value otherwise and rte_errno is set.
4008  */
4009 static int
4010 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
4011                                 uint64_t action_flags,
4012                                 const struct rte_flow_action *action,
4013                                 const struct rte_flow_attr *attr,
4014                                 struct rte_flow_error *error)
4015 {
4016         const struct rte_flow_action_port_id *port_id;
4017         struct mlx5_priv *act_priv;
4018         struct mlx5_priv *dev_priv;
4019         uint16_t port;
4020
4021         if (!attr->transfer)
4022                 return rte_flow_error_set(error, ENOTSUP,
4023                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4024                                           NULL,
4025                                           "port id action is valid in transfer"
4026                                           " mode only");
4027         if (!action || !action->conf)
4028                 return rte_flow_error_set(error, ENOTSUP,
4029                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
4030                                           NULL,
4031                                           "port id action parameters must be"
4032                                           " specified");
4033         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
4034                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
4035                 return rte_flow_error_set(error, EINVAL,
4036                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4037                                           "can have only one fate actions in"
4038                                           " a flow");
4039         dev_priv = mlx5_dev_to_eswitch_info(dev);
4040         if (!dev_priv)
4041                 return rte_flow_error_set(error, rte_errno,
4042                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4043                                           NULL,
4044                                           "failed to obtain E-Switch info");
4045         port_id = action->conf;
4046         port = port_id->original ? dev->data->port_id : port_id->id;
4047         act_priv = mlx5_port_to_eswitch_info(port, false);
4048         if (!act_priv)
4049                 return rte_flow_error_set
4050                                 (error, rte_errno,
4051                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
4052                                  "failed to obtain E-Switch port id for port");
4053         if (act_priv->domain_id != dev_priv->domain_id)
4054                 return rte_flow_error_set
4055                                 (error, EINVAL,
4056                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4057                                  "port does not belong to"
4058                                  " E-Switch being configured");
4059         return 0;
4060 }
4061
4062 /**
4063  * Get the maximum number of modify header actions.
4064  *
4065  * @param dev
4066  *   Pointer to rte_eth_dev structure.
4067  * @param flags
4068  *   Flags bits to check if root level.
4069  *
4070  * @return
4071  *   Max number of modify header actions device can support.
4072  */
4073 static inline unsigned int
4074 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
4075                               uint64_t flags)
4076 {
4077         /*
4078          * There's no way to directly query the max capacity from FW.
4079          * The maximal value on root table should be assumed to be supported.
4080          */
4081         if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
4082                 return MLX5_MAX_MODIFY_NUM;
4083         else
4084                 return MLX5_ROOT_TBL_MODIFY_NUM;
4085 }
4086
4087 /**
4088  * Validate the meter action.
4089  *
4090  * @param[in] dev
4091  *   Pointer to rte_eth_dev structure.
4092  * @param[in] action_flags
4093  *   Bit-fields that holds the actions detected until now.
4094  * @param[in] action
4095  *   Pointer to the meter action.
4096  * @param[in] attr
4097  *   Attributes of flow that includes this action.
4098  * @param[out] error
4099  *   Pointer to error structure.
4100  *
4101  * @return
4102  *   0 on success, a negative errno value otherwise and rte_ernno is set.
4103  */
4104 static int
4105 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
4106                                 uint64_t action_flags,
4107                                 const struct rte_flow_action *action,
4108                                 const struct rte_flow_attr *attr,
4109                                 struct rte_flow_error *error)
4110 {
4111         struct mlx5_priv *priv = dev->data->dev_private;
4112         const struct rte_flow_action_meter *am = action->conf;
4113         struct mlx5_flow_meter *fm;
4114
4115         if (!am)
4116                 return rte_flow_error_set(error, EINVAL,
4117                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4118                                           "meter action conf is NULL");
4119
4120         if (action_flags & MLX5_FLOW_ACTION_METER)
4121                 return rte_flow_error_set(error, ENOTSUP,
4122                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4123                                           "meter chaining not support");
4124         if (action_flags & MLX5_FLOW_ACTION_JUMP)
4125                 return rte_flow_error_set(error, ENOTSUP,
4126                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4127                                           "meter with jump not support");
4128         if (!priv->mtr_en)
4129                 return rte_flow_error_set(error, ENOTSUP,
4130                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4131                                           NULL,
4132                                           "meter action not supported");
4133         fm = mlx5_flow_meter_find(priv, am->mtr_id);
4134         if (!fm)
4135                 return rte_flow_error_set(error, EINVAL,
4136                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4137                                           "Meter not found");
4138         if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
4139               (!fm->ingress && !attr->ingress && attr->egress) ||
4140               (!fm->egress && !attr->egress && attr->ingress))))
4141                 return rte_flow_error_set(error, EINVAL,
4142                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4143                                           "Flow attributes are either invalid "
4144                                           "or have a conflict with current "
4145                                           "meter attributes");
4146         return 0;
4147 }
4148
4149 /**
4150  * Validate the age action.
4151  *
4152  * @param[in] action_flags
4153  *   Holds the actions detected until now.
4154  * @param[in] action
4155  *   Pointer to the age action.
4156  * @param[in] dev
4157  *   Pointer to the Ethernet device structure.
4158  * @param[out] error
4159  *   Pointer to error structure.
4160  *
4161  * @return
4162  *   0 on success, a negative errno value otherwise and rte_errno is set.
4163  */
4164 static int
4165 flow_dv_validate_action_age(uint64_t action_flags,
4166                             const struct rte_flow_action *action,
4167                             struct rte_eth_dev *dev,
4168                             struct rte_flow_error *error)
4169 {
4170         struct mlx5_priv *priv = dev->data->dev_private;
4171         const struct rte_flow_action_age *age = action->conf;
4172
4173         if (!priv->config.devx || priv->sh->cmng.counter_fallback)
4174                 return rte_flow_error_set(error, ENOTSUP,
4175                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4176                                           NULL,
4177                                           "age action not supported");
4178         if (!(action->conf))
4179                 return rte_flow_error_set(error, EINVAL,
4180                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4181                                           "configuration cannot be null");
4182         if (!(age->timeout))
4183                 return rte_flow_error_set(error, EINVAL,
4184                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4185                                           "invalid timeout value 0");
4186         if (action_flags & MLX5_FLOW_ACTION_AGE)
4187                 return rte_flow_error_set(error, EINVAL,
4188                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4189                                           "duplicate age actions set");
4190         return 0;
4191 }
4192
4193 /**
4194  * Validate the modify-header IPv4 DSCP actions.
4195  *
4196  * @param[in] action_flags
4197  *   Holds the actions detected until now.
4198  * @param[in] action
4199  *   Pointer to the modify action.
4200  * @param[in] item_flags
4201  *   Holds the items detected.
4202  * @param[out] error
4203  *   Pointer to error structure.
4204  *
4205  * @return
4206  *   0 on success, a negative errno value otherwise and rte_errno is set.
4207  */
4208 static int
4209 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
4210                                          const struct rte_flow_action *action,
4211                                          const uint64_t item_flags,
4212                                          struct rte_flow_error *error)
4213 {
4214         int ret = 0;
4215
4216         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4217         if (!ret) {
4218                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
4219                         return rte_flow_error_set(error, EINVAL,
4220                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4221                                                   NULL,
4222                                                   "no ipv4 item in pattern");
4223         }
4224         return ret;
4225 }
4226
4227 /**
4228  * Validate the modify-header IPv6 DSCP actions.
4229  *
4230  * @param[in] action_flags
4231  *   Holds the actions detected until now.
4232  * @param[in] action
4233  *   Pointer to the modify action.
4234  * @param[in] item_flags
4235  *   Holds the items detected.
4236  * @param[out] error
4237  *   Pointer to error structure.
4238  *
4239  * @return
4240  *   0 on success, a negative errno value otherwise and rte_errno is set.
4241  */
4242 static int
4243 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
4244                                          const struct rte_flow_action *action,
4245                                          const uint64_t item_flags,
4246                                          struct rte_flow_error *error)
4247 {
4248         int ret = 0;
4249
4250         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4251         if (!ret) {
4252                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
4253                         return rte_flow_error_set(error, EINVAL,
4254                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4255                                                   NULL,
4256                                                   "no ipv6 item in pattern");
4257         }
4258         return ret;
4259 }
4260
4261 /**
4262  * Match modify-header resource.
4263  *
4264  * @param entry
4265  *   Pointer to exist resource entry object.
4266  * @param ctx
4267  *   Pointer to new modify-header resource.
4268  *
4269  * @return
4270  *   0 on matching, -1 otherwise.
4271  */
4272 static int
4273 flow_dv_modify_hdr_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
4274 {
4275         struct mlx5_flow_dv_modify_hdr_resource *resource;
4276         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4277         uint32_t actions_len;
4278
4279         resource = (struct mlx5_flow_dv_modify_hdr_resource *)ctx;
4280         cache_resource = container_of(entry,
4281                                       struct mlx5_flow_dv_modify_hdr_resource,
4282                                       entry);
4283         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4284         if (resource->entry.key == cache_resource->entry.key &&
4285             resource->ft_type == cache_resource->ft_type &&
4286             resource->actions_num == cache_resource->actions_num &&
4287             resource->flags == cache_resource->flags &&
4288             !memcmp((const void *)resource->actions,
4289                     (const void *)cache_resource->actions,
4290                     actions_len))
4291                 return 0;
4292         return -1;
4293 }
4294
4295 /**
4296  * Validate the sample action.
4297  *
4298  * @param[in] action_flags
4299  *   Holds the actions detected until now.
4300  * @param[in] action
4301  *   Pointer to the sample action.
4302  * @param[in] dev
4303  *   Pointer to the Ethernet device structure.
4304  * @param[in] attr
4305  *   Attributes of flow that includes this action.
4306  * @param[out] error
4307  *   Pointer to error structure.
4308  *
4309  * @return
4310  *   0 on success, a negative errno value otherwise and rte_errno is set.
4311  */
4312 static int
4313 flow_dv_validate_action_sample(uint64_t action_flags,
4314                                const struct rte_flow_action *action,
4315                                struct rte_eth_dev *dev,
4316                                const struct rte_flow_attr *attr,
4317                                struct rte_flow_error *error)
4318 {
4319         struct mlx5_priv *priv = dev->data->dev_private;
4320         struct mlx5_dev_config *dev_conf = &priv->config;
4321         const struct rte_flow_action_sample *sample = action->conf;
4322         const struct rte_flow_action *act;
4323         uint64_t sub_action_flags = 0;
4324         uint16_t queue_index = 0xFFFF;
4325         int actions_n = 0;
4326         int ret;
4327         fdb_mirror = 0;
4328
4329         if (!sample)
4330                 return rte_flow_error_set(error, EINVAL,
4331                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4332                                           "configuration cannot be NULL");
4333         if (sample->ratio == 0)
4334                 return rte_flow_error_set(error, EINVAL,
4335                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4336                                           "ratio value starts from 1");
4337         if (!priv->config.devx || (sample->ratio > 0 && !priv->sampler_en))
4338                 return rte_flow_error_set(error, ENOTSUP,
4339                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4340                                           NULL,
4341                                           "sample action not supported");
4342         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
4343                 return rte_flow_error_set(error, EINVAL,
4344                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4345                                           "Multiple sample actions not "
4346                                           "supported");
4347         if (action_flags & MLX5_FLOW_ACTION_METER)
4348                 return rte_flow_error_set(error, EINVAL,
4349                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4350                                           "wrong action order, meter should "
4351                                           "be after sample action");
4352         if (action_flags & MLX5_FLOW_ACTION_JUMP)
4353                 return rte_flow_error_set(error, EINVAL,
4354                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4355                                           "wrong action order, jump should "
4356                                           "be after sample action");
4357         act = sample->actions;
4358         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
4359                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
4360                         return rte_flow_error_set(error, ENOTSUP,
4361                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4362                                                   act, "too many actions");
4363                 switch (act->type) {
4364                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4365                         ret = mlx5_flow_validate_action_queue(act,
4366                                                               sub_action_flags,
4367                                                               dev,
4368                                                               attr, error);
4369                         if (ret < 0)
4370                                 return ret;
4371                         queue_index = ((const struct rte_flow_action_queue *)
4372                                                         (act->conf))->index;
4373                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
4374                         ++actions_n;
4375                         break;
4376                 case RTE_FLOW_ACTION_TYPE_MARK:
4377                         ret = flow_dv_validate_action_mark(dev, act,
4378                                                            sub_action_flags,
4379                                                            attr, error);
4380                         if (ret < 0)
4381                                 return ret;
4382                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
4383                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
4384                                                 MLX5_FLOW_ACTION_MARK_EXT;
4385                         else
4386                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
4387                         ++actions_n;
4388                         break;
4389                 case RTE_FLOW_ACTION_TYPE_COUNT:
4390                         ret = flow_dv_validate_action_count(dev, error);
4391                         if (ret < 0)
4392                                 return ret;
4393                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
4394                         ++actions_n;
4395                         break;
4396                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
4397                         ret = flow_dv_validate_action_port_id(dev,
4398                                                               sub_action_flags,
4399                                                               act,
4400                                                               attr,
4401                                                               error);
4402                         if (ret)
4403                                 return ret;
4404                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
4405                         ++actions_n;
4406                         break;
4407                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4408                         ret = flow_dv_validate_action_raw_encap_decap
4409                                 (dev, NULL, act->conf, attr, &sub_action_flags,
4410                                  &actions_n, error);
4411                         if (ret < 0)
4412                                 return ret;
4413                         ++actions_n;
4414                         break;
4415                 default:
4416                         return rte_flow_error_set(error, ENOTSUP,
4417                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4418                                                   NULL,
4419                                                   "Doesn't support optional "
4420                                                   "action");
4421                 }
4422         }
4423         if (attr->ingress && !attr->transfer) {
4424                 if (!(sub_action_flags & MLX5_FLOW_ACTION_QUEUE))
4425                         return rte_flow_error_set(error, EINVAL,
4426                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4427                                                   NULL,
4428                                                   "Ingress must has a dest "
4429                                                   "QUEUE for Sample");
4430         } else if (attr->egress && !attr->transfer) {
4431                 return rte_flow_error_set(error, ENOTSUP,
4432                                           RTE_FLOW_ERROR_TYPE_ACTION,
4433                                           NULL,
4434                                           "Sample Only support Ingress "
4435                                           "or E-Switch");
4436         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
4437                 MLX5_ASSERT(attr->transfer);
4438                 if (sample->ratio > 1)
4439                         return rte_flow_error_set(error, ENOTSUP,
4440                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4441                                                   NULL,
4442                                                   "E-Switch doesn't support "
4443                                                   "any optional action "
4444                                                   "for sampling");
4445                 fdb_mirror = 1;
4446                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
4447                         return rte_flow_error_set(error, ENOTSUP,
4448                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4449                                                   NULL,
4450                                                   "unsupported action QUEUE");
4451                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
4452                         return rte_flow_error_set(error, EINVAL,
4453                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4454                                                   NULL,
4455                                                   "E-Switch must has a dest "
4456                                                   "port for mirroring");
4457         }
4458         /* Continue validation for Xcap actions.*/
4459         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
4460             (queue_index == 0xFFFF ||
4461              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
4462                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
4463                      MLX5_FLOW_XCAP_ACTIONS)
4464                         return rte_flow_error_set(error, ENOTSUP,
4465                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4466                                                   NULL, "encap and decap "
4467                                                   "combination aren't "
4468                                                   "supported");
4469                 if (!attr->transfer && attr->ingress && (sub_action_flags &
4470                                                         MLX5_FLOW_ACTION_ENCAP))
4471                         return rte_flow_error_set(error, ENOTSUP,
4472                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4473                                                   NULL, "encap is not supported"
4474                                                   " for ingress traffic");
4475         }
4476         return 0;
4477 }
4478
4479 /**
4480  * Find existing modify-header resource or create and register a new one.
4481  *
4482  * @param dev[in, out]
4483  *   Pointer to rte_eth_dev structure.
4484  * @param[in, out] resource
4485  *   Pointer to modify-header resource.
4486  * @parm[in, out] dev_flow
4487  *   Pointer to the dev_flow.
4488  * @param[out] error
4489  *   pointer to error structure.
4490  *
4491  * @return
4492  *   0 on success otherwise -errno and errno is set.
4493  */
4494 static int
4495 flow_dv_modify_hdr_resource_register
4496                         (struct rte_eth_dev *dev,
4497                          struct mlx5_flow_dv_modify_hdr_resource *resource,
4498                          struct mlx5_flow *dev_flow,
4499                          struct rte_flow_error *error)
4500 {
4501         struct mlx5_priv *priv = dev->data->dev_private;
4502         struct mlx5_dev_ctx_shared *sh = priv->sh;
4503         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4504         struct mlx5dv_dr_domain *ns;
4505         uint32_t actions_len;
4506         struct mlx5_hlist_entry *entry;
4507         union mlx5_flow_modify_hdr_key hdr_mod_key = {
4508                 {
4509                         .ft_type = resource->ft_type,
4510                         .actions_num = resource->actions_num,
4511                         .group = dev_flow->dv.group,
4512                         .cksum = 0,
4513                 }
4514         };
4515         int ret;
4516
4517         resource->flags = dev_flow->dv.group ? 0 :
4518                           MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4519         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
4520                                     resource->flags))
4521                 return rte_flow_error_set(error, EOVERFLOW,
4522                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4523                                           "too many modify header items");
4524         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
4525                 ns = sh->fdb_domain;
4526         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
4527                 ns = sh->tx_domain;
4528         else
4529                 ns = sh->rx_domain;
4530         /* Lookup a matching resource from cache. */
4531         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4532         hdr_mod_key.cksum = __rte_raw_cksum(resource->actions, actions_len, 0);
4533         resource->entry.key = hdr_mod_key.v64;
4534         entry = mlx5_hlist_lookup_ex(sh->modify_cmds, resource->entry.key,
4535                                      flow_dv_modify_hdr_resource_match,
4536                                      (void *)resource);
4537         if (entry) {
4538                 cache_resource = container_of(entry,
4539                                         struct mlx5_flow_dv_modify_hdr_resource,
4540                                         entry);
4541                 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
4542                         (void *)cache_resource,
4543                         rte_atomic32_read(&cache_resource->refcnt));
4544                 rte_atomic32_inc(&cache_resource->refcnt);
4545                 dev_flow->handle->dvh.modify_hdr = cache_resource;
4546                 return 0;
4547
4548         }
4549         /* Register new modify-header resource. */
4550         cache_resource = mlx5_malloc(MLX5_MEM_ZERO,
4551                                     sizeof(*cache_resource) + actions_len, 0,
4552                                     SOCKET_ID_ANY);
4553         if (!cache_resource)
4554                 return rte_flow_error_set(error, ENOMEM,
4555                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4556                                           "cannot allocate resource memory");
4557         *cache_resource = *resource;
4558         rte_memcpy(cache_resource->actions, resource->actions, actions_len);
4559         ret = mlx5_flow_os_create_flow_action_modify_header
4560                                         (sh->ctx, ns, cache_resource,
4561                                          actions_len, &cache_resource->action);
4562         if (ret) {
4563                 mlx5_free(cache_resource);
4564                 return rte_flow_error_set(error, ENOMEM,
4565                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4566                                           NULL, "cannot create action");
4567         }
4568         rte_atomic32_init(&cache_resource->refcnt);
4569         rte_atomic32_inc(&cache_resource->refcnt);
4570         if (mlx5_hlist_insert_ex(sh->modify_cmds, &cache_resource->entry,
4571                                  flow_dv_modify_hdr_resource_match,
4572                                  (void *)cache_resource)) {
4573                 claim_zero(mlx5_flow_os_destroy_flow_action
4574                                                 (cache_resource->action));
4575                 mlx5_free(cache_resource);
4576                 return rte_flow_error_set(error, EEXIST,
4577                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4578                                           NULL, "action exist");
4579         }
4580         dev_flow->handle->dvh.modify_hdr = cache_resource;
4581         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
4582                 (void *)cache_resource,
4583                 rte_atomic32_read(&cache_resource->refcnt));
4584         return 0;
4585 }
4586
4587 /**
4588  * Get DV flow counter by index.
4589  *
4590  * @param[in] dev
4591  *   Pointer to the Ethernet device structure.
4592  * @param[in] idx
4593  *   mlx5 flow counter index in the container.
4594  * @param[out] ppool
4595  *   mlx5 flow counter pool in the container,
4596  *
4597  * @return
4598  *   Pointer to the counter, NULL otherwise.
4599  */
4600 static struct mlx5_flow_counter *
4601 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
4602                            uint32_t idx,
4603                            struct mlx5_flow_counter_pool **ppool)
4604 {
4605         struct mlx5_priv *priv = dev->data->dev_private;
4606         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4607         struct mlx5_flow_counter_pool *pool;
4608
4609         /* Decrease to original index and clear shared bit. */
4610         idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
4611         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cmng->n);
4612         pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
4613         MLX5_ASSERT(pool);
4614         if (ppool)
4615                 *ppool = pool;
4616         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
4617 }
4618
4619 /**
4620  * Check the devx counter belongs to the pool.
4621  *
4622  * @param[in] pool
4623  *   Pointer to the counter pool.
4624  * @param[in] id
4625  *   The counter devx ID.
4626  *
4627  * @return
4628  *   True if counter belongs to the pool, false otherwise.
4629  */
4630 static bool
4631 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
4632 {
4633         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
4634                    MLX5_COUNTERS_PER_POOL;
4635
4636         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
4637                 return true;
4638         return false;
4639 }
4640
4641 /**
4642  * Get a pool by devx counter ID.
4643  *
4644  * @param[in] cmng
4645  *   Pointer to the counter management.
4646  * @param[in] id
4647  *   The counter devx ID.
4648  *
4649  * @return
4650  *   The counter pool pointer if exists, NULL otherwise,
4651  */
4652 static struct mlx5_flow_counter_pool *
4653 flow_dv_find_pool_by_id(struct mlx5_flow_counter_mng *cmng, int id)
4654 {
4655         uint32_t i;
4656         struct mlx5_flow_counter_pool *pool = NULL;
4657
4658         rte_spinlock_lock(&cmng->pool_update_sl);
4659         /* Check last used pool. */
4660         if (cmng->last_pool_idx != POOL_IDX_INVALID &&
4661             flow_dv_is_counter_in_pool(cmng->pools[cmng->last_pool_idx], id)) {
4662                 pool = cmng->pools[cmng->last_pool_idx];
4663                 goto out;
4664         }
4665         /* ID out of range means no suitable pool in the container. */
4666         if (id > cmng->max_id || id < cmng->min_id)
4667                 goto out;
4668         /*
4669          * Find the pool from the end of the container, since mostly counter
4670          * ID is sequence increasing, and the last pool should be the needed
4671          * one.
4672          */
4673         i = cmng->n_valid;
4674         while (i--) {
4675                 struct mlx5_flow_counter_pool *pool_tmp = cmng->pools[i];
4676
4677                 if (flow_dv_is_counter_in_pool(pool_tmp, id)) {
4678                         pool = pool_tmp;
4679                         break;
4680                 }
4681         }
4682 out:
4683         rte_spinlock_unlock(&cmng->pool_update_sl);
4684         return pool;
4685 }
4686
4687 /**
4688  * Resize a counter container.
4689  *
4690  * @param[in] dev
4691  *   Pointer to the Ethernet device structure.
4692  *
4693  * @return
4694  *   0 on success, otherwise negative errno value and rte_errno is set.
4695  */
4696 static int
4697 flow_dv_container_resize(struct rte_eth_dev *dev)
4698 {
4699         struct mlx5_priv *priv = dev->data->dev_private;
4700         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4701         void *old_pools = cmng->pools;
4702         uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
4703         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
4704         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
4705
4706         if (!pools) {
4707                 rte_errno = ENOMEM;
4708                 return -ENOMEM;
4709         }
4710         if (old_pools)
4711                 memcpy(pools, old_pools, cmng->n *
4712                                        sizeof(struct mlx5_flow_counter_pool *));
4713         cmng->n = resize;
4714         cmng->pools = pools;
4715         if (old_pools)
4716                 mlx5_free(old_pools);
4717         return 0;
4718 }
4719
4720 /**
4721  * Query a devx flow counter.
4722  *
4723  * @param[in] dev
4724  *   Pointer to the Ethernet device structure.
4725  * @param[in] cnt
4726  *   Index to the flow counter.
4727  * @param[out] pkts
4728  *   The statistics value of packets.
4729  * @param[out] bytes
4730  *   The statistics value of bytes.
4731  *
4732  * @return
4733  *   0 on success, otherwise a negative errno value and rte_errno is set.
4734  */
4735 static inline int
4736 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
4737                      uint64_t *bytes)
4738 {
4739         struct mlx5_priv *priv = dev->data->dev_private;
4740         struct mlx5_flow_counter_pool *pool = NULL;
4741         struct mlx5_flow_counter *cnt;
4742         int offset;
4743
4744         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4745         MLX5_ASSERT(pool);
4746         if (priv->sh->cmng.counter_fallback)
4747                 return mlx5_devx_cmd_flow_counter_query(cnt->dcs_when_active, 0,
4748                                         0, pkts, bytes, 0, NULL, NULL, 0);
4749         rte_spinlock_lock(&pool->sl);
4750         if (!pool->raw) {
4751                 *pkts = 0;
4752                 *bytes = 0;
4753         } else {
4754                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
4755                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
4756                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
4757         }
4758         rte_spinlock_unlock(&pool->sl);
4759         return 0;
4760 }
4761
4762 /**
4763  * Create and initialize a new counter pool.
4764  *
4765  * @param[in] dev
4766  *   Pointer to the Ethernet device structure.
4767  * @param[out] dcs
4768  *   The devX counter handle.
4769  * @param[in] age
4770  *   Whether the pool is for counter that was allocated for aging.
4771  * @param[in/out] cont_cur
4772  *   Pointer to the container pointer, it will be update in pool resize.
4773  *
4774  * @return
4775  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
4776  */
4777 static struct mlx5_flow_counter_pool *
4778 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
4779                     uint32_t age)
4780 {
4781         struct mlx5_priv *priv = dev->data->dev_private;
4782         struct mlx5_flow_counter_pool *pool;
4783         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4784         bool fallback = priv->sh->cmng.counter_fallback;
4785         uint32_t size = sizeof(*pool);
4786
4787         size += MLX5_COUNTERS_PER_POOL * MLX5_CNT_SIZE;
4788         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * MLX5_AGE_SIZE);
4789         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
4790         if (!pool) {
4791                 rte_errno = ENOMEM;
4792                 return NULL;
4793         }
4794         pool->raw = NULL;
4795         pool->is_aged = !!age;
4796         pool->query_gen = 0;
4797         pool->min_dcs = dcs;
4798         rte_spinlock_init(&pool->sl);
4799         rte_spinlock_init(&pool->csl);
4800         TAILQ_INIT(&pool->counters[0]);
4801         TAILQ_INIT(&pool->counters[1]);
4802         pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
4803         rte_spinlock_lock(&cmng->pool_update_sl);
4804         pool->index = cmng->n_valid;
4805         if (pool->index == cmng->n && flow_dv_container_resize(dev)) {
4806                 mlx5_free(pool);
4807                 rte_spinlock_unlock(&cmng->pool_update_sl);
4808                 return NULL;
4809         }
4810         cmng->pools[pool->index] = pool;
4811         cmng->n_valid++;
4812         if (unlikely(fallback)) {
4813                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
4814
4815                 if (base < cmng->min_id)
4816                         cmng->min_id = base;
4817                 if (base > cmng->max_id)
4818                         cmng->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
4819                 cmng->last_pool_idx = pool->index;
4820         }
4821         rte_spinlock_unlock(&cmng->pool_update_sl);
4822         return pool;
4823 }
4824
4825 /**
4826  * Prepare a new counter and/or a new counter pool.
4827  *
4828  * @param[in] dev
4829  *   Pointer to the Ethernet device structure.
4830  * @param[out] cnt_free
4831  *   Where to put the pointer of a new counter.
4832  * @param[in] age
4833  *   Whether the pool is for counter that was allocated for aging.
4834  *
4835  * @return
4836  *   The counter pool pointer and @p cnt_free is set on success,
4837  *   NULL otherwise and rte_errno is set.
4838  */
4839 static struct mlx5_flow_counter_pool *
4840 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
4841                              struct mlx5_flow_counter **cnt_free,
4842                              uint32_t age)
4843 {
4844         struct mlx5_priv *priv = dev->data->dev_private;
4845         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4846         struct mlx5_flow_counter_pool *pool;
4847         struct mlx5_counters tmp_tq;
4848         struct mlx5_devx_obj *dcs = NULL;
4849         struct mlx5_flow_counter *cnt;
4850         enum mlx5_counter_type cnt_type =
4851                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
4852         bool fallback = priv->sh->cmng.counter_fallback;
4853         uint32_t i;
4854
4855         if (fallback) {
4856                 /* bulk_bitmap must be 0 for single counter allocation. */
4857                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
4858                 if (!dcs)
4859                         return NULL;
4860                 pool = flow_dv_find_pool_by_id(cmng, dcs->id);
4861                 if (!pool) {
4862                         pool = flow_dv_pool_create(dev, dcs, age);
4863                         if (!pool) {
4864                                 mlx5_devx_cmd_destroy(dcs);
4865                                 return NULL;
4866                         }
4867                 }
4868                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
4869                 cnt = MLX5_POOL_GET_CNT(pool, i);
4870                 cnt->pool = pool;
4871                 cnt->dcs_when_free = dcs;
4872                 *cnt_free = cnt;
4873                 return pool;
4874         }
4875         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
4876         if (!dcs) {
4877                 rte_errno = ENODATA;
4878                 return NULL;
4879         }
4880         pool = flow_dv_pool_create(dev, dcs, age);
4881         if (!pool) {
4882                 mlx5_devx_cmd_destroy(dcs);
4883                 return NULL;
4884         }
4885         TAILQ_INIT(&tmp_tq);
4886         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
4887                 cnt = MLX5_POOL_GET_CNT(pool, i);
4888                 cnt->pool = pool;
4889                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
4890         }
4891         rte_spinlock_lock(&cmng->csl[cnt_type]);
4892         TAILQ_CONCAT(&cmng->counters[cnt_type], &tmp_tq, next);
4893         rte_spinlock_unlock(&cmng->csl[cnt_type]);
4894         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
4895         (*cnt_free)->pool = pool;
4896         return pool;
4897 }
4898
4899 /**
4900  * Allocate a flow counter.
4901  *
4902  * @param[in] dev
4903  *   Pointer to the Ethernet device structure.
4904  * @param[in] age
4905  *   Whether the counter was allocated for aging.
4906  *
4907  * @return
4908  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
4909  */
4910 static uint32_t
4911 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t age)
4912 {
4913         struct mlx5_priv *priv = dev->data->dev_private;
4914         struct mlx5_flow_counter_pool *pool = NULL;
4915         struct mlx5_flow_counter *cnt_free = NULL;
4916         bool fallback = priv->sh->cmng.counter_fallback;
4917         struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
4918         enum mlx5_counter_type cnt_type =
4919                         age ? MLX5_COUNTER_TYPE_AGE : MLX5_COUNTER_TYPE_ORIGIN;
4920         uint32_t cnt_idx;
4921
4922         if (!priv->config.devx) {
4923                 rte_errno = ENOTSUP;
4924                 return 0;
4925         }
4926         /* Get free counters from container. */
4927         rte_spinlock_lock(&cmng->csl[cnt_type]);
4928         cnt_free = TAILQ_FIRST(&cmng->counters[cnt_type]);
4929         if (cnt_free)
4930                 TAILQ_REMOVE(&cmng->counters[cnt_type], cnt_free, next);
4931         rte_spinlock_unlock(&cmng->csl[cnt_type]);
4932         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free, age))
4933                 goto err;
4934         pool = cnt_free->pool;
4935         if (fallback)
4936                 cnt_free->dcs_when_active = cnt_free->dcs_when_free;
4937         /* Create a DV counter action only in the first time usage. */
4938         if (!cnt_free->action) {
4939                 uint16_t offset;
4940                 struct mlx5_devx_obj *dcs;
4941                 int ret;
4942
4943                 if (!fallback) {
4944                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
4945                         dcs = pool->min_dcs;
4946                 } else {
4947                         offset = 0;
4948                         dcs = cnt_free->dcs_when_free;
4949                 }
4950                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
4951                                                             &cnt_free->action);
4952                 if (ret) {
4953                         rte_errno = errno;
4954                         goto err;
4955                 }
4956         }
4957         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
4958                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
4959         /* Update the counter reset values. */
4960         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
4961                                  &cnt_free->bytes))
4962                 goto err;
4963         if (!fallback && !priv->sh->cmng.query_thread_on)
4964                 /* Start the asynchronous batch query by the host thread. */
4965                 mlx5_set_query_alarm(priv->sh);
4966         return cnt_idx;
4967 err:
4968         if (cnt_free) {
4969                 cnt_free->pool = pool;
4970                 if (fallback)
4971                         cnt_free->dcs_when_free = cnt_free->dcs_when_active;
4972                 rte_spinlock_lock(&cmng->csl[cnt_type]);
4973                 TAILQ_INSERT_TAIL(&cmng->counters[cnt_type], cnt_free, next);
4974                 rte_spinlock_unlock(&cmng->csl[cnt_type]);
4975         }
4976         return 0;
4977 }
4978
4979 /**
4980  * Allocate a shared flow counter.
4981  *
4982  * @param[in] ctx
4983  *   Pointer to the shared counter configuration.
4984  * @param[in] data
4985  *   Pointer to save the allocated counter index.
4986  *
4987  * @return
4988  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
4989  */
4990
4991 static int32_t
4992 flow_dv_counter_alloc_shared_cb(void *ctx, union mlx5_l3t_data *data)
4993 {
4994         struct mlx5_shared_counter_conf *conf = ctx;
4995         struct rte_eth_dev *dev = conf->dev;
4996         struct mlx5_flow_counter *cnt;
4997
4998         data->dword = flow_dv_counter_alloc(dev, 0);
4999         data->dword |= MLX5_CNT_SHARED_OFFSET;
5000         cnt = flow_dv_counter_get_by_idx(dev, data->dword, NULL);
5001         cnt->shared_info.id = conf->id;
5002         return 0;
5003 }
5004
5005 /**
5006  * Get a shared flow counter.
5007  *
5008  * @param[in] dev
5009  *   Pointer to the Ethernet device structure.
5010  * @param[in] id
5011  *   Counter identifier.
5012  *
5013  * @return
5014  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
5015  */
5016 static uint32_t
5017 flow_dv_counter_get_shared(struct rte_eth_dev *dev, uint32_t id)
5018 {
5019         struct mlx5_priv *priv = dev->data->dev_private;
5020         struct mlx5_shared_counter_conf conf = {
5021                 .dev = dev,
5022                 .id = id,
5023         };
5024         union mlx5_l3t_data data = {
5025                 .dword = 0,
5026         };
5027
5028         mlx5_l3t_prepare_entry(priv->sh->cnt_id_tbl, id, &data,
5029                                flow_dv_counter_alloc_shared_cb, &conf);
5030         return data.dword;
5031 }
5032
5033 /**
5034  * Get age param from counter index.
5035  *
5036  * @param[in] dev
5037  *   Pointer to the Ethernet device structure.
5038  * @param[in] counter
5039  *   Index to the counter handler.
5040  *
5041  * @return
5042  *   The aging parameter specified for the counter index.
5043  */
5044 static struct mlx5_age_param*
5045 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
5046                                 uint32_t counter)
5047 {
5048         struct mlx5_flow_counter *cnt;
5049         struct mlx5_flow_counter_pool *pool = NULL;
5050
5051         flow_dv_counter_get_by_idx(dev, counter, &pool);
5052         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
5053         cnt = MLX5_POOL_GET_CNT(pool, counter);
5054         return MLX5_CNT_TO_AGE(cnt);
5055 }
5056
5057 /**
5058  * Remove a flow counter from aged counter list.
5059  *
5060  * @param[in] dev
5061  *   Pointer to the Ethernet device structure.
5062  * @param[in] counter
5063  *   Index to the counter handler.
5064  * @param[in] cnt
5065  *   Pointer to the counter handler.
5066  */
5067 static void
5068 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
5069                                 uint32_t counter, struct mlx5_flow_counter *cnt)
5070 {
5071         struct mlx5_age_info *age_info;
5072         struct mlx5_age_param *age_param;
5073         struct mlx5_priv *priv = dev->data->dev_private;
5074         uint16_t expected = AGE_CANDIDATE;
5075
5076         age_info = GET_PORT_AGE_INFO(priv);
5077         age_param = flow_dv_counter_idx_get_age(dev, counter);
5078         if (!__atomic_compare_exchange_n(&age_param->state, &expected,
5079                                          AGE_FREE, false, __ATOMIC_RELAXED,
5080                                          __ATOMIC_RELAXED)) {
5081                 /**
5082                  * We need the lock even it is age timeout,
5083                  * since counter may still in process.
5084                  */
5085                 rte_spinlock_lock(&age_info->aged_sl);
5086                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
5087                 rte_spinlock_unlock(&age_info->aged_sl);
5088                 __atomic_store_n(&age_param->state, AGE_FREE, __ATOMIC_RELAXED);
5089         }
5090 }
5091
5092 /**
5093  * Release a flow counter.
5094  *
5095  * @param[in] dev
5096  *   Pointer to the Ethernet device structure.
5097  * @param[in] counter
5098  *   Index to the counter handler.
5099  */
5100 static void
5101 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
5102 {
5103         struct mlx5_priv *priv = dev->data->dev_private;
5104         struct mlx5_flow_counter_pool *pool = NULL;
5105         struct mlx5_flow_counter *cnt;
5106         enum mlx5_counter_type cnt_type;
5107
5108         if (!counter)
5109                 return;
5110         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
5111         MLX5_ASSERT(pool);
5112         if (IS_SHARED_CNT(counter) &&
5113             mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, cnt->shared_info.id))
5114                 return;
5115         if (pool->is_aged)
5116                 flow_dv_counter_remove_from_age(dev, counter, cnt);
5117         cnt->pool = pool;
5118         /*
5119          * Put the counter back to list to be updated in none fallback mode.
5120          * Currently, we are using two list alternately, while one is in query,
5121          * add the freed counter to the other list based on the pool query_gen
5122          * value. After query finishes, add counter the list to the global
5123          * container counter list. The list changes while query starts. In
5124          * this case, lock will not be needed as query callback and release
5125          * function both operate with the different list.
5126          *
5127          */
5128         if (!priv->sh->cmng.counter_fallback) {
5129                 rte_spinlock_lock(&pool->csl);
5130                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
5131                 rte_spinlock_unlock(&pool->csl);
5132         } else {
5133                 cnt->dcs_when_free = cnt->dcs_when_active;
5134                 cnt_type = pool->is_aged ? MLX5_COUNTER_TYPE_AGE :
5135                                            MLX5_COUNTER_TYPE_ORIGIN;
5136                 rte_spinlock_lock(&priv->sh->cmng.csl[cnt_type]);
5137                 TAILQ_INSERT_TAIL(&priv->sh->cmng.counters[cnt_type],
5138                                   cnt, next);
5139                 rte_spinlock_unlock(&priv->sh->cmng.csl[cnt_type]);
5140         }
5141 }
5142
5143 /**
5144  * Verify the @p attributes will be correctly understood by the NIC and store
5145  * them in the @p flow if everything is correct.
5146  *
5147  * @param[in] dev
5148  *   Pointer to dev struct.
5149  * @param[in] attributes
5150  *   Pointer to flow attributes
5151  * @param[in] external
5152  *   This flow rule is created by request external to PMD.
5153  * @param[out] error
5154  *   Pointer to error structure.
5155  *
5156  * @return
5157  *   - 0 on success and non root table.
5158  *   - 1 on success and root table.
5159  *   - a negative errno value otherwise and rte_errno is set.
5160  */
5161 static int
5162 flow_dv_validate_attributes(struct rte_eth_dev *dev,
5163                             const struct rte_flow_attr *attributes,
5164                             bool external __rte_unused,
5165                             struct rte_flow_error *error)
5166 {
5167         struct mlx5_priv *priv = dev->data->dev_private;
5168         uint32_t priority_max = priv->config.flow_prio - 1;
5169         int ret = 0;
5170
5171 #ifndef HAVE_MLX5DV_DR
5172         if (attributes->group)
5173                 return rte_flow_error_set(error, ENOTSUP,
5174                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
5175                                           NULL,
5176                                           "groups are not supported");
5177 #else
5178         uint32_t table = 0;
5179
5180         ret = mlx5_flow_group_to_table(attributes, external,
5181                                        attributes->group, !!priv->fdb_def_rule,
5182                                        &table, error);
5183         if (ret)
5184                 return ret;
5185         if (!table)
5186                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
5187 #endif
5188         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
5189             attributes->priority >= priority_max)
5190                 return rte_flow_error_set(error, ENOTSUP,
5191                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
5192                                           NULL,
5193                                           "priority out of range");
5194         if (attributes->transfer) {
5195                 if (!priv->config.dv_esw_en)
5196                         return rte_flow_error_set
5197                                 (error, ENOTSUP,
5198                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5199                                  "E-Switch dr is not supported");
5200                 if (!(priv->representor || priv->master))
5201                         return rte_flow_error_set
5202                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5203                                  NULL, "E-Switch configuration can only be"
5204                                  " done by a master or a representor device");
5205                 if (attributes->egress)
5206                         return rte_flow_error_set
5207                                 (error, ENOTSUP,
5208                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
5209                                  "egress is not supported");
5210         }
5211         if (!(attributes->egress ^ attributes->ingress))
5212                 return rte_flow_error_set(error, ENOTSUP,
5213                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
5214                                           "must specify exactly one of "
5215                                           "ingress or egress");
5216         return ret;
5217 }
5218
5219 /**
5220  * Internal validation function. For validating both actions and items.
5221  *
5222  * @param[in] dev
5223  *   Pointer to the rte_eth_dev structure.
5224  * @param[in] attr
5225  *   Pointer to the flow attributes.
5226  * @param[in] items
5227  *   Pointer to the list of items.
5228  * @param[in] actions
5229  *   Pointer to the list of actions.
5230  * @param[in] external
5231  *   This flow rule is created by request external to PMD.
5232  * @param[in] hairpin
5233  *   Number of hairpin TX actions, 0 means classic flow.
5234  * @param[out] error
5235  *   Pointer to the error structure.
5236  *
5237  * @return
5238  *   0 on success, a negative errno value otherwise and rte_errno is set.
5239  */
5240 static int
5241 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
5242                  const struct rte_flow_item items[],
5243                  const struct rte_flow_action actions[],
5244                  bool external, int hairpin, struct rte_flow_error *error)
5245 {
5246         int ret;
5247         uint64_t action_flags = 0;
5248         uint64_t item_flags = 0;
5249         uint64_t last_item = 0;
5250         uint8_t next_protocol = 0xff;
5251         uint16_t ether_type = 0;
5252         int actions_n = 0;
5253         uint8_t item_ipv6_proto = 0;
5254         const struct rte_flow_item *gre_item = NULL;
5255         const struct rte_flow_action_raw_decap *decap;
5256         const struct rte_flow_action_raw_encap *encap;
5257         const struct rte_flow_action_rss *rss;
5258         const struct rte_flow_item_tcp nic_tcp_mask = {
5259                 .hdr = {
5260                         .tcp_flags = 0xFF,
5261                         .src_port = RTE_BE16(UINT16_MAX),
5262                         .dst_port = RTE_BE16(UINT16_MAX),
5263                 }
5264         };
5265         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
5266                 .hdr = {
5267                         .src_addr =
5268                         "\xff\xff\xff\xff\xff\xff\xff\xff"
5269                         "\xff\xff\xff\xff\xff\xff\xff\xff",
5270                         .dst_addr =
5271                         "\xff\xff\xff\xff\xff\xff\xff\xff"
5272                         "\xff\xff\xff\xff\xff\xff\xff\xff",
5273                         .vtc_flow = RTE_BE32(0xffffffff),
5274                         .proto = 0xff,
5275                         .hop_limits = 0xff,
5276                 },
5277                 .has_frag_ext = 1,
5278         };
5279         const struct rte_flow_item_ecpri nic_ecpri_mask = {
5280                 .hdr = {
5281                         .common = {
5282                                 .u32 =
5283                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
5284                                         .type = 0xFF,
5285                                         }).u32),
5286                         },
5287                         .dummy[0] = 0xffffffff,
5288                 },
5289         };
5290         struct mlx5_priv *priv = dev->data->dev_private;
5291         struct mlx5_dev_config *dev_conf = &priv->config;
5292         uint16_t queue_index = 0xFFFF;
5293         const struct rte_flow_item_vlan *vlan_m = NULL;
5294         int16_t rw_act_num = 0;
5295         uint64_t is_root;
5296
5297         if (items == NULL)
5298                 return -1;
5299         ret = flow_dv_validate_attributes(dev, attr, external, error);
5300         if (ret < 0)
5301                 return ret;
5302         is_root = (uint64_t)ret;
5303         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5304                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
5305                 int type = items->type;
5306
5307                 if (!mlx5_flow_os_item_supported(type))
5308                         return rte_flow_error_set(error, ENOTSUP,
5309                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5310                                                   NULL, "item not supported");
5311                 switch (type) {
5312                 case RTE_FLOW_ITEM_TYPE_VOID:
5313                         break;
5314                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
5315                         ret = flow_dv_validate_item_port_id
5316                                         (dev, items, attr, item_flags, error);
5317                         if (ret < 0)
5318                                 return ret;
5319                         last_item = MLX5_FLOW_ITEM_PORT_ID;
5320                         break;
5321                 case RTE_FLOW_ITEM_TYPE_ETH:
5322                         ret = mlx5_flow_validate_item_eth(items, item_flags,
5323                                                           error);
5324                         if (ret < 0)
5325                                 return ret;
5326                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
5327                                              MLX5_FLOW_LAYER_OUTER_L2;
5328                         if (items->mask != NULL && items->spec != NULL) {
5329                                 ether_type =
5330                                         ((const struct rte_flow_item_eth *)
5331                                          items->spec)->type;
5332                                 ether_type &=
5333                                         ((const struct rte_flow_item_eth *)
5334                                          items->mask)->type;
5335                                 ether_type = rte_be_to_cpu_16(ether_type);
5336                         } else {
5337                                 ether_type = 0;
5338                         }
5339                         break;
5340                 case RTE_FLOW_ITEM_TYPE_VLAN:
5341                         ret = flow_dv_validate_item_vlan(items, item_flags,
5342                                                          dev, error);
5343                         if (ret < 0)
5344                                 return ret;
5345                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
5346                                              MLX5_FLOW_LAYER_OUTER_VLAN;
5347                         if (items->mask != NULL && items->spec != NULL) {
5348                                 ether_type =
5349                                         ((const struct rte_flow_item_vlan *)
5350                                          items->spec)->inner_type;
5351                                 ether_type &=
5352                                         ((const struct rte_flow_item_vlan *)
5353                                          items->mask)->inner_type;
5354                                 ether_type = rte_be_to_cpu_16(ether_type);
5355                         } else {
5356                                 ether_type = 0;
5357                         }
5358                         /* Store outer VLAN mask for of_push_vlan action. */
5359                         if (!tunnel)
5360                                 vlan_m = items->mask;
5361                         break;
5362                 case RTE_FLOW_ITEM_TYPE_IPV4:
5363                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5364                                                   &item_flags, &tunnel);
5365                         ret = flow_dv_validate_item_ipv4(items, item_flags,
5366                                                          last_item, ether_type,
5367                                                          error);
5368                         if (ret < 0)
5369                                 return ret;
5370                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5371                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5372                         if (items->mask != NULL &&
5373                             ((const struct rte_flow_item_ipv4 *)
5374                              items->mask)->hdr.next_proto_id) {
5375                                 next_protocol =
5376                                         ((const struct rte_flow_item_ipv4 *)
5377                                          (items->spec))->hdr.next_proto_id;
5378                                 next_protocol &=
5379                                         ((const struct rte_flow_item_ipv4 *)
5380                                          (items->mask))->hdr.next_proto_id;
5381                         } else {
5382                                 /* Reset for inner layer. */
5383                                 next_protocol = 0xff;
5384                         }
5385                         break;
5386                 case RTE_FLOW_ITEM_TYPE_IPV6:
5387                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5388                                                   &item_flags, &tunnel);
5389                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
5390                                                            last_item,
5391                                                            ether_type,
5392                                                            &nic_ipv6_mask,
5393                                                            error);
5394                         if (ret < 0)
5395                                 return ret;
5396                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5397                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5398                         if (items->mask != NULL &&
5399                             ((const struct rte_flow_item_ipv6 *)
5400                              items->mask)->hdr.proto) {
5401                                 item_ipv6_proto =
5402                                         ((const struct rte_flow_item_ipv6 *)
5403                                          items->spec)->hdr.proto;
5404                                 next_protocol =
5405                                         ((const struct rte_flow_item_ipv6 *)
5406                                          items->spec)->hdr.proto;
5407                                 next_protocol &=
5408                                         ((const struct rte_flow_item_ipv6 *)
5409                                          items->mask)->hdr.proto;
5410                         } else {
5411                                 /* Reset for inner layer. */
5412                                 next_protocol = 0xff;
5413                         }
5414                         break;
5415                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
5416                         ret = flow_dv_validate_item_ipv6_frag_ext(items,
5417                                                                   item_flags,
5418                                                                   error);
5419                         if (ret < 0)
5420                                 return ret;
5421                         last_item = tunnel ?
5422                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
5423                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
5424                         if (items->mask != NULL &&
5425                             ((const struct rte_flow_item_ipv6_frag_ext *)
5426                              items->mask)->hdr.next_header) {
5427                                 next_protocol =
5428                                 ((const struct rte_flow_item_ipv6_frag_ext *)
5429                                  items->spec)->hdr.next_header;
5430                                 next_protocol &=
5431                                 ((const struct rte_flow_item_ipv6_frag_ext *)
5432                                  items->mask)->hdr.next_header;
5433                         } else {
5434                                 /* Reset for inner layer. */
5435                                 next_protocol = 0xff;
5436                         }
5437                         break;
5438                 case RTE_FLOW_ITEM_TYPE_TCP:
5439                         ret = mlx5_flow_validate_item_tcp
5440                                                 (items, item_flags,
5441                                                  next_protocol,
5442                                                  &nic_tcp_mask,
5443                                                  error);
5444                         if (ret < 0)
5445                                 return ret;
5446                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5447                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5448                         break;
5449                 case RTE_FLOW_ITEM_TYPE_UDP:
5450                         ret = mlx5_flow_validate_item_udp(items, item_flags,
5451                                                           next_protocol,
5452                                                           error);
5453                         if (ret < 0)
5454                                 return ret;
5455                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5456                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5457                         break;
5458                 case RTE_FLOW_ITEM_TYPE_GRE:
5459                         ret = mlx5_flow_validate_item_gre(items, item_flags,
5460                                                           next_protocol, error);
5461                         if (ret < 0)
5462                                 return ret;
5463                         gre_item = items;
5464                         last_item = MLX5_FLOW_LAYER_GRE;
5465                         break;
5466                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5467                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
5468                                                             next_protocol,
5469                                                             error);
5470                         if (ret < 0)
5471                                 return ret;
5472                         last_item = MLX5_FLOW_LAYER_NVGRE;
5473                         break;
5474                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5475                         ret = mlx5_flow_validate_item_gre_key
5476                                 (items, item_flags, gre_item, error);
5477                         if (ret < 0)
5478                                 return ret;
5479                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5480                         break;
5481                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5482                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
5483                                                             error);
5484                         if (ret < 0)
5485                                 return ret;
5486                         last_item = MLX5_FLOW_LAYER_VXLAN;
5487                         break;
5488                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5489                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
5490                                                                 item_flags, dev,
5491                                                                 error);
5492                         if (ret < 0)
5493                                 return ret;
5494                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5495                         break;
5496                 case RTE_FLOW_ITEM_TYPE_GENEVE:
5497                         ret = mlx5_flow_validate_item_geneve(items,
5498                                                              item_flags, dev,
5499                                                              error);
5500                         if (ret < 0)
5501                                 return ret;
5502                         last_item = MLX5_FLOW_LAYER_GENEVE;
5503                         break;
5504                 case RTE_FLOW_ITEM_TYPE_MPLS:
5505                         ret = mlx5_flow_validate_item_mpls(dev, items,
5506                                                            item_flags,
5507                                                            last_item, error);
5508                         if (ret < 0)
5509                                 return ret;
5510                         last_item = MLX5_FLOW_LAYER_MPLS;
5511                         break;
5512
5513                 case RTE_FLOW_ITEM_TYPE_MARK:
5514                         ret = flow_dv_validate_item_mark(dev, items, attr,
5515                                                          error);
5516                         if (ret < 0)
5517                                 return ret;
5518                         last_item = MLX5_FLOW_ITEM_MARK;
5519                         break;
5520                 case RTE_FLOW_ITEM_TYPE_META:
5521                         ret = flow_dv_validate_item_meta(dev, items, attr,
5522                                                          error);
5523                         if (ret < 0)
5524                                 return ret;
5525                         last_item = MLX5_FLOW_ITEM_METADATA;
5526                         break;
5527                 case RTE_FLOW_ITEM_TYPE_ICMP:
5528                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
5529                                                            next_protocol,
5530                                                            error);
5531                         if (ret < 0)
5532                                 return ret;
5533                         last_item = MLX5_FLOW_LAYER_ICMP;
5534                         break;
5535                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5536                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
5537                                                             next_protocol,
5538                                                             error);
5539                         if (ret < 0)
5540                                 return ret;
5541                         item_ipv6_proto = IPPROTO_ICMPV6;
5542                         last_item = MLX5_FLOW_LAYER_ICMP6;
5543                         break;
5544                 case RTE_FLOW_ITEM_TYPE_TAG:
5545                         ret = flow_dv_validate_item_tag(dev, items,
5546                                                         attr, error);
5547                         if (ret < 0)
5548                                 return ret;
5549                         last_item = MLX5_FLOW_ITEM_TAG;
5550                         break;
5551                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
5552                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
5553                         break;
5554                 case RTE_FLOW_ITEM_TYPE_GTP:
5555                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
5556                                                         error);
5557                         if (ret < 0)
5558                                 return ret;
5559                         last_item = MLX5_FLOW_LAYER_GTP;
5560                         break;
5561                 case RTE_FLOW_ITEM_TYPE_ECPRI:
5562                         /* Capacity will be checked in the translate stage. */
5563                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
5564                                                             last_item,
5565                                                             ether_type,
5566                                                             &nic_ecpri_mask,
5567                                                             error);
5568                         if (ret < 0)
5569                                 return ret;
5570                         last_item = MLX5_FLOW_LAYER_ECPRI;
5571                         break;
5572                 default:
5573                         return rte_flow_error_set(error, ENOTSUP,
5574                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5575                                                   NULL, "item not supported");
5576                 }
5577                 item_flags |= last_item;
5578         }
5579         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5580                 int type = actions->type;
5581
5582                 if (!mlx5_flow_os_action_supported(type))
5583                         return rte_flow_error_set(error, ENOTSUP,
5584                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5585                                                   actions,
5586                                                   "action not supported");
5587                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5588                         return rte_flow_error_set(error, ENOTSUP,
5589                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5590                                                   actions, "too many actions");
5591                 switch (type) {
5592                 case RTE_FLOW_ACTION_TYPE_VOID:
5593                         break;
5594                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5595                         ret = flow_dv_validate_action_port_id(dev,
5596                                                               action_flags,
5597                                                               actions,
5598                                                               attr,
5599                                                               error);
5600                         if (ret)
5601                                 return ret;
5602                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5603                         ++actions_n;
5604                         break;
5605                 case RTE_FLOW_ACTION_TYPE_FLAG:
5606                         ret = flow_dv_validate_action_flag(dev, action_flags,
5607                                                            attr, error);
5608                         if (ret < 0)
5609                                 return ret;
5610                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5611                                 /* Count all modify-header actions as one. */
5612                                 if (!(action_flags &
5613                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5614                                         ++actions_n;
5615                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
5616                                                 MLX5_FLOW_ACTION_MARK_EXT;
5617                         } else {
5618                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
5619                                 ++actions_n;
5620                         }
5621                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5622                         break;
5623                 case RTE_FLOW_ACTION_TYPE_MARK:
5624                         ret = flow_dv_validate_action_mark(dev, actions,
5625                                                            action_flags,
5626                                                            attr, error);
5627                         if (ret < 0)
5628                                 return ret;
5629                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5630                                 /* Count all modify-header actions as one. */
5631                                 if (!(action_flags &
5632                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5633                                         ++actions_n;
5634                                 action_flags |= MLX5_FLOW_ACTION_MARK |
5635                                                 MLX5_FLOW_ACTION_MARK_EXT;
5636                         } else {
5637                                 action_flags |= MLX5_FLOW_ACTION_MARK;
5638                                 ++actions_n;
5639                         }
5640                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5641                         break;
5642                 case RTE_FLOW_ACTION_TYPE_SET_META:
5643                         ret = flow_dv_validate_action_set_meta(dev, actions,
5644                                                                action_flags,
5645                                                                attr, error);
5646                         if (ret < 0)
5647                                 return ret;
5648                         /* Count all modify-header actions as one action. */
5649                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5650                                 ++actions_n;
5651                         action_flags |= MLX5_FLOW_ACTION_SET_META;
5652                         rw_act_num += MLX5_ACT_NUM_SET_META;
5653                         break;
5654                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5655                         ret = flow_dv_validate_action_set_tag(dev, actions,
5656                                                               action_flags,
5657                                                               attr, error);
5658                         if (ret < 0)
5659                                 return ret;
5660                         /* Count all modify-header actions as one action. */
5661                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5662                                 ++actions_n;
5663                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
5664                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5665                         break;
5666                 case RTE_FLOW_ACTION_TYPE_DROP:
5667                         ret = mlx5_flow_validate_action_drop(action_flags,
5668                                                              attr, error);
5669                         if (ret < 0)
5670                                 return ret;
5671                         action_flags |= MLX5_FLOW_ACTION_DROP;
5672                         ++actions_n;
5673                         break;
5674                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5675                         ret = mlx5_flow_validate_action_queue(actions,
5676                                                               action_flags, dev,
5677                                                               attr, error);
5678                         if (ret < 0)
5679                                 return ret;
5680                         queue_index = ((const struct rte_flow_action_queue *)
5681                                                         (actions->conf))->index;
5682                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
5683                         ++actions_n;
5684                         break;
5685                 case RTE_FLOW_ACTION_TYPE_RSS:
5686                         rss = actions->conf;
5687                         ret = mlx5_flow_validate_action_rss(actions,
5688                                                             action_flags, dev,
5689                                                             attr, item_flags,
5690                                                             error);
5691                         if (ret < 0)
5692                                 return ret;
5693                         if (rss != NULL && rss->queue_num)
5694                                 queue_index = rss->queue[0];
5695                         action_flags |= MLX5_FLOW_ACTION_RSS;
5696                         ++actions_n;
5697                         break;
5698                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
5699                         ret =
5700                         mlx5_flow_validate_action_default_miss(action_flags,
5701                                         attr, error);
5702                         if (ret < 0)
5703                                 return ret;
5704                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
5705                         ++actions_n;
5706                         break;
5707                 case RTE_FLOW_ACTION_TYPE_COUNT:
5708                         ret = flow_dv_validate_action_count(dev, error);
5709                         if (ret < 0)
5710                                 return ret;
5711                         action_flags |= MLX5_FLOW_ACTION_COUNT;
5712                         ++actions_n;
5713                         break;
5714                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5715                         if (flow_dv_validate_action_pop_vlan(dev,
5716                                                              action_flags,
5717                                                              actions,
5718                                                              item_flags, attr,
5719                                                              error))
5720                                 return -rte_errno;
5721                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5722                         ++actions_n;
5723                         break;
5724                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5725                         ret = flow_dv_validate_action_push_vlan(dev,
5726                                                                 action_flags,
5727                                                                 vlan_m,
5728                                                                 actions, attr,
5729                                                                 error);
5730                         if (ret < 0)
5731                                 return ret;
5732                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5733                         ++actions_n;
5734                         break;
5735                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5736                         ret = flow_dv_validate_action_set_vlan_pcp
5737                                                 (action_flags, actions, error);
5738                         if (ret < 0)
5739                                 return ret;
5740                         /* Count PCP with push_vlan command. */
5741                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
5742                         break;
5743                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5744                         ret = flow_dv_validate_action_set_vlan_vid
5745                                                 (item_flags, action_flags,
5746                                                  actions, error);
5747                         if (ret < 0)
5748                                 return ret;
5749                         /* Count VID with push_vlan command. */
5750                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
5751                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
5752                         break;
5753                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5754                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5755                         ret = flow_dv_validate_action_l2_encap(dev,
5756                                                                action_flags,
5757                                                                actions, attr,
5758                                                                error);
5759                         if (ret < 0)
5760                                 return ret;
5761                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
5762                         ++actions_n;
5763                         break;
5764                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5765                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5766                         ret = flow_dv_validate_action_decap(dev, action_flags,
5767                                                             attr, error);
5768                         if (ret < 0)
5769                                 return ret;
5770                         action_flags |= MLX5_FLOW_ACTION_DECAP;
5771                         ++actions_n;
5772                         break;
5773                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5774                         ret = flow_dv_validate_action_raw_encap_decap
5775                                 (dev, NULL, actions->conf, attr, &action_flags,
5776                                  &actions_n, error);
5777                         if (ret < 0)
5778                                 return ret;
5779                         break;
5780                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5781                         decap = actions->conf;
5782                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5783                                 ;
5784                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5785                                 encap = NULL;
5786                                 actions--;
5787                         } else {
5788                                 encap = actions->conf;
5789                         }
5790                         ret = flow_dv_validate_action_raw_encap_decap
5791                                            (dev,
5792                                             decap ? decap : &empty_decap, encap,
5793                                             attr, &action_flags, &actions_n,
5794                                             error);
5795                         if (ret < 0)
5796                                 return ret;
5797                         break;
5798                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5799                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5800                         ret = flow_dv_validate_action_modify_mac(action_flags,
5801                                                                  actions,
5802                                                                  item_flags,
5803                                                                  error);
5804                         if (ret < 0)
5805                                 return ret;
5806                         /* Count all modify-header actions as one action. */
5807                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5808                                 ++actions_n;
5809                         action_flags |= actions->type ==
5810                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5811                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
5812                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
5813                         /*
5814                          * Even if the source and destination MAC addresses have
5815                          * overlap in the header with 4B alignment, the convert
5816                          * function will handle them separately and 4 SW actions
5817                          * will be created. And 2 actions will be added each
5818                          * time no matter how many bytes of address will be set.
5819                          */
5820                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
5821                         break;
5822                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5823                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5824                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
5825                                                                   actions,
5826                                                                   item_flags,
5827                                                                   error);
5828                         if (ret < 0)
5829                                 return ret;
5830                         /* Count all modify-header actions as one action. */
5831                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5832                                 ++actions_n;
5833                         action_flags |= actions->type ==
5834                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5835                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
5836                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
5837                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
5838                         break;
5839                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5840                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5841                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
5842                                                                   actions,
5843                                                                   item_flags,
5844                                                                   error);
5845                         if (ret < 0)
5846                                 return ret;
5847                         if (item_ipv6_proto == IPPROTO_ICMPV6)
5848                                 return rte_flow_error_set(error, ENOTSUP,
5849                                         RTE_FLOW_ERROR_TYPE_ACTION,
5850                                         actions,
5851                                         "Can't change header "
5852                                         "with ICMPv6 proto");
5853                         /* Count all modify-header actions as one action. */
5854                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5855                                 ++actions_n;
5856                         action_flags |= actions->type ==
5857                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5858                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
5859                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
5860                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
5861                         break;
5862                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5863                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5864                         ret = flow_dv_validate_action_modify_tp(action_flags,
5865                                                                 actions,
5866                                                                 item_flags,
5867                                                                 error);
5868                         if (ret < 0)
5869                                 return ret;
5870                         /* Count all modify-header actions as one action. */
5871                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5872                                 ++actions_n;
5873                         action_flags |= actions->type ==
5874                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5875                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
5876                                                 MLX5_FLOW_ACTION_SET_TP_DST;
5877                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
5878                         break;
5879                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5880                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5881                         ret = flow_dv_validate_action_modify_ttl(action_flags,
5882                                                                  actions,
5883                                                                  item_flags,
5884                                                                  error);
5885                         if (ret < 0)
5886                                 return ret;
5887                         /* Count all modify-header actions as one action. */
5888                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5889                                 ++actions_n;
5890                         action_flags |= actions->type ==
5891                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
5892                                                 MLX5_FLOW_ACTION_SET_TTL :
5893                                                 MLX5_FLOW_ACTION_DEC_TTL;
5894                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
5895                         break;
5896                 case RTE_FLOW_ACTION_TYPE_JUMP:
5897                         ret = flow_dv_validate_action_jump(actions,
5898                                                            action_flags,
5899                                                            attr, external,
5900                                                            error);
5901                         if (ret)
5902                                 return ret;
5903                         ++actions_n;
5904                         action_flags |= MLX5_FLOW_ACTION_JUMP;
5905                         break;
5906                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5907                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5908                         ret = flow_dv_validate_action_modify_tcp_seq
5909                                                                 (action_flags,
5910                                                                  actions,
5911                                                                  item_flags,
5912                                                                  error);
5913                         if (ret < 0)
5914                                 return ret;
5915                         /* Count all modify-header actions as one action. */
5916                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5917                                 ++actions_n;
5918                         action_flags |= actions->type ==
5919                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5920                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
5921                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5922                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
5923                         break;
5924                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5925                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5926                         ret = flow_dv_validate_action_modify_tcp_ack
5927                                                                 (action_flags,
5928                                                                  actions,
5929                                                                  item_flags,
5930                                                                  error);
5931                         if (ret < 0)
5932                                 return ret;
5933                         /* Count all modify-header actions as one action. */
5934                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5935                                 ++actions_n;
5936                         action_flags |= actions->type ==
5937                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
5938                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
5939                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
5940                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
5941                         break;
5942                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
5943                         break;
5944                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
5945                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
5946                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5947                         break;
5948                 case RTE_FLOW_ACTION_TYPE_METER:
5949                         ret = mlx5_flow_validate_action_meter(dev,
5950                                                               action_flags,
5951                                                               actions, attr,
5952                                                               error);
5953                         if (ret < 0)
5954                                 return ret;
5955                         action_flags |= MLX5_FLOW_ACTION_METER;
5956                         ++actions_n;
5957                         /* Meter action will add one more TAG action. */
5958                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5959                         break;
5960                 case RTE_FLOW_ACTION_TYPE_AGE:
5961                         ret = flow_dv_validate_action_age(action_flags,
5962                                                           actions, dev,
5963                                                           error);
5964                         if (ret < 0)
5965                                 return ret;
5966                         action_flags |= MLX5_FLOW_ACTION_AGE;
5967                         ++actions_n;
5968                         break;
5969                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5970                         ret = flow_dv_validate_action_modify_ipv4_dscp
5971                                                          (action_flags,
5972                                                           actions,
5973                                                           item_flags,
5974                                                           error);
5975                         if (ret < 0)
5976                                 return ret;
5977                         /* Count all modify-header actions as one action. */
5978                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5979                                 ++actions_n;
5980                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
5981                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5982                         break;
5983                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5984                         ret = flow_dv_validate_action_modify_ipv6_dscp
5985                                                                 (action_flags,
5986                                                                  actions,
5987                                                                  item_flags,
5988                                                                  error);
5989                         if (ret < 0)
5990                                 return ret;
5991                         /* Count all modify-header actions as one action. */
5992                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5993                                 ++actions_n;
5994                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
5995                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5996                         break;
5997                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
5998                         ret = flow_dv_validate_action_sample(action_flags,
5999                                                              actions, dev,
6000                                                              attr, error);
6001                         if (ret < 0)
6002                                 return ret;
6003                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
6004                         ++actions_n;
6005                         break;
6006                 default:
6007                         return rte_flow_error_set(error, ENOTSUP,
6008                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6009                                                   actions,
6010                                                   "action not supported");
6011                 }
6012         }
6013         /*
6014          * Validate the drop action mutual exclusion with other actions.
6015          * Drop action is mutually-exclusive with any other action, except for
6016          * Count action.
6017          */
6018         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
6019             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
6020                 return rte_flow_error_set(error, EINVAL,
6021                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
6022                                           "Drop action is mutually-exclusive "
6023                                           "with any other action, except for "
6024                                           "Count action");
6025         /* Eswitch has few restrictions on using items and actions */
6026         if (attr->transfer) {
6027                 if (!mlx5_flow_ext_mreg_supported(dev) &&
6028                     action_flags & MLX5_FLOW_ACTION_FLAG)
6029                         return rte_flow_error_set(error, ENOTSUP,
6030                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6031                                                   NULL,
6032                                                   "unsupported action FLAG");
6033                 if (!mlx5_flow_ext_mreg_supported(dev) &&
6034                     action_flags & MLX5_FLOW_ACTION_MARK)
6035                         return rte_flow_error_set(error, ENOTSUP,
6036                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6037                                                   NULL,
6038                                                   "unsupported action MARK");
6039                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
6040                         return rte_flow_error_set(error, ENOTSUP,
6041                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6042                                                   NULL,
6043                                                   "unsupported action QUEUE");
6044                 if (action_flags & MLX5_FLOW_ACTION_RSS)
6045                         return rte_flow_error_set(error, ENOTSUP,
6046                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6047                                                   NULL,
6048                                                   "unsupported action RSS");
6049                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
6050                         return rte_flow_error_set(error, EINVAL,
6051                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6052                                                   actions,
6053                                                   "no fate action is found");
6054         } else {
6055                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
6056                         return rte_flow_error_set(error, EINVAL,
6057                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6058                                                   actions,
6059                                                   "no fate action is found");
6060         }
6061         /* Continue validation for Xcap and VLAN actions.*/
6062         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
6063                              MLX5_FLOW_VLAN_ACTIONS)) &&
6064             (queue_index == 0xFFFF ||
6065              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
6066                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
6067                     MLX5_FLOW_XCAP_ACTIONS)
6068                         return rte_flow_error_set(error, ENOTSUP,
6069                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6070                                                   NULL, "encap and decap "
6071                                                   "combination aren't supported");
6072                 if (!attr->transfer && attr->ingress) {
6073                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
6074                                 return rte_flow_error_set
6075                                                 (error, ENOTSUP,
6076                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6077                                                  NULL, "encap is not supported"
6078                                                  " for ingress traffic");
6079                         else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
6080                                 return rte_flow_error_set
6081                                                 (error, ENOTSUP,
6082                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6083                                                  NULL, "push VLAN action not "
6084                                                  "supported for ingress");
6085                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
6086                                         MLX5_FLOW_VLAN_ACTIONS)
6087                                 return rte_flow_error_set
6088                                                 (error, ENOTSUP,
6089                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6090                                                  NULL, "no support for "
6091                                                  "multiple VLAN actions");
6092                 }
6093         }
6094         /* Hairpin flow will add one more TAG action. */
6095         if (hairpin > 0)
6096                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6097         /* extra metadata enabled: one more TAG action will be add. */
6098         if (dev_conf->dv_flow_en &&
6099             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
6100             mlx5_flow_ext_mreg_supported(dev))
6101                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6102         if ((uint32_t)rw_act_num >
6103                         flow_dv_modify_hdr_action_max(dev, is_root)) {
6104                 return rte_flow_error_set(error, ENOTSUP,
6105                                           RTE_FLOW_ERROR_TYPE_ACTION,
6106                                           NULL, "too many header modify"
6107                                           " actions to support");
6108         }
6109         return 0;
6110 }
6111
6112 /**
6113  * Internal preparation function. Allocates the DV flow size,
6114  * this size is constant.
6115  *
6116  * @param[in] dev
6117  *   Pointer to the rte_eth_dev structure.
6118  * @param[in] attr
6119  *   Pointer to the flow attributes.
6120  * @param[in] items
6121  *   Pointer to the list of items.
6122  * @param[in] actions
6123  *   Pointer to the list of actions.
6124  * @param[out] error
6125  *   Pointer to the error structure.
6126  *
6127  * @return
6128  *   Pointer to mlx5_flow object on success,
6129  *   otherwise NULL and rte_errno is set.
6130  */
6131 static struct mlx5_flow *
6132 flow_dv_prepare(struct rte_eth_dev *dev,
6133                 const struct rte_flow_attr *attr __rte_unused,
6134                 const struct rte_flow_item items[] __rte_unused,
6135                 const struct rte_flow_action actions[] __rte_unused,
6136                 struct rte_flow_error *error)
6137 {
6138         uint32_t handle_idx = 0;
6139         struct mlx5_flow *dev_flow;
6140         struct mlx5_flow_handle *dev_handle;
6141         struct mlx5_priv *priv = dev->data->dev_private;
6142
6143         /* In case of corrupting the memory. */
6144         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
6145                 rte_flow_error_set(error, ENOSPC,
6146                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6147                                    "not free temporary device flow");
6148                 return NULL;
6149         }
6150         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
6151                                    &handle_idx);
6152         if (!dev_handle) {
6153                 rte_flow_error_set(error, ENOMEM,
6154                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6155                                    "not enough memory to create flow handle");
6156                 return NULL;
6157         }
6158         /* No multi-thread supporting. */
6159         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
6160         dev_flow->handle = dev_handle;
6161         dev_flow->handle_idx = handle_idx;
6162         /*
6163          * In some old rdma-core releases, before continuing, a check of the
6164          * length of matching parameter will be done at first. It needs to use
6165          * the length without misc4 param. If the flow has misc4 support, then
6166          * the length needs to be adjusted accordingly. Each param member is
6167          * aligned with a 64B boundary naturally.
6168          */
6169         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) -
6170                                   MLX5_ST_SZ_BYTES(fte_match_set_misc4);
6171         /*
6172          * The matching value needs to be cleared to 0 before using. In the
6173          * past, it will be automatically cleared when using rte_*alloc
6174          * API. The time consumption will be almost the same as before.
6175          */
6176         memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
6177         dev_flow->ingress = attr->ingress;
6178         dev_flow->dv.transfer = attr->transfer;
6179         return dev_flow;
6180 }
6181
6182 #ifdef RTE_LIBRTE_MLX5_DEBUG
6183 /**
6184  * Sanity check for match mask and value. Similar to check_valid_spec() in
6185  * kernel driver. If unmasked bit is present in value, it returns failure.
6186  *
6187  * @param match_mask
6188  *   pointer to match mask buffer.
6189  * @param match_value
6190  *   pointer to match value buffer.
6191  *
6192  * @return
6193  *   0 if valid, -EINVAL otherwise.
6194  */
6195 static int
6196 flow_dv_check_valid_spec(void *match_mask, void *match_value)
6197 {
6198         uint8_t *m = match_mask;
6199         uint8_t *v = match_value;
6200         unsigned int i;
6201
6202         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
6203                 if (v[i] & ~m[i]) {
6204                         DRV_LOG(ERR,
6205                                 "match_value differs from match_criteria"
6206                                 " %p[%u] != %p[%u]",
6207                                 match_value, i, match_mask, i);
6208                         return -EINVAL;
6209                 }
6210         }
6211         return 0;
6212 }
6213 #endif
6214
6215 /**
6216  * Add match of ip_version.
6217  *
6218  * @param[in] group
6219  *   Flow group.
6220  * @param[in] headers_v
6221  *   Values header pointer.
6222  * @param[in] headers_m
6223  *   Masks header pointer.
6224  * @param[in] ip_version
6225  *   The IP version to set.
6226  */
6227 static inline void
6228 flow_dv_set_match_ip_version(uint32_t group,
6229                              void *headers_v,
6230                              void *headers_m,
6231                              uint8_t ip_version)
6232 {
6233         if (group == 0)
6234                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
6235         else
6236                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
6237                          ip_version);
6238         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
6239         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
6240         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
6241 }
6242
6243 /**
6244  * Add Ethernet item to matcher and to the value.
6245  *
6246  * @param[in, out] matcher
6247  *   Flow matcher.
6248  * @param[in, out] key
6249  *   Flow matcher value.
6250  * @param[in] item
6251  *   Flow pattern to translate.
6252  * @param[in] inner
6253  *   Item is inner pattern.
6254  */
6255 static void
6256 flow_dv_translate_item_eth(void *matcher, void *key,
6257                            const struct rte_flow_item *item, int inner,
6258                            uint32_t group)
6259 {
6260         const struct rte_flow_item_eth *eth_m = item->mask;
6261         const struct rte_flow_item_eth *eth_v = item->spec;
6262         const struct rte_flow_item_eth nic_mask = {
6263                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6264                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6265                 .type = RTE_BE16(0xffff),
6266         };
6267         void *headers_m;
6268         void *headers_v;
6269         char *l24_v;
6270         unsigned int i;
6271
6272         if (!eth_v)
6273                 return;
6274         if (!eth_m)
6275                 eth_m = &nic_mask;
6276         if (inner) {
6277                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6278                                          inner_headers);
6279                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6280         } else {
6281                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6282                                          outer_headers);
6283                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6284         }
6285         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
6286                &eth_m->dst, sizeof(eth_m->dst));
6287         /* The value must be in the range of the mask. */
6288         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
6289         for (i = 0; i < sizeof(eth_m->dst); ++i)
6290                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
6291         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
6292                &eth_m->src, sizeof(eth_m->src));
6293         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
6294         /* The value must be in the range of the mask. */
6295         for (i = 0; i < sizeof(eth_m->dst); ++i)
6296                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
6297         if (eth_v->type) {
6298                 /* When ethertype is present set mask for tagged VLAN. */
6299                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6300                 /* Set value for tagged VLAN if ethertype is 802.1Q. */
6301                 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
6302                     eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
6303                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag,
6304                                  1);
6305                         /* Return here to avoid setting match on ethertype. */
6306                         return;
6307                 }
6308         }
6309         /*
6310          * HW supports match on one Ethertype, the Ethertype following the last
6311          * VLAN tag of the packet (see PRM).
6312          * Set match on ethertype only if ETH header is not followed by VLAN.
6313          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6314          * ethertype, and use ip_version field instead.
6315          * eCPRI over Ether layer will use type value 0xAEFE.
6316          */
6317         if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6318             eth_m->type == 0xFFFF) {
6319                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6320         } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6321                    eth_m->type == 0xFFFF) {
6322                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6323         } else {
6324                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6325                          rte_be_to_cpu_16(eth_m->type));
6326                 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6327                                      ethertype);
6328                 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
6329         }
6330 }
6331
6332 /**
6333  * Add VLAN item to matcher and to the value.
6334  *
6335  * @param[in, out] dev_flow
6336  *   Flow descriptor.
6337  * @param[in, out] matcher
6338  *   Flow matcher.
6339  * @param[in, out] key
6340  *   Flow matcher value.
6341  * @param[in] item
6342  *   Flow pattern to translate.
6343  * @param[in] inner
6344  *   Item is inner pattern.
6345  */
6346 static void
6347 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
6348                             void *matcher, void *key,
6349                             const struct rte_flow_item *item,
6350                             int inner, uint32_t group)
6351 {
6352         const struct rte_flow_item_vlan *vlan_m = item->mask;
6353         const struct rte_flow_item_vlan *vlan_v = item->spec;
6354         void *headers_m;
6355         void *headers_v;
6356         uint16_t tci_m;
6357         uint16_t tci_v;
6358
6359         if (inner) {
6360                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6361                                          inner_headers);
6362                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6363         } else {
6364                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6365                                          outer_headers);
6366                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6367                 /*
6368                  * This is workaround, masks are not supported,
6369                  * and pre-validated.
6370                  */
6371                 if (vlan_v)
6372                         dev_flow->handle->vf_vlan.tag =
6373                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
6374         }
6375         /*
6376          * When VLAN item exists in flow, mark packet as tagged,
6377          * even if TCI is not specified.
6378          */
6379         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6380         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
6381         if (!vlan_v)
6382                 return;
6383         if (!vlan_m)
6384                 vlan_m = &rte_flow_item_vlan_mask;
6385         tci_m = rte_be_to_cpu_16(vlan_m->tci);
6386         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
6387         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
6388         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
6389         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
6390         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
6391         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
6392         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
6393         /*
6394          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6395          * ethertype, and use ip_version field instead.
6396          */
6397         if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6398             vlan_m->inner_type == 0xFFFF) {
6399                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6400         } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6401                    vlan_m->inner_type == 0xFFFF) {
6402                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6403         } else {
6404                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6405                          rte_be_to_cpu_16(vlan_m->inner_type));
6406                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
6407                          rte_be_to_cpu_16(vlan_m->inner_type &
6408                                           vlan_v->inner_type));
6409         }
6410 }
6411
6412 /**
6413  * Add IPV4 item to matcher and to the value.
6414  *
6415  * @param[in, out] matcher
6416  *   Flow matcher.
6417  * @param[in, out] key
6418  *   Flow matcher value.
6419  * @param[in] item
6420  *   Flow pattern to translate.
6421  * @param[in] item_flags
6422  *   Bit-fields that holds the items detected until now.
6423  * @param[in] inner
6424  *   Item is inner pattern.
6425  * @param[in] group
6426  *   The group to insert the rule.
6427  */
6428 static void
6429 flow_dv_translate_item_ipv4(void *matcher, void *key,
6430                             const struct rte_flow_item *item,
6431                             const uint64_t item_flags,
6432                             int inner, uint32_t group)
6433 {
6434         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
6435         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
6436         const struct rte_flow_item_ipv4 nic_mask = {
6437                 .hdr = {
6438                         .src_addr = RTE_BE32(0xffffffff),
6439                         .dst_addr = RTE_BE32(0xffffffff),
6440                         .type_of_service = 0xff,
6441                         .next_proto_id = 0xff,
6442                         .time_to_live = 0xff,
6443                 },
6444         };
6445         void *headers_m;
6446         void *headers_v;
6447         char *l24_m;
6448         char *l24_v;
6449         uint8_t tos;
6450
6451         if (inner) {
6452                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6453                                          inner_headers);
6454                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6455         } else {
6456                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6457                                          outer_headers);
6458                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6459         }
6460         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6461         /*
6462          * On outer header (which must contains L2), or inner header with L2,
6463          * set cvlan_tag mask bit to mark this packet as untagged.
6464          * This should be done even if item->spec is empty.
6465          */
6466         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6467                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6468         if (!ipv4_v)
6469                 return;
6470         if (!ipv4_m)
6471                 ipv4_m = &nic_mask;
6472         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6473                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6474         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6475                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6476         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
6477         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
6478         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6479                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6480         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6481                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6482         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
6483         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
6484         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
6485         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
6486                  ipv4_m->hdr.type_of_service);
6487         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
6488         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
6489                  ipv4_m->hdr.type_of_service >> 2);
6490         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
6491         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6492                  ipv4_m->hdr.next_proto_id);
6493         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6494                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
6495         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6496                  ipv4_m->hdr.time_to_live);
6497         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6498                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
6499         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
6500                  !!(ipv4_m->hdr.fragment_offset));
6501         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
6502                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
6503 }
6504
6505 /**
6506  * Add IPV6 item to matcher and to the value.
6507  *
6508  * @param[in, out] matcher
6509  *   Flow matcher.
6510  * @param[in, out] key
6511  *   Flow matcher value.
6512  * @param[in] item
6513  *   Flow pattern to translate.
6514  * @param[in] item_flags
6515  *   Bit-fields that holds the items detected until now.
6516  * @param[in] inner
6517  *   Item is inner pattern.
6518  * @param[in] group
6519  *   The group to insert the rule.
6520  */
6521 static void
6522 flow_dv_translate_item_ipv6(void *matcher, void *key,
6523                             const struct rte_flow_item *item,
6524                             const uint64_t item_flags,
6525                             int inner, uint32_t group)
6526 {
6527         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
6528         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
6529         const struct rte_flow_item_ipv6 nic_mask = {
6530                 .hdr = {
6531                         .src_addr =
6532                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6533                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6534                         .dst_addr =
6535                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6536                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6537                         .vtc_flow = RTE_BE32(0xffffffff),
6538                         .proto = 0xff,
6539                         .hop_limits = 0xff,
6540                 },
6541         };
6542         void *headers_m;
6543         void *headers_v;
6544         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6545         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6546         char *l24_m;
6547         char *l24_v;
6548         uint32_t vtc_m;
6549         uint32_t vtc_v;
6550         int i;
6551         int size;
6552
6553         if (inner) {
6554                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6555                                          inner_headers);
6556                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6557         } else {
6558                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6559                                          outer_headers);
6560                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6561         }
6562         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6563         /*
6564          * On outer header (which must contains L2), or inner header with L2,
6565          * set cvlan_tag mask bit to mark this packet as untagged.
6566          * This should be done even if item->spec is empty.
6567          */
6568         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6569                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6570         if (!ipv6_v)
6571                 return;
6572         if (!ipv6_m)
6573                 ipv6_m = &nic_mask;
6574         size = sizeof(ipv6_m->hdr.dst_addr);
6575         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6576                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6577         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6578                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6579         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
6580         for (i = 0; i < size; ++i)
6581                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
6582         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6583                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6584         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6585                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6586         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
6587         for (i = 0; i < size; ++i)
6588                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
6589         /* TOS. */
6590         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
6591         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
6592         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
6593         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
6594         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
6595         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
6596         /* Label. */
6597         if (inner) {
6598                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
6599                          vtc_m);
6600                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
6601                          vtc_v);
6602         } else {
6603                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
6604                          vtc_m);
6605                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
6606                          vtc_v);
6607         }
6608         /* Protocol. */
6609         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6610                  ipv6_m->hdr.proto);
6611         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6612                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
6613         /* Hop limit. */
6614         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6615                  ipv6_m->hdr.hop_limits);
6616         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6617                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
6618         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
6619                  !!(ipv6_m->has_frag_ext));
6620         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
6621                  !!(ipv6_v->has_frag_ext & ipv6_m->has_frag_ext));
6622 }
6623
6624 /**
6625  * Add IPV6 fragment extension item to matcher and to the value.
6626  *
6627  * @param[in, out] matcher
6628  *   Flow matcher.
6629  * @param[in, out] key
6630  *   Flow matcher value.
6631  * @param[in] item
6632  *   Flow pattern to translate.
6633  * @param[in] inner
6634  *   Item is inner pattern.
6635  */
6636 static void
6637 flow_dv_translate_item_ipv6_frag_ext(void *matcher, void *key,
6638                                      const struct rte_flow_item *item,
6639                                      int inner)
6640 {
6641         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_m = item->mask;
6642         const struct rte_flow_item_ipv6_frag_ext *ipv6_frag_ext_v = item->spec;
6643         const struct rte_flow_item_ipv6_frag_ext nic_mask = {
6644                 .hdr = {
6645                         .next_header = 0xff,
6646                         .frag_data = RTE_BE16(0xffff),
6647                 },
6648         };
6649         void *headers_m;
6650         void *headers_v;
6651
6652         if (inner) {
6653                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6654                                          inner_headers);
6655                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6656         } else {
6657                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6658                                          outer_headers);
6659                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6660         }
6661         /* IPv6 fragment extension item exists, so packet is IP fragment. */
6662         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
6663         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 1);
6664         if (!ipv6_frag_ext_v)
6665                 return;
6666         if (!ipv6_frag_ext_m)
6667                 ipv6_frag_ext_m = &nic_mask;
6668         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6669                  ipv6_frag_ext_m->hdr.next_header);
6670         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6671                  ipv6_frag_ext_v->hdr.next_header &
6672                  ipv6_frag_ext_m->hdr.next_header);
6673 }
6674
6675 /**
6676  * Add TCP item to matcher and to the value.
6677  *
6678  * @param[in, out] matcher
6679  *   Flow matcher.
6680  * @param[in, out] key
6681  *   Flow matcher value.
6682  * @param[in] item
6683  *   Flow pattern to translate.
6684  * @param[in] inner
6685  *   Item is inner pattern.
6686  */
6687 static void
6688 flow_dv_translate_item_tcp(void *matcher, void *key,
6689                            const struct rte_flow_item *item,
6690                            int inner)
6691 {
6692         const struct rte_flow_item_tcp *tcp_m = item->mask;
6693         const struct rte_flow_item_tcp *tcp_v = item->spec;
6694         void *headers_m;
6695         void *headers_v;
6696
6697         if (inner) {
6698                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6699                                          inner_headers);
6700                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6701         } else {
6702                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6703                                          outer_headers);
6704                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6705         }
6706         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6707         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
6708         if (!tcp_v)
6709                 return;
6710         if (!tcp_m)
6711                 tcp_m = &rte_flow_item_tcp_mask;
6712         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
6713                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
6714         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
6715                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
6716         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
6717                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
6718         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
6719                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
6720         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
6721                  tcp_m->hdr.tcp_flags);
6722         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
6723                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
6724 }
6725
6726 /**
6727  * Add UDP item to matcher and to the value.
6728  *
6729  * @param[in, out] matcher
6730  *   Flow matcher.
6731  * @param[in, out] key
6732  *   Flow matcher value.
6733  * @param[in] item
6734  *   Flow pattern to translate.
6735  * @param[in] inner
6736  *   Item is inner pattern.
6737  */
6738 static void
6739 flow_dv_translate_item_udp(void *matcher, void *key,
6740                            const struct rte_flow_item *item,
6741                            int inner)
6742 {
6743         const struct rte_flow_item_udp *udp_m = item->mask;
6744         const struct rte_flow_item_udp *udp_v = item->spec;
6745         void *headers_m;
6746         void *headers_v;
6747
6748         if (inner) {
6749                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6750                                          inner_headers);
6751                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6752         } else {
6753                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6754                                          outer_headers);
6755                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6756         }
6757         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6758         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
6759         if (!udp_v)
6760                 return;
6761         if (!udp_m)
6762                 udp_m = &rte_flow_item_udp_mask;
6763         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
6764                  rte_be_to_cpu_16(udp_m->hdr.src_port));
6765         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
6766                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
6767         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
6768                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
6769         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6770                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
6771 }
6772
6773 /**
6774  * Add GRE optional Key item to matcher and to the value.
6775  *
6776  * @param[in, out] matcher
6777  *   Flow matcher.
6778  * @param[in, out] key
6779  *   Flow matcher value.
6780  * @param[in] item
6781  *   Flow pattern to translate.
6782  * @param[in] inner
6783  *   Item is inner pattern.
6784  */
6785 static void
6786 flow_dv_translate_item_gre_key(void *matcher, void *key,
6787                                    const struct rte_flow_item *item)
6788 {
6789         const rte_be32_t *key_m = item->mask;
6790         const rte_be32_t *key_v = item->spec;
6791         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6792         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6793         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
6794
6795         /* GRE K bit must be on and should already be validated */
6796         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
6797         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
6798         if (!key_v)
6799                 return;
6800         if (!key_m)
6801                 key_m = &gre_key_default_mask;
6802         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
6803                  rte_be_to_cpu_32(*key_m) >> 8);
6804         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
6805                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
6806         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
6807                  rte_be_to_cpu_32(*key_m) & 0xFF);
6808         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
6809                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
6810 }
6811
6812 /**
6813  * Add GRE item to matcher and to the value.
6814  *
6815  * @param[in, out] matcher
6816  *   Flow matcher.
6817  * @param[in, out] key
6818  *   Flow matcher value.
6819  * @param[in] item
6820  *   Flow pattern to translate.
6821  * @param[in] inner
6822  *   Item is inner pattern.
6823  */
6824 static void
6825 flow_dv_translate_item_gre(void *matcher, void *key,
6826                            const struct rte_flow_item *item,
6827                            int inner)
6828 {
6829         const struct rte_flow_item_gre *gre_m = item->mask;
6830         const struct rte_flow_item_gre *gre_v = item->spec;
6831         void *headers_m;
6832         void *headers_v;
6833         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6834         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6835         struct {
6836                 union {
6837                         __extension__
6838                         struct {
6839                                 uint16_t version:3;
6840                                 uint16_t rsvd0:9;
6841                                 uint16_t s_present:1;
6842                                 uint16_t k_present:1;
6843                                 uint16_t rsvd_bit1:1;
6844                                 uint16_t c_present:1;
6845                         };
6846                         uint16_t value;
6847                 };
6848         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
6849
6850         if (inner) {
6851                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6852                                          inner_headers);
6853                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6854         } else {
6855                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6856                                          outer_headers);
6857                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6858         }
6859         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6860         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
6861         if (!gre_v)
6862                 return;
6863         if (!gre_m)
6864                 gre_m = &rte_flow_item_gre_mask;
6865         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
6866                  rte_be_to_cpu_16(gre_m->protocol));
6867         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6868                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
6869         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
6870         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
6871         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
6872                  gre_crks_rsvd0_ver_m.c_present);
6873         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
6874                  gre_crks_rsvd0_ver_v.c_present &
6875                  gre_crks_rsvd0_ver_m.c_present);
6876         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
6877                  gre_crks_rsvd0_ver_m.k_present);
6878         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
6879                  gre_crks_rsvd0_ver_v.k_present &
6880                  gre_crks_rsvd0_ver_m.k_present);
6881         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
6882                  gre_crks_rsvd0_ver_m.s_present);
6883         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
6884                  gre_crks_rsvd0_ver_v.s_present &
6885                  gre_crks_rsvd0_ver_m.s_present);
6886 }
6887
6888 /**
6889  * Add NVGRE item to matcher and to the value.
6890  *
6891  * @param[in, out] matcher
6892  *   Flow matcher.
6893  * @param[in, out] key
6894  *   Flow matcher value.
6895  * @param[in] item
6896  *   Flow pattern to translate.
6897  * @param[in] inner
6898  *   Item is inner pattern.
6899  */
6900 static void
6901 flow_dv_translate_item_nvgre(void *matcher, void *key,
6902                              const struct rte_flow_item *item,
6903                              int inner)
6904 {
6905         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
6906         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
6907         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6908         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6909         const char *tni_flow_id_m;
6910         const char *tni_flow_id_v;
6911         char *gre_key_m;
6912         char *gre_key_v;
6913         int size;
6914         int i;
6915
6916         /* For NVGRE, GRE header fields must be set with defined values. */
6917         const struct rte_flow_item_gre gre_spec = {
6918                 .c_rsvd0_ver = RTE_BE16(0x2000),
6919                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
6920         };
6921         const struct rte_flow_item_gre gre_mask = {
6922                 .c_rsvd0_ver = RTE_BE16(0xB000),
6923                 .protocol = RTE_BE16(UINT16_MAX),
6924         };
6925         const struct rte_flow_item gre_item = {
6926                 .spec = &gre_spec,
6927                 .mask = &gre_mask,
6928                 .last = NULL,
6929         };
6930         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
6931         if (!nvgre_v)
6932                 return;
6933         if (!nvgre_m)
6934                 nvgre_m = &rte_flow_item_nvgre_mask;
6935         tni_flow_id_m = (const char *)nvgre_m->tni;
6936         tni_flow_id_v = (const char *)nvgre_v->tni;
6937         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
6938         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
6939         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
6940         memcpy(gre_key_m, tni_flow_id_m, size);
6941         for (i = 0; i < size; ++i)
6942                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
6943 }
6944
6945 /**
6946  * Add VXLAN item to matcher and to the value.
6947  *
6948  * @param[in, out] matcher
6949  *   Flow matcher.
6950  * @param[in, out] key
6951  *   Flow matcher value.
6952  * @param[in] item
6953  *   Flow pattern to translate.
6954  * @param[in] inner
6955  *   Item is inner pattern.
6956  */
6957 static void
6958 flow_dv_translate_item_vxlan(void *matcher, void *key,
6959                              const struct rte_flow_item *item,
6960                              int inner)
6961 {
6962         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
6963         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
6964         void *headers_m;
6965         void *headers_v;
6966         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6967         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6968         char *vni_m;
6969         char *vni_v;
6970         uint16_t dport;
6971         int size;
6972         int i;
6973
6974         if (inner) {
6975                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6976                                          inner_headers);
6977                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6978         } else {
6979                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6980                                          outer_headers);
6981                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6982         }
6983         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6984                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6985         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6986                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6987                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6988         }
6989         if (!vxlan_v)
6990                 return;
6991         if (!vxlan_m)
6992                 vxlan_m = &rte_flow_item_vxlan_mask;
6993         size = sizeof(vxlan_m->vni);
6994         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
6995         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
6996         memcpy(vni_m, vxlan_m->vni, size);
6997         for (i = 0; i < size; ++i)
6998                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6999 }
7000
7001 /**
7002  * Add VXLAN-GPE item to matcher and to the value.
7003  *
7004  * @param[in, out] matcher
7005  *   Flow matcher.
7006  * @param[in, out] key
7007  *   Flow matcher value.
7008  * @param[in] item
7009  *   Flow pattern to translate.
7010  * @param[in] inner
7011  *   Item is inner pattern.
7012  */
7013
7014 static void
7015 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
7016                                  const struct rte_flow_item *item, int inner)
7017 {
7018         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
7019         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
7020         void *headers_m;
7021         void *headers_v;
7022         void *misc_m =
7023                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
7024         void *misc_v =
7025                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7026         char *vni_m;
7027         char *vni_v;
7028         uint16_t dport;
7029         int size;
7030         int i;
7031         uint8_t flags_m = 0xff;
7032         uint8_t flags_v = 0xc;
7033
7034         if (inner) {
7035                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7036                                          inner_headers);
7037                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7038         } else {
7039                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7040                                          outer_headers);
7041                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7042         }
7043         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
7044                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
7045         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7046                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7047                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7048         }
7049         if (!vxlan_v)
7050                 return;
7051         if (!vxlan_m)
7052                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
7053         size = sizeof(vxlan_m->vni);
7054         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
7055         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
7056         memcpy(vni_m, vxlan_m->vni, size);
7057         for (i = 0; i < size; ++i)
7058                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
7059         if (vxlan_m->flags) {
7060                 flags_m = vxlan_m->flags;
7061                 flags_v = vxlan_v->flags;
7062         }
7063         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
7064         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
7065         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
7066                  vxlan_m->protocol);
7067         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
7068                  vxlan_v->protocol);
7069 }
7070
7071 /**
7072  * Add Geneve item to matcher and to the value.
7073  *
7074  * @param[in, out] matcher
7075  *   Flow matcher.
7076  * @param[in, out] key
7077  *   Flow matcher value.
7078  * @param[in] item
7079  *   Flow pattern to translate.
7080  * @param[in] inner
7081  *   Item is inner pattern.
7082  */
7083
7084 static void
7085 flow_dv_translate_item_geneve(void *matcher, void *key,
7086                               const struct rte_flow_item *item, int inner)
7087 {
7088         const struct rte_flow_item_geneve *geneve_m = item->mask;
7089         const struct rte_flow_item_geneve *geneve_v = item->spec;
7090         void *headers_m;
7091         void *headers_v;
7092         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7093         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7094         uint16_t dport;
7095         uint16_t gbhdr_m;
7096         uint16_t gbhdr_v;
7097         char *vni_m;
7098         char *vni_v;
7099         size_t size, i;
7100
7101         if (inner) {
7102                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7103                                          inner_headers);
7104                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7105         } else {
7106                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7107                                          outer_headers);
7108                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7109         }
7110         dport = MLX5_UDP_PORT_GENEVE;
7111         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7112                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7113                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7114         }
7115         if (!geneve_v)
7116                 return;
7117         if (!geneve_m)
7118                 geneve_m = &rte_flow_item_geneve_mask;
7119         size = sizeof(geneve_m->vni);
7120         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
7121         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
7122         memcpy(vni_m, geneve_m->vni, size);
7123         for (i = 0; i < size; ++i)
7124                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
7125         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
7126                  rte_be_to_cpu_16(geneve_m->protocol));
7127         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
7128                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
7129         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
7130         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
7131         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
7132                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7133         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
7134                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7135         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
7136                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7137         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
7138                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
7139                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7140 }
7141
7142 /**
7143  * Add MPLS item to matcher and to the value.
7144  *
7145  * @param[in, out] matcher
7146  *   Flow matcher.
7147  * @param[in, out] key
7148  *   Flow matcher value.
7149  * @param[in] item
7150  *   Flow pattern to translate.
7151  * @param[in] prev_layer
7152  *   The protocol layer indicated in previous item.
7153  * @param[in] inner
7154  *   Item is inner pattern.
7155  */
7156 static void
7157 flow_dv_translate_item_mpls(void *matcher, void *key,
7158                             const struct rte_flow_item *item,
7159                             uint64_t prev_layer,
7160                             int inner)
7161 {
7162         const uint32_t *in_mpls_m = item->mask;
7163         const uint32_t *in_mpls_v = item->spec;
7164         uint32_t *out_mpls_m = 0;
7165         uint32_t *out_mpls_v = 0;
7166         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7167         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7168         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
7169                                      misc_parameters_2);
7170         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7171         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
7172         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7173
7174         switch (prev_layer) {
7175         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7176                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
7177                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
7178                          MLX5_UDP_PORT_MPLS);
7179                 break;
7180         case MLX5_FLOW_LAYER_GRE:
7181                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
7182                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
7183                          RTE_ETHER_TYPE_MPLS);
7184                 break;
7185         default:
7186                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
7187                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
7188                          IPPROTO_MPLS);
7189                 break;
7190         }
7191         if (!in_mpls_v)
7192                 return;
7193         if (!in_mpls_m)
7194                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
7195         switch (prev_layer) {
7196         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7197                 out_mpls_m =
7198                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7199                                                  outer_first_mpls_over_udp);
7200                 out_mpls_v =
7201                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7202                                                  outer_first_mpls_over_udp);
7203                 break;
7204         case MLX5_FLOW_LAYER_GRE:
7205                 out_mpls_m =
7206                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7207                                                  outer_first_mpls_over_gre);
7208                 out_mpls_v =
7209                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7210                                                  outer_first_mpls_over_gre);
7211                 break;
7212         default:
7213                 /* Inner MPLS not over GRE is not supported. */
7214                 if (!inner) {
7215                         out_mpls_m =
7216                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7217                                                          misc2_m,
7218                                                          outer_first_mpls);
7219                         out_mpls_v =
7220                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7221                                                          misc2_v,
7222                                                          outer_first_mpls);
7223                 }
7224                 break;
7225         }
7226         if (out_mpls_m && out_mpls_v) {
7227                 *out_mpls_m = *in_mpls_m;
7228                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
7229         }
7230 }
7231
7232 /**
7233  * Add metadata register item to matcher
7234  *
7235  * @param[in, out] matcher
7236  *   Flow matcher.
7237  * @param[in, out] key
7238  *   Flow matcher value.
7239  * @param[in] reg_type
7240  *   Type of device metadata register
7241  * @param[in] value
7242  *   Register value
7243  * @param[in] mask
7244  *   Register mask
7245  */
7246 static void
7247 flow_dv_match_meta_reg(void *matcher, void *key,
7248                        enum modify_reg reg_type,
7249                        uint32_t data, uint32_t mask)
7250 {
7251         void *misc2_m =
7252                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
7253         void *misc2_v =
7254                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7255         uint32_t temp;
7256
7257         data &= mask;
7258         switch (reg_type) {
7259         case REG_A:
7260                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
7261                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
7262                 break;
7263         case REG_B:
7264                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
7265                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
7266                 break;
7267         case REG_C_0:
7268                 /*
7269                  * The metadata register C0 field might be divided into
7270                  * source vport index and META item value, we should set
7271                  * this field according to specified mask, not as whole one.
7272                  */
7273                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
7274                 temp |= mask;
7275                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
7276                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
7277                 temp &= ~mask;
7278                 temp |= data;
7279                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
7280                 break;
7281         case REG_C_1:
7282                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
7283                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
7284                 break;
7285         case REG_C_2:
7286                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
7287                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
7288                 break;
7289         case REG_C_3:
7290                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
7291                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
7292                 break;
7293         case REG_C_4:
7294                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
7295                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
7296                 break;
7297         case REG_C_5:
7298                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
7299                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
7300                 break;
7301         case REG_C_6:
7302                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
7303                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
7304                 break;
7305         case REG_C_7:
7306                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
7307                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
7308                 break;
7309         default:
7310                 MLX5_ASSERT(false);
7311                 break;
7312         }
7313 }
7314
7315 /**
7316  * Add MARK item to matcher
7317  *
7318  * @param[in] dev
7319  *   The device to configure through.
7320  * @param[in, out] matcher
7321  *   Flow matcher.
7322  * @param[in, out] key
7323  *   Flow matcher value.
7324  * @param[in] item
7325  *   Flow pattern to translate.
7326  */
7327 static void
7328 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
7329                             void *matcher, void *key,
7330                             const struct rte_flow_item *item)
7331 {
7332         struct mlx5_priv *priv = dev->data->dev_private;
7333         const struct rte_flow_item_mark *mark;
7334         uint32_t value;
7335         uint32_t mask;
7336
7337         mark = item->mask ? (const void *)item->mask :
7338                             &rte_flow_item_mark_mask;
7339         mask = mark->id & priv->sh->dv_mark_mask;
7340         mark = (const void *)item->spec;
7341         MLX5_ASSERT(mark);
7342         value = mark->id & priv->sh->dv_mark_mask & mask;
7343         if (mask) {
7344                 enum modify_reg reg;
7345
7346                 /* Get the metadata register index for the mark. */
7347                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
7348                 MLX5_ASSERT(reg > 0);
7349                 if (reg == REG_C_0) {
7350                         struct mlx5_priv *priv = dev->data->dev_private;
7351                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7352                         uint32_t shl_c0 = rte_bsf32(msk_c0);
7353
7354                         mask &= msk_c0;
7355                         mask <<= shl_c0;
7356                         value <<= shl_c0;
7357                 }
7358                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7359         }
7360 }
7361
7362 /**
7363  * Add META item to matcher
7364  *
7365  * @param[in] dev
7366  *   The devich to configure through.
7367  * @param[in, out] matcher
7368  *   Flow matcher.
7369  * @param[in, out] key
7370  *   Flow matcher value.
7371  * @param[in] attr
7372  *   Attributes of flow that includes this item.
7373  * @param[in] item
7374  *   Flow pattern to translate.
7375  */
7376 static void
7377 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
7378                             void *matcher, void *key,
7379                             const struct rte_flow_attr *attr,
7380                             const struct rte_flow_item *item)
7381 {
7382         const struct rte_flow_item_meta *meta_m;
7383         const struct rte_flow_item_meta *meta_v;
7384
7385         meta_m = (const void *)item->mask;
7386         if (!meta_m)
7387                 meta_m = &rte_flow_item_meta_mask;
7388         meta_v = (const void *)item->spec;
7389         if (meta_v) {
7390                 int reg;
7391                 uint32_t value = meta_v->data;
7392                 uint32_t mask = meta_m->data;
7393
7394                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
7395                 if (reg < 0)
7396                         return;
7397                 /*
7398                  * In datapath code there is no endianness
7399                  * coversions for perfromance reasons, all
7400                  * pattern conversions are done in rte_flow.
7401                  */
7402                 value = rte_cpu_to_be_32(value);
7403                 mask = rte_cpu_to_be_32(mask);
7404                 if (reg == REG_C_0) {
7405                         struct mlx5_priv *priv = dev->data->dev_private;
7406                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7407                         uint32_t shl_c0 = rte_bsf32(msk_c0);
7408 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
7409                         uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
7410
7411                         value >>= shr_c0;
7412                         mask >>= shr_c0;
7413 #endif
7414                         value <<= shl_c0;
7415                         mask <<= shl_c0;
7416                         MLX5_ASSERT(msk_c0);
7417                         MLX5_ASSERT(!(~msk_c0 & mask));
7418                 }
7419                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7420         }
7421 }
7422
7423 /**
7424  * Add vport metadata Reg C0 item to matcher
7425  *
7426  * @param[in, out] matcher
7427  *   Flow matcher.
7428  * @param[in, out] key
7429  *   Flow matcher value.
7430  * @param[in] reg
7431  *   Flow pattern to translate.
7432  */
7433 static void
7434 flow_dv_translate_item_meta_vport(void *matcher, void *key,
7435                                   uint32_t value, uint32_t mask)
7436 {
7437         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
7438 }
7439
7440 /**
7441  * Add tag item to matcher
7442  *
7443  * @param[in] dev
7444  *   The devich to configure through.
7445  * @param[in, out] matcher
7446  *   Flow matcher.
7447  * @param[in, out] key
7448  *   Flow matcher value.
7449  * @param[in] item
7450  *   Flow pattern to translate.
7451  */
7452 static void
7453 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
7454                                 void *matcher, void *key,
7455                                 const struct rte_flow_item *item)
7456 {
7457         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
7458         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
7459         uint32_t mask, value;
7460
7461         MLX5_ASSERT(tag_v);
7462         value = tag_v->data;
7463         mask = tag_m ? tag_m->data : UINT32_MAX;
7464         if (tag_v->id == REG_C_0) {
7465                 struct mlx5_priv *priv = dev->data->dev_private;
7466                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7467                 uint32_t shl_c0 = rte_bsf32(msk_c0);
7468
7469                 mask &= msk_c0;
7470                 mask <<= shl_c0;
7471                 value <<= shl_c0;
7472         }
7473         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
7474 }
7475
7476 /**
7477  * Add TAG item to matcher
7478  *
7479  * @param[in] dev
7480  *   The devich to configure through.
7481  * @param[in, out] matcher
7482  *   Flow matcher.
7483  * @param[in, out] key
7484  *   Flow matcher value.
7485  * @param[in] item
7486  *   Flow pattern to translate.
7487  */
7488 static void
7489 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
7490                            void *matcher, void *key,
7491                            const struct rte_flow_item *item)
7492 {
7493         const struct rte_flow_item_tag *tag_v = item->spec;
7494         const struct rte_flow_item_tag *tag_m = item->mask;
7495         enum modify_reg reg;
7496
7497         MLX5_ASSERT(tag_v);
7498         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
7499         /* Get the metadata register index for the tag. */
7500         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
7501         MLX5_ASSERT(reg > 0);
7502         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
7503 }
7504
7505 /**
7506  * Add source vport match to the specified matcher.
7507  *
7508  * @param[in, out] matcher
7509  *   Flow matcher.
7510  * @param[in, out] key
7511  *   Flow matcher value.
7512  * @param[in] port
7513  *   Source vport value to match
7514  * @param[in] mask
7515  *   Mask
7516  */
7517 static void
7518 flow_dv_translate_item_source_vport(void *matcher, void *key,
7519                                     int16_t port, uint16_t mask)
7520 {
7521         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7522         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7523
7524         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
7525         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
7526 }
7527
7528 /**
7529  * Translate port-id item to eswitch match on  port-id.
7530  *
7531  * @param[in] dev
7532  *   The devich to configure through.
7533  * @param[in, out] matcher
7534  *   Flow matcher.
7535  * @param[in, out] key
7536  *   Flow matcher value.
7537  * @param[in] item
7538  *   Flow pattern to translate.
7539  *
7540  * @return
7541  *   0 on success, a negative errno value otherwise.
7542  */
7543 static int
7544 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
7545                                void *key, const struct rte_flow_item *item)
7546 {
7547         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
7548         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
7549         struct mlx5_priv *priv;
7550         uint16_t mask, id;
7551
7552         mask = pid_m ? pid_m->id : 0xffff;
7553         id = pid_v ? pid_v->id : dev->data->port_id;
7554         priv = mlx5_port_to_eswitch_info(id, item == NULL);
7555         if (!priv)
7556                 return -rte_errno;
7557         /* Translate to vport field or to metadata, depending on mode. */
7558         if (priv->vport_meta_mask)
7559                 flow_dv_translate_item_meta_vport(matcher, key,
7560                                                   priv->vport_meta_tag,
7561                                                   priv->vport_meta_mask);
7562         else
7563                 flow_dv_translate_item_source_vport(matcher, key,
7564                                                     priv->vport_id, mask);
7565         return 0;
7566 }
7567
7568 /**
7569  * Add ICMP6 item to matcher and to the value.
7570  *
7571  * @param[in, out] matcher
7572  *   Flow matcher.
7573  * @param[in, out] key
7574  *   Flow matcher value.
7575  * @param[in] item
7576  *   Flow pattern to translate.
7577  * @param[in] inner
7578  *   Item is inner pattern.
7579  */
7580 static void
7581 flow_dv_translate_item_icmp6(void *matcher, void *key,
7582                               const struct rte_flow_item *item,
7583                               int inner)
7584 {
7585         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
7586         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
7587         void *headers_m;
7588         void *headers_v;
7589         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7590                                      misc_parameters_3);
7591         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7592         if (inner) {
7593                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7594                                          inner_headers);
7595                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7596         } else {
7597                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7598                                          outer_headers);
7599                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7600         }
7601         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7602         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
7603         if (!icmp6_v)
7604                 return;
7605         if (!icmp6_m)
7606                 icmp6_m = &rte_flow_item_icmp6_mask;
7607         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
7608         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
7609                  icmp6_v->type & icmp6_m->type);
7610         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
7611         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
7612                  icmp6_v->code & icmp6_m->code);
7613 }
7614
7615 /**
7616  * Add ICMP item to matcher and to the value.
7617  *
7618  * @param[in, out] matcher
7619  *   Flow matcher.
7620  * @param[in, out] key
7621  *   Flow matcher value.
7622  * @param[in] item
7623  *   Flow pattern to translate.
7624  * @param[in] inner
7625  *   Item is inner pattern.
7626  */
7627 static void
7628 flow_dv_translate_item_icmp(void *matcher, void *key,
7629                             const struct rte_flow_item *item,
7630                             int inner)
7631 {
7632         const struct rte_flow_item_icmp *icmp_m = item->mask;
7633         const struct rte_flow_item_icmp *icmp_v = item->spec;
7634         uint32_t icmp_header_data_m = 0;
7635         uint32_t icmp_header_data_v = 0;
7636         void *headers_m;
7637         void *headers_v;
7638         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7639                                      misc_parameters_3);
7640         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7641         if (inner) {
7642                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7643                                          inner_headers);
7644                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7645         } else {
7646                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7647                                          outer_headers);
7648                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7649         }
7650         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7651         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
7652         if (!icmp_v)
7653                 return;
7654         if (!icmp_m)
7655                 icmp_m = &rte_flow_item_icmp_mask;
7656         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
7657                  icmp_m->hdr.icmp_type);
7658         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
7659                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
7660         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
7661                  icmp_m->hdr.icmp_code);
7662         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
7663                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
7664         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
7665         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
7666         if (icmp_header_data_m) {
7667                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
7668                 icmp_header_data_v |=
7669                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
7670                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
7671                          icmp_header_data_m);
7672                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
7673                          icmp_header_data_v & icmp_header_data_m);
7674         }
7675 }
7676
7677 /**
7678  * Add GTP item to matcher and to the value.
7679  *
7680  * @param[in, out] matcher
7681  *   Flow matcher.
7682  * @param[in, out] key
7683  *   Flow matcher value.
7684  * @param[in] item
7685  *   Flow pattern to translate.
7686  * @param[in] inner
7687  *   Item is inner pattern.
7688  */
7689 static void
7690 flow_dv_translate_item_gtp(void *matcher, void *key,
7691                            const struct rte_flow_item *item, int inner)
7692 {
7693         const struct rte_flow_item_gtp *gtp_m = item->mask;
7694         const struct rte_flow_item_gtp *gtp_v = item->spec;
7695         void *headers_m;
7696         void *headers_v;
7697         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7698                                      misc_parameters_3);
7699         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7700         uint16_t dport = RTE_GTPU_UDP_PORT;
7701
7702         if (inner) {
7703                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7704                                          inner_headers);
7705                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7706         } else {
7707                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7708                                          outer_headers);
7709                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7710         }
7711         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7712                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7713                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7714         }
7715         if (!gtp_v)
7716                 return;
7717         if (!gtp_m)
7718                 gtp_m = &rte_flow_item_gtp_mask;
7719         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
7720                  gtp_m->v_pt_rsv_flags);
7721         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
7722                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
7723         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
7724         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
7725                  gtp_v->msg_type & gtp_m->msg_type);
7726         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
7727                  rte_be_to_cpu_32(gtp_m->teid));
7728         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
7729                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
7730 }
7731
7732 /**
7733  * Add eCPRI item to matcher and to the value.
7734  *
7735  * @param[in] dev
7736  *   The devich to configure through.
7737  * @param[in, out] matcher
7738  *   Flow matcher.
7739  * @param[in, out] key
7740  *   Flow matcher value.
7741  * @param[in] item
7742  *   Flow pattern to translate.
7743  * @param[in] samples
7744  *   Sample IDs to be used in the matching.
7745  */
7746 static void
7747 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
7748                              void *key, const struct rte_flow_item *item)
7749 {
7750         struct mlx5_priv *priv = dev->data->dev_private;
7751         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
7752         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
7753         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
7754                                      misc_parameters_4);
7755         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
7756         uint32_t *samples;
7757         void *dw_m;
7758         void *dw_v;
7759
7760         if (!ecpri_v)
7761                 return;
7762         if (!ecpri_m)
7763                 ecpri_m = &rte_flow_item_ecpri_mask;
7764         /*
7765          * Maximal four DW samples are supported in a single matching now.
7766          * Two are used now for a eCPRI matching:
7767          * 1. Type: one byte, mask should be 0x00ff0000 in network order
7768          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
7769          *    if any.
7770          */
7771         if (!ecpri_m->hdr.common.u32)
7772                 return;
7773         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
7774         /* Need to take the whole DW as the mask to fill the entry. */
7775         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
7776                             prog_sample_field_value_0);
7777         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
7778                             prog_sample_field_value_0);
7779         /* Already big endian (network order) in the header. */
7780         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
7781         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32;
7782         /* Sample#0, used for matching type, offset 0. */
7783         MLX5_SET(fte_match_set_misc4, misc4_m,
7784                  prog_sample_field_id_0, samples[0]);
7785         /* It makes no sense to set the sample ID in the mask field. */
7786         MLX5_SET(fte_match_set_misc4, misc4_v,
7787                  prog_sample_field_id_0, samples[0]);
7788         /*
7789          * Checking if message body part needs to be matched.
7790          * Some wildcard rules only matching type field should be supported.
7791          */
7792         if (ecpri_m->hdr.dummy[0]) {
7793                 switch (ecpri_v->hdr.common.type) {
7794                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
7795                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
7796                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
7797                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
7798                                             prog_sample_field_value_1);
7799                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
7800                                             prog_sample_field_value_1);
7801                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
7802                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0];
7803                         /* Sample#1, to match message body, offset 4. */
7804                         MLX5_SET(fte_match_set_misc4, misc4_m,
7805                                  prog_sample_field_id_1, samples[1]);
7806                         MLX5_SET(fte_match_set_misc4, misc4_v,
7807                                  prog_sample_field_id_1, samples[1]);
7808                         break;
7809                 default:
7810                         /* Others, do not match any sample ID. */
7811                         break;
7812                 }
7813         }
7814 }
7815
7816 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
7817
7818 #define HEADER_IS_ZERO(match_criteria, headers)                              \
7819         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
7820                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
7821
7822 /**
7823  * Calculate flow matcher enable bitmap.
7824  *
7825  * @param match_criteria
7826  *   Pointer to flow matcher criteria.
7827  *
7828  * @return
7829  *   Bitmap of enabled fields.
7830  */
7831 static uint8_t
7832 flow_dv_matcher_enable(uint32_t *match_criteria)
7833 {
7834         uint8_t match_criteria_enable;
7835
7836         match_criteria_enable =
7837                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
7838                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
7839         match_criteria_enable |=
7840                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
7841                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
7842         match_criteria_enable |=
7843                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
7844                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
7845         match_criteria_enable |=
7846                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
7847                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
7848         match_criteria_enable |=
7849                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
7850                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
7851         match_criteria_enable |=
7852                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
7853                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
7854         return match_criteria_enable;
7855 }
7856
7857
7858 /**
7859  * Get a flow table.
7860  *
7861  * @param[in, out] dev
7862  *   Pointer to rte_eth_dev structure.
7863  * @param[in] table_id
7864  *   Table id to use.
7865  * @param[in] egress
7866  *   Direction of the table.
7867  * @param[in] transfer
7868  *   E-Switch or NIC flow.
7869  * @param[out] error
7870  *   pointer to error structure.
7871  *
7872  * @return
7873  *   Returns tables resource based on the index, NULL in case of failed.
7874  */
7875 static struct mlx5_flow_tbl_resource *
7876 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
7877                          uint32_t table_id, uint8_t egress,
7878                          uint8_t transfer,
7879                          struct rte_flow_error *error)
7880 {
7881         struct mlx5_priv *priv = dev->data->dev_private;
7882         struct mlx5_dev_ctx_shared *sh = priv->sh;
7883         struct mlx5_flow_tbl_resource *tbl;
7884         union mlx5_flow_tbl_key table_key = {
7885                 {
7886                         .table_id = table_id,
7887                         .reserved = 0,
7888                         .domain = !!transfer,
7889                         .direction = !!egress,
7890                 }
7891         };
7892         struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
7893                                                          table_key.v64);
7894         struct mlx5_flow_tbl_data_entry *tbl_data;
7895         uint32_t idx = 0;
7896         int ret;
7897         void *domain;
7898
7899         if (pos) {
7900                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
7901                                         entry);
7902                 tbl = &tbl_data->tbl;
7903                 rte_atomic32_inc(&tbl->refcnt);
7904                 return tbl;
7905         }
7906         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
7907         if (!tbl_data) {
7908                 rte_flow_error_set(error, ENOMEM,
7909                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7910                                    NULL,
7911                                    "cannot allocate flow table data entry");
7912                 return NULL;
7913         }
7914         tbl_data->idx = idx;
7915         tbl = &tbl_data->tbl;
7916         pos = &tbl_data->entry;
7917         if (transfer)
7918                 domain = sh->fdb_domain;
7919         else if (egress)
7920                 domain = sh->tx_domain;
7921         else
7922                 domain = sh->rx_domain;
7923         ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj);
7924         if (ret) {
7925                 rte_flow_error_set(error, ENOMEM,
7926                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7927                                    NULL, "cannot create flow table object");
7928                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7929                 return NULL;
7930         }
7931         /*
7932          * No multi-threads now, but still better to initialize the reference
7933          * count before insert it into the hash list.
7934          */
7935         rte_atomic32_init(&tbl->refcnt);
7936         /* Jump action reference count is initialized here. */
7937         rte_atomic32_init(&tbl_data->jump.refcnt);
7938         pos->key = table_key.v64;
7939         ret = mlx5_hlist_insert(sh->flow_tbls, pos);
7940         if (ret < 0) {
7941                 rte_flow_error_set(error, -ret,
7942                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7943                                    "cannot insert flow table data entry");
7944                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
7945                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7946         }
7947         rte_atomic32_inc(&tbl->refcnt);
7948         return tbl;
7949 }
7950
7951 /**
7952  * Release a flow table.
7953  *
7954  * @param[in] dev
7955  *   Pointer to rte_eth_dev structure.
7956  * @param[in] tbl
7957  *   Table resource to be released.
7958  *
7959  * @return
7960  *   Returns 0 if table was released, else return 1;
7961  */
7962 static int
7963 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
7964                              struct mlx5_flow_tbl_resource *tbl)
7965 {
7966         struct mlx5_priv *priv = dev->data->dev_private;
7967         struct mlx5_dev_ctx_shared *sh = priv->sh;
7968         struct mlx5_flow_tbl_data_entry *tbl_data =
7969                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7970
7971         if (!tbl)
7972                 return 0;
7973         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
7974                 struct mlx5_hlist_entry *pos = &tbl_data->entry;
7975
7976                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
7977                 tbl->obj = NULL;
7978                 /* remove the entry from the hash list and free memory. */
7979                 mlx5_hlist_remove(sh->flow_tbls, pos);
7980                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
7981                                 tbl_data->idx);
7982                 return 0;
7983         }
7984         return 1;
7985 }
7986
7987 /**
7988  * Register the flow matcher.
7989  *
7990  * @param[in, out] dev
7991  *   Pointer to rte_eth_dev structure.
7992  * @param[in, out] matcher
7993  *   Pointer to flow matcher.
7994  * @param[in, out] key
7995  *   Pointer to flow table key.
7996  * @parm[in, out] dev_flow
7997  *   Pointer to the dev_flow.
7998  * @param[out] error
7999  *   pointer to error structure.
8000  *
8001  * @return
8002  *   0 on success otherwise -errno and errno is set.
8003  */
8004 static int
8005 flow_dv_matcher_register(struct rte_eth_dev *dev,
8006                          struct mlx5_flow_dv_matcher *matcher,
8007                          union mlx5_flow_tbl_key *key,
8008                          struct mlx5_flow *dev_flow,
8009                          struct rte_flow_error *error)
8010 {
8011         struct mlx5_priv *priv = dev->data->dev_private;
8012         struct mlx5_dev_ctx_shared *sh = priv->sh;
8013         struct mlx5_flow_dv_matcher *cache_matcher;
8014         struct mlx5dv_flow_matcher_attr dv_attr = {
8015                 .type = IBV_FLOW_ATTR_NORMAL,
8016                 .match_mask = (void *)&matcher->mask,
8017         };
8018         struct mlx5_flow_tbl_resource *tbl;
8019         struct mlx5_flow_tbl_data_entry *tbl_data;
8020         int ret;
8021
8022         tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
8023                                        key->domain, error);
8024         if (!tbl)
8025                 return -rte_errno;      /* No need to refill the error info */
8026         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
8027         /* Lookup from cache. */
8028         LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
8029                 if (matcher->crc == cache_matcher->crc &&
8030                     matcher->priority == cache_matcher->priority &&
8031                     !memcmp((const void *)matcher->mask.buf,
8032                             (const void *)cache_matcher->mask.buf,
8033                             cache_matcher->mask.size)) {
8034                         DRV_LOG(DEBUG,
8035                                 "%s group %u priority %hd use %s "
8036                                 "matcher %p: refcnt %d++",
8037                                 key->domain ? "FDB" : "NIC", key->table_id,
8038                                 cache_matcher->priority,
8039                                 key->direction ? "tx" : "rx",
8040                                 (void *)cache_matcher,
8041                                 rte_atomic32_read(&cache_matcher->refcnt));
8042                         rte_atomic32_inc(&cache_matcher->refcnt);
8043                         dev_flow->handle->dvh.matcher = cache_matcher;
8044                         /* old matcher should not make the table ref++. */
8045                         flow_dv_tbl_resource_release(dev, tbl);
8046                         return 0;
8047                 }
8048         }
8049         /* Register new matcher. */
8050         cache_matcher = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache_matcher), 0,
8051                                     SOCKET_ID_ANY);
8052         if (!cache_matcher) {
8053                 flow_dv_tbl_resource_release(dev, tbl);
8054                 return rte_flow_error_set(error, ENOMEM,
8055                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8056                                           "cannot allocate matcher memory");
8057         }
8058         *cache_matcher = *matcher;
8059         dv_attr.match_criteria_enable =
8060                 flow_dv_matcher_enable(cache_matcher->mask.buf);
8061         dv_attr.priority = matcher->priority;
8062         if (key->direction)
8063                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
8064         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
8065                                                &cache_matcher->matcher_object);
8066         if (ret) {
8067                 mlx5_free(cache_matcher);
8068 #ifdef HAVE_MLX5DV_DR
8069                 flow_dv_tbl_resource_release(dev, tbl);
8070 #endif
8071                 return rte_flow_error_set(error, ENOMEM,
8072                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8073                                           NULL, "cannot create matcher");
8074         }
8075         /* Save the table information */
8076         cache_matcher->tbl = tbl;
8077         rte_atomic32_init(&cache_matcher->refcnt);
8078         /* only matcher ref++, table ref++ already done above in get API. */
8079         rte_atomic32_inc(&cache_matcher->refcnt);
8080         LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
8081         dev_flow->handle->dvh.matcher = cache_matcher;
8082         DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
8083                 key->domain ? "FDB" : "NIC", key->table_id,
8084                 cache_matcher->priority,
8085                 key->direction ? "tx" : "rx", (void *)cache_matcher,
8086                 rte_atomic32_read(&cache_matcher->refcnt));
8087         return 0;
8088 }
8089
8090 /**
8091  * Find existing tag resource or create and register a new one.
8092  *
8093  * @param dev[in, out]
8094  *   Pointer to rte_eth_dev structure.
8095  * @param[in, out] tag_be24
8096  *   Tag value in big endian then R-shift 8.
8097  * @parm[in, out] dev_flow
8098  *   Pointer to the dev_flow.
8099  * @param[out] error
8100  *   pointer to error structure.
8101  *
8102  * @return
8103  *   0 on success otherwise -errno and errno is set.
8104  */
8105 static int
8106 flow_dv_tag_resource_register
8107                         (struct rte_eth_dev *dev,
8108                          uint32_t tag_be24,
8109                          struct mlx5_flow *dev_flow,
8110                          struct rte_flow_error *error)
8111 {
8112         struct mlx5_priv *priv = dev->data->dev_private;
8113         struct mlx5_dev_ctx_shared *sh = priv->sh;
8114         struct mlx5_flow_dv_tag_resource *cache_resource;
8115         struct mlx5_hlist_entry *entry;
8116         int ret;
8117
8118         /* Lookup a matching resource from cache. */
8119         entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
8120         if (entry) {
8121                 cache_resource = container_of
8122                         (entry, struct mlx5_flow_dv_tag_resource, entry);
8123                 rte_atomic32_inc(&cache_resource->refcnt);
8124                 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
8125                 dev_flow->dv.tag_resource = cache_resource;
8126                 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
8127                         (void *)cache_resource,
8128                         rte_atomic32_read(&cache_resource->refcnt));
8129                 return 0;
8130         }
8131         /* Register new resource. */
8132         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG],
8133                                        &dev_flow->handle->dvh.rix_tag);
8134         if (!cache_resource)
8135                 return rte_flow_error_set(error, ENOMEM,
8136                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8137                                           "cannot allocate resource memory");
8138         cache_resource->entry.key = (uint64_t)tag_be24;
8139         ret = mlx5_flow_os_create_flow_action_tag(tag_be24,
8140                                                   &cache_resource->action);
8141         if (ret) {
8142                 mlx5_free(cache_resource);
8143                 return rte_flow_error_set(error, ENOMEM,
8144                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8145                                           NULL, "cannot create action");
8146         }
8147         rte_atomic32_init(&cache_resource->refcnt);
8148         rte_atomic32_inc(&cache_resource->refcnt);
8149         if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
8150                 mlx5_flow_os_destroy_flow_action(cache_resource->action);
8151                 mlx5_free(cache_resource);
8152                 return rte_flow_error_set(error, EEXIST,
8153                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8154                                           NULL, "cannot insert tag");
8155         }
8156         dev_flow->dv.tag_resource = cache_resource;
8157         DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
8158                 (void *)cache_resource,
8159                 rte_atomic32_read(&cache_resource->refcnt));
8160         return 0;
8161 }
8162
8163 /**
8164  * Release the tag.
8165  *
8166  * @param dev
8167  *   Pointer to Ethernet device.
8168  * @param tag_idx
8169  *   Tag index.
8170  *
8171  * @return
8172  *   1 while a reference on it exists, 0 when freed.
8173  */
8174 static int
8175 flow_dv_tag_release(struct rte_eth_dev *dev,
8176                     uint32_t tag_idx)
8177 {
8178         struct mlx5_priv *priv = dev->data->dev_private;
8179         struct mlx5_dev_ctx_shared *sh = priv->sh;
8180         struct mlx5_flow_dv_tag_resource *tag;
8181
8182         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8183         if (!tag)
8184                 return 0;
8185         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
8186                 dev->data->port_id, (void *)tag,
8187                 rte_atomic32_read(&tag->refcnt));
8188         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
8189                 claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
8190                 mlx5_hlist_remove(sh->tag_table, &tag->entry);
8191                 DRV_LOG(DEBUG, "port %u tag %p: removed",
8192                         dev->data->port_id, (void *)tag);
8193                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8194                 return 0;
8195         }
8196         return 1;
8197 }
8198
8199 /**
8200  * Translate port ID action to vport.
8201  *
8202  * @param[in] dev
8203  *   Pointer to rte_eth_dev structure.
8204  * @param[in] action
8205  *   Pointer to the port ID action.
8206  * @param[out] dst_port_id
8207  *   The target port ID.
8208  * @param[out] error
8209  *   Pointer to the error structure.
8210  *
8211  * @return
8212  *   0 on success, a negative errno value otherwise and rte_errno is set.
8213  */
8214 static int
8215 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
8216                                  const struct rte_flow_action *action,
8217                                  uint32_t *dst_port_id,
8218                                  struct rte_flow_error *error)
8219 {
8220         uint32_t port;
8221         struct mlx5_priv *priv;
8222         const struct rte_flow_action_port_id *conf =
8223                         (const struct rte_flow_action_port_id *)action->conf;
8224
8225         port = conf->original ? dev->data->port_id : conf->id;
8226         priv = mlx5_port_to_eswitch_info(port, false);
8227         if (!priv)
8228                 return rte_flow_error_set(error, -rte_errno,
8229                                           RTE_FLOW_ERROR_TYPE_ACTION,
8230                                           NULL,
8231                                           "No eswitch info was found for port");
8232 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
8233         /*
8234          * This parameter is transferred to
8235          * mlx5dv_dr_action_create_dest_ib_port().
8236          */
8237         *dst_port_id = priv->dev_port;
8238 #else
8239         /*
8240          * Legacy mode, no LAG configurations is supported.
8241          * This parameter is transferred to
8242          * mlx5dv_dr_action_create_dest_vport().
8243          */
8244         *dst_port_id = priv->vport_id;
8245 #endif
8246         return 0;
8247 }
8248
8249 /**
8250  * Create a counter with aging configuration.
8251  *
8252  * @param[in] dev
8253  *   Pointer to rte_eth_dev structure.
8254  * @param[out] count
8255  *   Pointer to the counter action configuration.
8256  * @param[in] age
8257  *   Pointer to the aging action configuration.
8258  *
8259  * @return
8260  *   Index to flow counter on success, 0 otherwise.
8261  */
8262 static uint32_t
8263 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
8264                                 struct mlx5_flow *dev_flow,
8265                                 const struct rte_flow_action_count *count,
8266                                 const struct rte_flow_action_age *age)
8267 {
8268         uint32_t counter;
8269         struct mlx5_age_param *age_param;
8270
8271         if (count && count->shared)
8272                 counter = flow_dv_counter_get_shared(dev, count->id);
8273         else
8274                 counter = flow_dv_counter_alloc(dev, !!age);
8275         if (!counter || age == NULL)
8276                 return counter;
8277         age_param  = flow_dv_counter_idx_get_age(dev, counter);
8278         age_param->context = age->context ? age->context :
8279                 (void *)(uintptr_t)(dev_flow->flow_idx);
8280         age_param->timeout = age->timeout;
8281         age_param->port_id = dev->data->port_id;
8282         __atomic_store_n(&age_param->sec_since_last_hit, 0, __ATOMIC_RELAXED);
8283         __atomic_store_n(&age_param->state, AGE_CANDIDATE, __ATOMIC_RELAXED);
8284         return counter;
8285 }
8286 /**
8287  * Add Tx queue matcher
8288  *
8289  * @param[in] dev
8290  *   Pointer to the dev struct.
8291  * @param[in, out] matcher
8292  *   Flow matcher.
8293  * @param[in, out] key
8294  *   Flow matcher value.
8295  * @param[in] item
8296  *   Flow pattern to translate.
8297  * @param[in] inner
8298  *   Item is inner pattern.
8299  */
8300 static void
8301 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
8302                                 void *matcher, void *key,
8303                                 const struct rte_flow_item *item)
8304 {
8305         const struct mlx5_rte_flow_item_tx_queue *queue_m;
8306         const struct mlx5_rte_flow_item_tx_queue *queue_v;
8307         void *misc_m =
8308                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8309         void *misc_v =
8310                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8311         struct mlx5_txq_ctrl *txq;
8312         uint32_t queue;
8313
8314
8315         queue_m = (const void *)item->mask;
8316         if (!queue_m)
8317                 return;
8318         queue_v = (const void *)item->spec;
8319         if (!queue_v)
8320                 return;
8321         txq = mlx5_txq_get(dev, queue_v->queue);
8322         if (!txq)
8323                 return;
8324         queue = txq->obj->sq->id;
8325         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
8326         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
8327                  queue & queue_m->queue);
8328         mlx5_txq_release(dev, queue_v->queue);
8329 }
8330
8331 /**
8332  * Set the hash fields according to the @p flow information.
8333  *
8334  * @param[in] dev_flow
8335  *   Pointer to the mlx5_flow.
8336  * @param[in] rss_desc
8337  *   Pointer to the mlx5_flow_rss_desc.
8338  */
8339 static void
8340 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
8341                        struct mlx5_flow_rss_desc *rss_desc)
8342 {
8343         uint64_t items = dev_flow->handle->layers;
8344         int rss_inner = 0;
8345         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
8346
8347         dev_flow->hash_fields = 0;
8348 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
8349         if (rss_desc->level >= 2) {
8350                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
8351                 rss_inner = 1;
8352         }
8353 #endif
8354         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
8355             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
8356                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
8357                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
8358                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
8359                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
8360                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
8361                         else
8362                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
8363                 }
8364         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
8365                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
8366                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
8367                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
8368                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
8369                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
8370                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
8371                         else
8372                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
8373                 }
8374         }
8375         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
8376             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
8377                 if (rss_types & ETH_RSS_UDP) {
8378                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
8379                                 dev_flow->hash_fields |=
8380                                                 IBV_RX_HASH_SRC_PORT_UDP;
8381                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
8382                                 dev_flow->hash_fields |=
8383                                                 IBV_RX_HASH_DST_PORT_UDP;
8384                         else
8385                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
8386                 }
8387         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
8388                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
8389                 if (rss_types & ETH_RSS_TCP) {
8390                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
8391                                 dev_flow->hash_fields |=
8392                                                 IBV_RX_HASH_SRC_PORT_TCP;
8393                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
8394                                 dev_flow->hash_fields |=
8395                                                 IBV_RX_HASH_DST_PORT_TCP;
8396                         else
8397                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
8398                 }
8399         }
8400 }
8401
8402 /**
8403  * Create an Rx Hash queue.
8404  *
8405  * @param dev
8406  *   Pointer to Ethernet device.
8407  * @param[in] dev_flow
8408  *   Pointer to the mlx5_flow.
8409  * @param[in] rss_desc
8410  *   Pointer to the mlx5_flow_rss_desc.
8411  * @param[out] hrxq_idx
8412  *   Hash Rx queue index.
8413  *
8414  * @return
8415  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
8416  */
8417 static struct mlx5_hrxq *
8418 flow_dv_handle_rx_queue(struct rte_eth_dev *dev,
8419                         struct mlx5_flow *dev_flow,
8420                         struct mlx5_flow_rss_desc *rss_desc,
8421                         uint32_t *hrxq_idx)
8422 {
8423         struct mlx5_priv *priv = dev->data->dev_private;
8424         struct mlx5_flow_handle *dh = dev_flow->handle;
8425         struct mlx5_hrxq *hrxq;
8426
8427         MLX5_ASSERT(rss_desc->queue_num);
8428         *hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key, MLX5_RSS_HASH_KEY_LEN,
8429                                   dev_flow->hash_fields,
8430                                   rss_desc->queue, rss_desc->queue_num);
8431         if (!*hrxq_idx) {
8432                 *hrxq_idx = mlx5_hrxq_new
8433                                 (dev, rss_desc->key, MLX5_RSS_HASH_KEY_LEN,
8434                                  dev_flow->hash_fields,
8435                                  rss_desc->queue, rss_desc->queue_num,
8436                                  !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL),
8437                                  false);
8438                 if (!*hrxq_idx)
8439                         return NULL;
8440         }
8441         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
8442                               *hrxq_idx);
8443         return hrxq;
8444 }
8445
8446 /**
8447  * Find existing sample resource or create and register a new one.
8448  *
8449  * @param[in, out] dev
8450  *   Pointer to rte_eth_dev structure.
8451  * @param[in] attr
8452  *   Attributes of flow that includes this item.
8453  * @param[in] resource
8454  *   Pointer to sample resource.
8455  * @parm[in, out] dev_flow
8456  *   Pointer to the dev_flow.
8457  * @param[in, out] sample_dv_actions
8458  *   Pointer to sample actions list.
8459  * @param[out] error
8460  *   pointer to error structure.
8461  *
8462  * @return
8463  *   0 on success otherwise -errno and errno is set.
8464  */
8465 static int
8466 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
8467                          const struct rte_flow_attr *attr,
8468                          struct mlx5_flow_dv_sample_resource *resource,
8469                          struct mlx5_flow *dev_flow,
8470                          void **sample_dv_actions,
8471                          struct rte_flow_error *error)
8472 {
8473         struct mlx5_flow_dv_sample_resource *cache_resource;
8474         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
8475         struct mlx5_priv *priv = dev->data->dev_private;
8476         struct mlx5_dev_ctx_shared *sh = priv->sh;
8477         struct mlx5_flow_tbl_resource *tbl;
8478         uint32_t idx = 0;
8479         const uint32_t next_ft_step = 1;
8480         uint32_t next_ft_id = resource->ft_id + next_ft_step;
8481
8482         /* Lookup a matching resource from cache. */
8483         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_SAMPLE], sh->sample_action_list,
8484                       idx, cache_resource, next) {
8485                 if (resource->ratio == cache_resource->ratio &&
8486                     resource->ft_type == cache_resource->ft_type &&
8487                     resource->ft_id == cache_resource->ft_id &&
8488                     resource->set_action == cache_resource->set_action &&
8489                     !memcmp((void *)&resource->sample_act,
8490                             (void *)&cache_resource->sample_act,
8491                             sizeof(struct mlx5_flow_sub_actions_list))) {
8492                         DRV_LOG(DEBUG, "sample resource %p: refcnt %d++",
8493                                 (void *)cache_resource,
8494                                 __atomic_load_n(&cache_resource->refcnt,
8495                                                 __ATOMIC_RELAXED));
8496                         __atomic_fetch_add(&cache_resource->refcnt, 1,
8497                                            __ATOMIC_RELAXED);
8498                         dev_flow->handle->dvh.rix_sample = idx;
8499                         dev_flow->dv.sample_res = cache_resource;
8500                         return 0;
8501                 }
8502         }
8503         /* Register new sample resource. */
8504         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE],
8505                                        &dev_flow->handle->dvh.rix_sample);
8506         if (!cache_resource)
8507                 return rte_flow_error_set(error, ENOMEM,
8508                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8509                                           NULL,
8510                                           "cannot allocate resource memory");
8511         *cache_resource = *resource;
8512         /* Create normal path table level */
8513         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
8514                                         attr->egress, attr->transfer, error);
8515         if (!tbl) {
8516                 rte_flow_error_set(error, ENOMEM,
8517                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8518                                           NULL,
8519                                           "fail to create normal path table "
8520                                           "for sample");
8521                 goto error;
8522         }
8523         cache_resource->normal_path_tbl = tbl;
8524         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
8525                 cache_resource->default_miss =
8526                                 mlx5_glue->dr_create_flow_action_default_miss();
8527                 if (!cache_resource->default_miss) {
8528                         rte_flow_error_set(error, ENOMEM,
8529                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8530                                                 NULL,
8531                                                 "cannot create default miss "
8532                                                 "action");
8533                         goto error;
8534                 }
8535                 sample_dv_actions[resource->sample_act.actions_num++] =
8536                                                 cache_resource->default_miss;
8537         }
8538         /* Create a DR sample action */
8539         sampler_attr.sample_ratio = cache_resource->ratio;
8540         sampler_attr.default_next_table = tbl->obj;
8541         sampler_attr.num_sample_actions = resource->sample_act.actions_num;
8542         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
8543                                                         &sample_dv_actions[0];
8544         sampler_attr.action = cache_resource->set_action;
8545         cache_resource->verbs_action =
8546                 mlx5_glue->dr_create_flow_action_sampler(&sampler_attr);
8547         if (!cache_resource->verbs_action) {
8548                 rte_flow_error_set(error, ENOMEM,
8549                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8550                                         NULL, "cannot create sample action");
8551                 goto error;
8552         }
8553         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8554         ILIST_INSERT(sh->ipool[MLX5_IPOOL_SAMPLE], &sh->sample_action_list,
8555                      dev_flow->handle->dvh.rix_sample, cache_resource,
8556                      next);
8557         dev_flow->dv.sample_res = cache_resource;
8558         DRV_LOG(DEBUG, "new sample resource %p: refcnt %d++",
8559                 (void *)cache_resource,
8560                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8561         return 0;
8562 error:
8563         if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
8564                 if (cache_resource->default_miss)
8565                         claim_zero(mlx5_glue->destroy_flow_action
8566                                 (cache_resource->default_miss));
8567         } else {
8568                 if (cache_resource->sample_idx.rix_hrxq &&
8569                     !mlx5_hrxq_release(dev,
8570                                 cache_resource->sample_idx.rix_hrxq))
8571                         cache_resource->sample_idx.rix_hrxq = 0;
8572                 if (cache_resource->sample_idx.rix_tag &&
8573                     !flow_dv_tag_release(dev,
8574                                 cache_resource->sample_idx.rix_tag))
8575                         cache_resource->sample_idx.rix_tag = 0;
8576                 if (cache_resource->sample_idx.cnt) {
8577                         flow_dv_counter_release(dev,
8578                                 cache_resource->sample_idx.cnt);
8579                         cache_resource->sample_idx.cnt = 0;
8580                 }
8581         }
8582         if (cache_resource->normal_path_tbl)
8583                 flow_dv_tbl_resource_release(dev,
8584                                 cache_resource->normal_path_tbl);
8585         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE],
8586                                 dev_flow->handle->dvh.rix_sample);
8587         dev_flow->handle->dvh.rix_sample = 0;
8588         return -rte_errno;
8589 }
8590
8591 /**
8592  * Find existing destination array resource or create and register a new one.
8593  *
8594  * @param[in, out] dev
8595  *   Pointer to rte_eth_dev structure.
8596  * @param[in] attr
8597  *   Attributes of flow that includes this item.
8598  * @param[in] resource
8599  *   Pointer to destination array resource.
8600  * @parm[in, out] dev_flow
8601  *   Pointer to the dev_flow.
8602  * @param[out] error
8603  *   pointer to error structure.
8604  *
8605  * @return
8606  *   0 on success otherwise -errno and errno is set.
8607  */
8608 static int
8609 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
8610                          const struct rte_flow_attr *attr,
8611                          struct mlx5_flow_dv_dest_array_resource *resource,
8612                          struct mlx5_flow *dev_flow,
8613                          struct rte_flow_error *error)
8614 {
8615         struct mlx5_flow_dv_dest_array_resource *cache_resource;
8616         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
8617         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
8618         struct mlx5_priv *priv = dev->data->dev_private;
8619         struct mlx5_dev_ctx_shared *sh = priv->sh;
8620         struct mlx5_flow_sub_actions_list *sample_act;
8621         struct mlx5dv_dr_domain *domain;
8622         uint32_t idx = 0;
8623
8624         /* Lookup a matching resource from cache. */
8625         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8626                       sh->dest_array_list,
8627                       idx, cache_resource, next) {
8628                 if (resource->num_of_dest == cache_resource->num_of_dest &&
8629                     resource->ft_type == cache_resource->ft_type &&
8630                     !memcmp((void *)cache_resource->sample_act,
8631                             (void *)resource->sample_act,
8632                            (resource->num_of_dest *
8633                            sizeof(struct mlx5_flow_sub_actions_list)))) {
8634                         DRV_LOG(DEBUG, "dest array resource %p: refcnt %d++",
8635                                 (void *)cache_resource,
8636                                 __atomic_load_n(&cache_resource->refcnt,
8637                                                 __ATOMIC_RELAXED));
8638                         __atomic_fetch_add(&cache_resource->refcnt, 1,
8639                                            __ATOMIC_RELAXED);
8640                         dev_flow->handle->dvh.rix_dest_array = idx;
8641                         dev_flow->dv.dest_array_res = cache_resource;
8642                         return 0;
8643                 }
8644         }
8645         /* Register new destination array resource. */
8646         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8647                                        &dev_flow->handle->dvh.rix_dest_array);
8648         if (!cache_resource)
8649                 return rte_flow_error_set(error, ENOMEM,
8650                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8651                                           NULL,
8652                                           "cannot allocate resource memory");
8653         *cache_resource = *resource;
8654         if (attr->transfer)
8655                 domain = sh->fdb_domain;
8656         else if (attr->ingress)
8657                 domain = sh->rx_domain;
8658         else
8659                 domain = sh->tx_domain;
8660         for (idx = 0; idx < resource->num_of_dest; idx++) {
8661                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
8662                                  mlx5_malloc(MLX5_MEM_ZERO,
8663                                  sizeof(struct mlx5dv_dr_action_dest_attr),
8664                                  0, SOCKET_ID_ANY);
8665                 if (!dest_attr[idx]) {
8666                         rte_flow_error_set(error, ENOMEM,
8667                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8668                                            NULL,
8669                                            "cannot allocate resource memory");
8670                         goto error;
8671                 }
8672                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
8673                 sample_act = &resource->sample_act[idx];
8674                 if (sample_act->action_flags == MLX5_FLOW_ACTION_QUEUE) {
8675                         dest_attr[idx]->dest = sample_act->dr_queue_action;
8676                 } else if (sample_act->action_flags ==
8677                           (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP)) {
8678                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
8679                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
8680                         dest_attr[idx]->dest_reformat->reformat =
8681                                         sample_act->dr_encap_action;
8682                         dest_attr[idx]->dest_reformat->dest =
8683                                         sample_act->dr_port_id_action;
8684                 } else if (sample_act->action_flags ==
8685                            MLX5_FLOW_ACTION_PORT_ID) {
8686                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
8687                 }
8688         }
8689         /* create a dest array actioin */
8690         cache_resource->action = mlx5_glue->dr_create_flow_action_dest_array
8691                                                 (domain,
8692                                                  cache_resource->num_of_dest,
8693                                                  dest_attr);
8694         if (!cache_resource->action) {
8695                 rte_flow_error_set(error, ENOMEM,
8696                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8697                                    NULL,
8698                                    "cannot create destination array action");
8699                 goto error;
8700         }
8701         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8702         ILIST_INSERT(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8703                      &sh->dest_array_list,
8704                      dev_flow->handle->dvh.rix_dest_array, cache_resource,
8705                      next);
8706         dev_flow->dv.dest_array_res = cache_resource;
8707         DRV_LOG(DEBUG, "new destination array resource %p: refcnt %d++",
8708                 (void *)cache_resource,
8709                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8710         for (idx = 0; idx < resource->num_of_dest; idx++)
8711                 mlx5_free(dest_attr[idx]);
8712         return 0;
8713 error:
8714         for (idx = 0; idx < resource->num_of_dest; idx++) {
8715                 struct mlx5_flow_sub_actions_idx *act_res =
8716                                         &cache_resource->sample_idx[idx];
8717                 if (act_res->rix_hrxq &&
8718                     !mlx5_hrxq_release(dev,
8719                                 act_res->rix_hrxq))
8720                         act_res->rix_hrxq = 0;
8721                 if (act_res->rix_encap_decap &&
8722                         !flow_dv_encap_decap_resource_release(dev,
8723                                 act_res->rix_encap_decap))
8724                         act_res->rix_encap_decap = 0;
8725                 if (act_res->rix_port_id_action &&
8726                         !flow_dv_port_id_action_resource_release(dev,
8727                                 act_res->rix_port_id_action))
8728                         act_res->rix_port_id_action = 0;
8729                 if (dest_attr[idx])
8730                         mlx5_free(dest_attr[idx]);
8731         }
8732
8733         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8734                                 dev_flow->handle->dvh.rix_dest_array);
8735         dev_flow->handle->dvh.rix_dest_array = 0;
8736         return -rte_errno;
8737 }
8738
8739 /**
8740  * Convert Sample action to DV specification.
8741  *
8742  * @param[in] dev
8743  *   Pointer to rte_eth_dev structure.
8744  * @param[in] action
8745  *   Pointer to action structure.
8746  * @param[in, out] dev_flow
8747  *   Pointer to the mlx5_flow.
8748  * @param[in] attr
8749  *   Pointer to the flow attributes.
8750  * @param[in, out] num_of_dest
8751  *   Pointer to the num of destination.
8752  * @param[in, out] sample_actions
8753  *   Pointer to sample actions list.
8754  * @param[in, out] res
8755  *   Pointer to sample resource.
8756  * @param[out] error
8757  *   Pointer to the error structure.
8758  *
8759  * @return
8760  *   0 on success, a negative errno value otherwise and rte_errno is set.
8761  */
8762 static int
8763 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
8764                                 const struct rte_flow_action *action,
8765                                 struct mlx5_flow *dev_flow,
8766                                 const struct rte_flow_attr *attr,
8767                                 uint32_t *num_of_dest,
8768                                 void **sample_actions,
8769                                 struct mlx5_flow_dv_sample_resource *res,
8770                                 struct rte_flow_error *error)
8771 {
8772         struct mlx5_priv *priv = dev->data->dev_private;
8773         const struct rte_flow_action_sample *sample_action;
8774         const struct rte_flow_action *sub_actions;
8775         const struct rte_flow_action_queue *queue;
8776         struct mlx5_flow_sub_actions_list *sample_act;
8777         struct mlx5_flow_sub_actions_idx *sample_idx;
8778         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
8779                                               priv->rss_desc)
8780                                               [!!priv->flow_nested_idx];
8781         uint64_t action_flags = 0;
8782
8783         sample_act = &res->sample_act;
8784         sample_idx = &res->sample_idx;
8785         sample_action = (const struct rte_flow_action_sample *)action->conf;
8786         res->ratio = sample_action->ratio;
8787         sub_actions = sample_action->actions;
8788         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
8789                 int type = sub_actions->type;
8790                 uint32_t pre_rix = 0;
8791                 void *pre_r;
8792                 switch (type) {
8793                 case RTE_FLOW_ACTION_TYPE_QUEUE:
8794                 {
8795                         struct mlx5_hrxq *hrxq;
8796                         uint32_t hrxq_idx;
8797
8798                         queue = sub_actions->conf;
8799                         rss_desc->queue_num = 1;
8800                         rss_desc->queue[0] = queue->index;
8801                         hrxq = flow_dv_handle_rx_queue(dev, dev_flow,
8802                                         rss_desc, &hrxq_idx);
8803                         if (!hrxq)
8804                                 return rte_flow_error_set
8805                                         (error, rte_errno,
8806                                          RTE_FLOW_ERROR_TYPE_ACTION,
8807                                          NULL,
8808                                          "cannot create fate queue");
8809                         sample_act->dr_queue_action = hrxq->action;
8810                         sample_idx->rix_hrxq = hrxq_idx;
8811                         sample_actions[sample_act->actions_num++] =
8812                                                 hrxq->action;
8813                         (*num_of_dest)++;
8814                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
8815                         if (action_flags & MLX5_FLOW_ACTION_MARK)
8816                                 dev_flow->handle->rix_hrxq = hrxq_idx;
8817                         dev_flow->handle->fate_action =
8818                                         MLX5_FLOW_FATE_QUEUE;
8819                         break;
8820                 }
8821                 case RTE_FLOW_ACTION_TYPE_MARK:
8822                 {
8823                         uint32_t tag_be = mlx5_flow_mark_set
8824                                 (((const struct rte_flow_action_mark *)
8825                                 (sub_actions->conf))->id);
8826
8827                         dev_flow->handle->mark = 1;
8828                         pre_rix = dev_flow->handle->dvh.rix_tag;
8829                         /* Save the mark resource before sample */
8830                         pre_r = dev_flow->dv.tag_resource;
8831                         if (flow_dv_tag_resource_register(dev, tag_be,
8832                                                   dev_flow, error))
8833                                 return -rte_errno;
8834                         MLX5_ASSERT(dev_flow->dv.tag_resource);
8835                         sample_act->dr_tag_action =
8836                                 dev_flow->dv.tag_resource->action;
8837                         sample_idx->rix_tag =
8838                                 dev_flow->handle->dvh.rix_tag;
8839                         sample_actions[sample_act->actions_num++] =
8840                                                 sample_act->dr_tag_action;
8841                         /* Recover the mark resource after sample */
8842                         dev_flow->dv.tag_resource = pre_r;
8843                         dev_flow->handle->dvh.rix_tag = pre_rix;
8844                         action_flags |= MLX5_FLOW_ACTION_MARK;
8845                         break;
8846                 }
8847                 case RTE_FLOW_ACTION_TYPE_COUNT:
8848                 {
8849                         uint32_t counter;
8850
8851                         counter = flow_dv_translate_create_counter(dev,
8852                                         dev_flow, sub_actions->conf, 0);
8853                         if (!counter)
8854                                 return rte_flow_error_set
8855                                                 (error, rte_errno,
8856                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8857                                                  NULL,
8858                                                  "cannot create counter"
8859                                                  " object.");
8860                         sample_idx->cnt = counter;
8861                         sample_act->dr_cnt_action =
8862                                   (flow_dv_counter_get_by_idx(dev,
8863                                   counter, NULL))->action;
8864                         sample_actions[sample_act->actions_num++] =
8865                                                 sample_act->dr_cnt_action;
8866                         action_flags |= MLX5_FLOW_ACTION_COUNT;
8867                         break;
8868                 }
8869                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
8870                 {
8871                         struct mlx5_flow_dv_port_id_action_resource
8872                                         port_id_resource;
8873                         uint32_t port_id = 0;
8874
8875                         memset(&port_id_resource, 0, sizeof(port_id_resource));
8876                         /* Save the port id resource before sample */
8877                         pre_rix = dev_flow->handle->rix_port_id_action;
8878                         pre_r = dev_flow->dv.port_id_action;
8879                         if (flow_dv_translate_action_port_id(dev, sub_actions,
8880                                                              &port_id, error))
8881                                 return -rte_errno;
8882                         port_id_resource.port_id = port_id;
8883                         if (flow_dv_port_id_action_resource_register
8884                             (dev, &port_id_resource, dev_flow, error))
8885                                 return -rte_errno;
8886                         sample_act->dr_port_id_action =
8887                                 dev_flow->dv.port_id_action->action;
8888                         sample_idx->rix_port_id_action =
8889                                 dev_flow->handle->rix_port_id_action;
8890                         sample_actions[sample_act->actions_num++] =
8891                                                 sample_act->dr_port_id_action;
8892                         /* Recover the port id resource after sample */
8893                         dev_flow->dv.port_id_action = pre_r;
8894                         dev_flow->handle->rix_port_id_action = pre_rix;
8895                         (*num_of_dest)++;
8896                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
8897                         break;
8898                 }
8899                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
8900                         /* Save the encap resource before sample */
8901                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
8902                         pre_r = dev_flow->dv.encap_decap;
8903                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
8904                                                            dev_flow,
8905                                                            attr->transfer,
8906                                                            error))
8907                                 return -rte_errno;
8908                         sample_act->dr_encap_action =
8909                                 dev_flow->dv.encap_decap->action;
8910                         sample_idx->rix_encap_decap =
8911                                 dev_flow->handle->dvh.rix_encap_decap;
8912                         sample_actions[sample_act->actions_num++] =
8913                                                 sample_act->dr_encap_action;
8914                         /* Recover the encap resource after sample */
8915                         dev_flow->dv.encap_decap = pre_r;
8916                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
8917                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
8918                         break;
8919                 default:
8920                         return rte_flow_error_set(error, EINVAL,
8921                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8922                                 NULL,
8923                                 "Not support for sampler action");
8924                 }
8925         }
8926         sample_act->action_flags = action_flags;
8927         res->ft_id = dev_flow->dv.group;
8928         if (attr->transfer) {
8929                 union {
8930                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
8931                         uint64_t set_action;
8932                 } action_ctx = { .set_action = 0 };
8933
8934                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
8935                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
8936                          MLX5_MODIFICATION_TYPE_SET);
8937                 MLX5_SET(set_action_in, action_ctx.action_in, field,
8938                          MLX5_MODI_META_REG_C_0);
8939                 MLX5_SET(set_action_in, action_ctx.action_in, data,
8940                          priv->vport_meta_tag);
8941                 res->set_action = action_ctx.set_action;
8942         } else if (attr->ingress) {
8943                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
8944         }
8945         return 0;
8946 }
8947
8948 /**
8949  * Convert Sample action to DV specification.
8950  *
8951  * @param[in] dev
8952  *   Pointer to rte_eth_dev structure.
8953  * @param[in, out] dev_flow
8954  *   Pointer to the mlx5_flow.
8955  * @param[in] attr
8956  *   Pointer to the flow attributes.
8957  * @param[in] num_of_dest
8958  *   The num of destination.
8959  * @param[in, out] res
8960  *   Pointer to sample resource.
8961  * @param[in, out] mdest_res
8962  *   Pointer to destination array resource.
8963  * @param[in] sample_actions
8964  *   Pointer to sample path actions list.
8965  * @param[in] action_flags
8966  *   Holds the actions detected until now.
8967  * @param[out] error
8968  *   Pointer to the error structure.
8969  *
8970  * @return
8971  *   0 on success, a negative errno value otherwise and rte_errno is set.
8972  */
8973 static int
8974 flow_dv_create_action_sample(struct rte_eth_dev *dev,
8975                              struct mlx5_flow *dev_flow,
8976                              const struct rte_flow_attr *attr,
8977                              uint32_t num_of_dest,
8978                              struct mlx5_flow_dv_sample_resource *res,
8979                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
8980                              void **sample_actions,
8981                              uint64_t action_flags,
8982                              struct rte_flow_error *error)
8983 {
8984         struct mlx5_priv *priv = dev->data->dev_private;
8985         /* update normal path action resource into last index of array */
8986         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
8987         struct mlx5_flow_sub_actions_list *sample_act =
8988                                         &mdest_res->sample_act[dest_index];
8989         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
8990                                               priv->rss_desc)
8991                                               [!!priv->flow_nested_idx];
8992         uint32_t normal_idx = 0;
8993         struct mlx5_hrxq *hrxq;
8994         uint32_t hrxq_idx;
8995
8996         if (num_of_dest > 1) {
8997                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
8998                         /* Handle QP action for mirroring */
8999                         hrxq = flow_dv_handle_rx_queue(dev, dev_flow,
9000                                                        rss_desc, &hrxq_idx);
9001                         if (!hrxq)
9002                                 return rte_flow_error_set
9003                                      (error, rte_errno,
9004                                       RTE_FLOW_ERROR_TYPE_ACTION,
9005                                       NULL,
9006                                       "cannot create rx queue");
9007                         normal_idx++;
9008                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
9009                         sample_act->dr_queue_action = hrxq->action;
9010                         if (action_flags & MLX5_FLOW_ACTION_MARK)
9011                                 dev_flow->handle->rix_hrxq = hrxq_idx;
9012                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9013                 }
9014                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
9015                         normal_idx++;
9016                         mdest_res->sample_idx[dest_index].rix_encap_decap =
9017                                 dev_flow->handle->dvh.rix_encap_decap;
9018                         sample_act->dr_encap_action =
9019                                 dev_flow->dv.encap_decap->action;
9020                 }
9021                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
9022                         normal_idx++;
9023                         mdest_res->sample_idx[dest_index].rix_port_id_action =
9024                                 dev_flow->handle->rix_port_id_action;
9025                         sample_act->dr_port_id_action =
9026                                 dev_flow->dv.port_id_action->action;
9027                 }
9028                 sample_act->actions_num = normal_idx;
9029                 /* update sample action resource into first index of array */
9030                 mdest_res->ft_type = res->ft_type;
9031                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
9032                                 sizeof(struct mlx5_flow_sub_actions_idx));
9033                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
9034                                 sizeof(struct mlx5_flow_sub_actions_list));
9035                 mdest_res->num_of_dest = num_of_dest;
9036                 if (flow_dv_dest_array_resource_register(dev, attr, mdest_res,
9037                                                          dev_flow, error))
9038                         return rte_flow_error_set(error, EINVAL,
9039                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9040                                                   NULL, "can't create sample "
9041                                                   "action");
9042         } else {
9043                 if (flow_dv_sample_resource_register(dev, attr, res, dev_flow,
9044                                                      sample_actions, error))
9045                         return rte_flow_error_set(error, EINVAL,
9046                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9047                                                   NULL,
9048                                                   "can't create sample action");
9049         }
9050         return 0;
9051 }
9052
9053 /**
9054  * Fill the flow with DV spec, lock free
9055  * (mutex should be acquired by caller).
9056  *
9057  * @param[in] dev
9058  *   Pointer to rte_eth_dev structure.
9059  * @param[in, out] dev_flow
9060  *   Pointer to the sub flow.
9061  * @param[in] attr
9062  *   Pointer to the flow attributes.
9063  * @param[in] items
9064  *   Pointer to the list of items.
9065  * @param[in] actions
9066  *   Pointer to the list of actions.
9067  * @param[out] error
9068  *   Pointer to the error structure.
9069  *
9070  * @return
9071  *   0 on success, a negative errno value otherwise and rte_errno is set.
9072  */
9073 static int
9074 __flow_dv_translate(struct rte_eth_dev *dev,
9075                     struct mlx5_flow *dev_flow,
9076                     const struct rte_flow_attr *attr,
9077                     const struct rte_flow_item items[],
9078                     const struct rte_flow_action actions[],
9079                     struct rte_flow_error *error)
9080 {
9081         struct mlx5_priv *priv = dev->data->dev_private;
9082         struct mlx5_dev_config *dev_conf = &priv->config;
9083         struct rte_flow *flow = dev_flow->flow;
9084         struct mlx5_flow_handle *handle = dev_flow->handle;
9085         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
9086                                               priv->rss_desc)
9087                                               [!!priv->flow_nested_idx];
9088         uint64_t item_flags = 0;
9089         uint64_t last_item = 0;
9090         uint64_t action_flags = 0;
9091         uint64_t priority = attr->priority;
9092         struct mlx5_flow_dv_matcher matcher = {
9093                 .mask = {
9094                         .size = sizeof(matcher.mask.buf) -
9095                                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
9096                 },
9097         };
9098         int actions_n = 0;
9099         bool actions_end = false;
9100         union {
9101                 struct mlx5_flow_dv_modify_hdr_resource res;
9102                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
9103                             sizeof(struct mlx5_modification_cmd) *
9104                             (MLX5_MAX_MODIFY_NUM + 1)];
9105         } mhdr_dummy;
9106         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
9107         const struct rte_flow_action_count *count = NULL;
9108         const struct rte_flow_action_age *age = NULL;
9109         union flow_dv_attr flow_attr = { .attr = 0 };
9110         uint32_t tag_be;
9111         union mlx5_flow_tbl_key tbl_key;
9112         uint32_t modify_action_position = UINT32_MAX;
9113         void *match_mask = matcher.mask.buf;
9114         void *match_value = dev_flow->dv.value.buf;
9115         uint8_t next_protocol = 0xff;
9116         struct rte_vlan_hdr vlan = { 0 };
9117         struct mlx5_flow_dv_dest_array_resource mdest_res;
9118         struct mlx5_flow_dv_sample_resource sample_res;
9119         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
9120         struct mlx5_flow_sub_actions_list *sample_act;
9121         uint32_t sample_act_pos = UINT32_MAX;
9122         uint32_t num_of_dest = 0;
9123         int tmp_actions_n = 0;
9124         uint32_t table;
9125         int ret = 0;
9126
9127         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
9128         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
9129         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
9130                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
9131         /* update normal path action resource into last index of array */
9132         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
9133         ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
9134                                        !!priv->fdb_def_rule, &table, error);
9135         if (ret)
9136                 return ret;
9137         dev_flow->dv.group = table;
9138         if (attr->transfer)
9139                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
9140         if (priority == MLX5_FLOW_PRIO_RSVD)
9141                 priority = dev_conf->flow_prio - 1;
9142         /* number of actions must be set to 0 in case of dirty stack. */
9143         mhdr_res->actions_num = 0;
9144         for (; !actions_end ; actions++) {
9145                 const struct rte_flow_action_queue *queue;
9146                 const struct rte_flow_action_rss *rss;
9147                 const struct rte_flow_action *action = actions;
9148                 const uint8_t *rss_key;
9149                 const struct rte_flow_action_meter *mtr;
9150                 struct mlx5_flow_tbl_resource *tbl;
9151                 uint32_t port_id = 0;
9152                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
9153                 int action_type = actions->type;
9154                 const struct rte_flow_action *found_action = NULL;
9155                 struct mlx5_flow_meter *fm = NULL;
9156                 uint32_t jump_group = 0;
9157
9158                 if (!mlx5_flow_os_action_supported(action_type))
9159                         return rte_flow_error_set(error, ENOTSUP,
9160                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9161                                                   actions,
9162                                                   "action not supported");
9163                 switch (action_type) {
9164                 case RTE_FLOW_ACTION_TYPE_VOID:
9165                         break;
9166                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
9167                         if (flow_dv_translate_action_port_id(dev, action,
9168                                                              &port_id, error))
9169                                 return -rte_errno;
9170                         port_id_resource.port_id = port_id;
9171                         MLX5_ASSERT(!handle->rix_port_id_action);
9172                         if (flow_dv_port_id_action_resource_register
9173                             (dev, &port_id_resource, dev_flow, error))
9174                                 return -rte_errno;
9175                         dev_flow->dv.actions[actions_n++] =
9176                                         dev_flow->dv.port_id_action->action;
9177                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9178                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
9179                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9180                         num_of_dest++;
9181                         break;
9182                 case RTE_FLOW_ACTION_TYPE_FLAG:
9183                         action_flags |= MLX5_FLOW_ACTION_FLAG;
9184                         dev_flow->handle->mark = 1;
9185                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
9186                                 struct rte_flow_action_mark mark = {
9187                                         .id = MLX5_FLOW_MARK_DEFAULT,
9188                                 };
9189
9190                                 if (flow_dv_convert_action_mark(dev, &mark,
9191                                                                 mhdr_res,
9192                                                                 error))
9193                                         return -rte_errno;
9194                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
9195                                 break;
9196                         }
9197                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
9198                         /*
9199                          * Only one FLAG or MARK is supported per device flow
9200                          * right now. So the pointer to the tag resource must be
9201                          * zero before the register process.
9202                          */
9203                         MLX5_ASSERT(!handle->dvh.rix_tag);
9204                         if (flow_dv_tag_resource_register(dev, tag_be,
9205                                                           dev_flow, error))
9206                                 return -rte_errno;
9207                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9208                         dev_flow->dv.actions[actions_n++] =
9209                                         dev_flow->dv.tag_resource->action;
9210                         break;
9211                 case RTE_FLOW_ACTION_TYPE_MARK:
9212                         action_flags |= MLX5_FLOW_ACTION_MARK;
9213                         dev_flow->handle->mark = 1;
9214                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
9215                                 const struct rte_flow_action_mark *mark =
9216                                         (const struct rte_flow_action_mark *)
9217                                                 actions->conf;
9218
9219                                 if (flow_dv_convert_action_mark(dev, mark,
9220                                                                 mhdr_res,
9221                                                                 error))
9222                                         return -rte_errno;
9223                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
9224                                 break;
9225                         }
9226                         /* Fall-through */
9227                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
9228                         /* Legacy (non-extensive) MARK action. */
9229                         tag_be = mlx5_flow_mark_set
9230                               (((const struct rte_flow_action_mark *)
9231                                (actions->conf))->id);
9232                         MLX5_ASSERT(!handle->dvh.rix_tag);
9233                         if (flow_dv_tag_resource_register(dev, tag_be,
9234                                                           dev_flow, error))
9235                                 return -rte_errno;
9236                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9237                         dev_flow->dv.actions[actions_n++] =
9238                                         dev_flow->dv.tag_resource->action;
9239                         break;
9240                 case RTE_FLOW_ACTION_TYPE_SET_META:
9241                         if (flow_dv_convert_action_set_meta
9242                                 (dev, mhdr_res, attr,
9243                                  (const struct rte_flow_action_set_meta *)
9244                                   actions->conf, error))
9245                                 return -rte_errno;
9246                         action_flags |= MLX5_FLOW_ACTION_SET_META;
9247                         break;
9248                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
9249                         if (flow_dv_convert_action_set_tag
9250                                 (dev, mhdr_res,
9251                                  (const struct rte_flow_action_set_tag *)
9252                                   actions->conf, error))
9253                                 return -rte_errno;
9254                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9255                         break;
9256                 case RTE_FLOW_ACTION_TYPE_DROP:
9257                         action_flags |= MLX5_FLOW_ACTION_DROP;
9258                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
9259                         break;
9260                 case RTE_FLOW_ACTION_TYPE_QUEUE:
9261                         queue = actions->conf;
9262                         rss_desc->queue_num = 1;
9263                         rss_desc->queue[0] = queue->index;
9264                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
9265                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9266                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
9267                         num_of_dest++;
9268                         break;
9269                 case RTE_FLOW_ACTION_TYPE_RSS:
9270                         rss = actions->conf;
9271                         memcpy(rss_desc->queue, rss->queue,
9272                                rss->queue_num * sizeof(uint16_t));
9273                         rss_desc->queue_num = rss->queue_num;
9274                         /* NULL RSS key indicates default RSS key. */
9275                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
9276                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
9277                         /*
9278                          * rss->level and rss.types should be set in advance
9279                          * when expanding items for RSS.
9280                          */
9281                         action_flags |= MLX5_FLOW_ACTION_RSS;
9282                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9283                         break;
9284                 case RTE_FLOW_ACTION_TYPE_AGE:
9285                 case RTE_FLOW_ACTION_TYPE_COUNT:
9286                         if (!dev_conf->devx) {
9287                                 return rte_flow_error_set
9288                                               (error, ENOTSUP,
9289                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9290                                                NULL,
9291                                                "count action not supported");
9292                         }
9293                         /* Save information first, will apply later. */
9294                         if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT)
9295                                 count = action->conf;
9296                         else
9297                                 age = action->conf;
9298                         action_flags |= MLX5_FLOW_ACTION_COUNT;
9299                         break;
9300                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
9301                         dev_flow->dv.actions[actions_n++] =
9302                                                 priv->sh->pop_vlan_action;
9303                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
9304                         break;
9305                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
9306                         if (!(action_flags &
9307                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
9308                                 flow_dev_get_vlan_info_from_items(items, &vlan);
9309                         vlan.eth_proto = rte_be_to_cpu_16
9310                              ((((const struct rte_flow_action_of_push_vlan *)
9311                                                    actions->conf)->ethertype));
9312                         found_action = mlx5_flow_find_action
9313                                         (actions + 1,
9314                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
9315                         if (found_action)
9316                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
9317                         found_action = mlx5_flow_find_action
9318                                         (actions + 1,
9319                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
9320                         if (found_action)
9321                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
9322                         if (flow_dv_create_action_push_vlan
9323                                             (dev, attr, &vlan, dev_flow, error))
9324                                 return -rte_errno;
9325                         dev_flow->dv.actions[actions_n++] =
9326                                         dev_flow->dv.push_vlan_res->action;
9327                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
9328                         break;
9329                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
9330                         /* of_vlan_push action handled this action */
9331                         MLX5_ASSERT(action_flags &
9332                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
9333                         break;
9334                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
9335                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
9336                                 break;
9337                         flow_dev_get_vlan_info_from_items(items, &vlan);
9338                         mlx5_update_vlan_vid_pcp(actions, &vlan);
9339                         /* If no VLAN push - this is a modify header action */
9340                         if (flow_dv_convert_action_modify_vlan_vid
9341                                                 (mhdr_res, actions, error))
9342                                 return -rte_errno;
9343                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
9344                         break;
9345                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
9346                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
9347                         if (flow_dv_create_action_l2_encap(dev, actions,
9348                                                            dev_flow,
9349                                                            attr->transfer,
9350                                                            error))
9351                                 return -rte_errno;
9352                         dev_flow->dv.actions[actions_n++] =
9353                                         dev_flow->dv.encap_decap->action;
9354                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9355                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
9356                                 sample_act->action_flags |=
9357                                                         MLX5_FLOW_ACTION_ENCAP;
9358                         break;
9359                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
9360                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
9361                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
9362                                                            attr->transfer,
9363                                                            error))
9364                                 return -rte_errno;
9365                         dev_flow->dv.actions[actions_n++] =
9366                                         dev_flow->dv.encap_decap->action;
9367                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9368                         break;
9369                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
9370                         /* Handle encap with preceding decap. */
9371                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
9372                                 if (flow_dv_create_action_raw_encap
9373                                         (dev, actions, dev_flow, attr, error))
9374                                         return -rte_errno;
9375                                 dev_flow->dv.actions[actions_n++] =
9376                                         dev_flow->dv.encap_decap->action;
9377                         } else {
9378                                 /* Handle encap without preceding decap. */
9379                                 if (flow_dv_create_action_l2_encap
9380                                     (dev, actions, dev_flow, attr->transfer,
9381                                      error))
9382                                         return -rte_errno;
9383                                 dev_flow->dv.actions[actions_n++] =
9384                                         dev_flow->dv.encap_decap->action;
9385                         }
9386                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9387                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
9388                                 sample_act->action_flags |=
9389                                                         MLX5_FLOW_ACTION_ENCAP;
9390                         break;
9391                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
9392                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
9393                                 ;
9394                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
9395                                 if (flow_dv_create_action_l2_decap
9396                                     (dev, dev_flow, attr->transfer, error))
9397                                         return -rte_errno;
9398                                 dev_flow->dv.actions[actions_n++] =
9399                                         dev_flow->dv.encap_decap->action;
9400                         }
9401                         /* If decap is followed by encap, handle it at encap. */
9402                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9403                         break;
9404                 case RTE_FLOW_ACTION_TYPE_JUMP:
9405                         jump_group = ((const struct rte_flow_action_jump *)
9406                                                         action->conf)->group;
9407                         if (dev_flow->external && jump_group <
9408                                         MLX5_MAX_TABLES_EXTERNAL)
9409                                 jump_group *= MLX5_FLOW_TABLE_FACTOR;
9410                         ret = mlx5_flow_group_to_table(attr, dev_flow->external,
9411                                                        jump_group,
9412                                                        !!priv->fdb_def_rule,
9413                                                        &table, error);
9414                         if (ret)
9415                                 return ret;
9416                         tbl = flow_dv_tbl_resource_get(dev, table,
9417                                                        attr->egress,
9418                                                        attr->transfer, error);
9419                         if (!tbl)
9420                                 return rte_flow_error_set
9421                                                 (error, errno,
9422                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9423                                                  NULL,
9424                                                  "cannot create jump action.");
9425                         if (flow_dv_jump_tbl_resource_register
9426                             (dev, tbl, dev_flow, error)) {
9427                                 flow_dv_tbl_resource_release(dev, tbl);
9428                                 return rte_flow_error_set
9429                                                 (error, errno,
9430                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9431                                                  NULL,
9432                                                  "cannot create jump action.");
9433                         }
9434                         dev_flow->dv.actions[actions_n++] =
9435                                         dev_flow->dv.jump->action;
9436                         action_flags |= MLX5_FLOW_ACTION_JUMP;
9437                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
9438                         break;
9439                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
9440                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
9441                         if (flow_dv_convert_action_modify_mac
9442                                         (mhdr_res, actions, error))
9443                                 return -rte_errno;
9444                         action_flags |= actions->type ==
9445                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
9446                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
9447                                         MLX5_FLOW_ACTION_SET_MAC_DST;
9448                         break;
9449                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
9450                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
9451                         if (flow_dv_convert_action_modify_ipv4
9452                                         (mhdr_res, actions, error))
9453                                 return -rte_errno;
9454                         action_flags |= actions->type ==
9455                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
9456                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
9457                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
9458                         break;
9459                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
9460                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
9461                         if (flow_dv_convert_action_modify_ipv6
9462                                         (mhdr_res, actions, error))
9463                                 return -rte_errno;
9464                         action_flags |= actions->type ==
9465                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
9466                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
9467                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
9468                         break;
9469                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
9470                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
9471                         if (flow_dv_convert_action_modify_tp
9472                                         (mhdr_res, actions, items,
9473                                          &flow_attr, dev_flow, !!(action_flags &
9474                                          MLX5_FLOW_ACTION_DECAP), error))
9475                                 return -rte_errno;
9476                         action_flags |= actions->type ==
9477                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
9478                                         MLX5_FLOW_ACTION_SET_TP_SRC :
9479                                         MLX5_FLOW_ACTION_SET_TP_DST;
9480                         break;
9481                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
9482                         if (flow_dv_convert_action_modify_dec_ttl
9483                                         (mhdr_res, items, &flow_attr, dev_flow,
9484                                          !!(action_flags &
9485                                          MLX5_FLOW_ACTION_DECAP), error))
9486                                 return -rte_errno;
9487                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
9488                         break;
9489                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
9490                         if (flow_dv_convert_action_modify_ttl
9491                                         (mhdr_res, actions, items, &flow_attr,
9492                                          dev_flow, !!(action_flags &
9493                                          MLX5_FLOW_ACTION_DECAP), error))
9494                                 return -rte_errno;
9495                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
9496                         break;
9497                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
9498                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
9499                         if (flow_dv_convert_action_modify_tcp_seq
9500                                         (mhdr_res, actions, error))
9501                                 return -rte_errno;
9502                         action_flags |= actions->type ==
9503                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
9504                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
9505                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
9506                         break;
9507
9508                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
9509                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
9510                         if (flow_dv_convert_action_modify_tcp_ack
9511                                         (mhdr_res, actions, error))
9512                                 return -rte_errno;
9513                         action_flags |= actions->type ==
9514                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
9515                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
9516                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
9517                         break;
9518                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
9519                         if (flow_dv_convert_action_set_reg
9520                                         (mhdr_res, actions, error))
9521                                 return -rte_errno;
9522                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9523                         break;
9524                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
9525                         if (flow_dv_convert_action_copy_mreg
9526                                         (dev, mhdr_res, actions, error))
9527                                 return -rte_errno;
9528                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9529                         break;
9530                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
9531                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
9532                         dev_flow->handle->fate_action =
9533                                         MLX5_FLOW_FATE_DEFAULT_MISS;
9534                         break;
9535                 case RTE_FLOW_ACTION_TYPE_METER:
9536                         mtr = actions->conf;
9537                         if (!flow->meter) {
9538                                 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
9539                                                             attr, error);
9540                                 if (!fm)
9541                                         return rte_flow_error_set(error,
9542                                                 rte_errno,
9543                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9544                                                 NULL,
9545                                                 "meter not found "
9546                                                 "or invalid parameters");
9547                                 flow->meter = fm->idx;
9548                         }
9549                         /* Set the meter action. */
9550                         if (!fm) {
9551                                 fm = mlx5_ipool_get(priv->sh->ipool
9552                                                 [MLX5_IPOOL_MTR], flow->meter);
9553                                 if (!fm)
9554                                         return rte_flow_error_set(error,
9555                                                 rte_errno,
9556                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9557                                                 NULL,
9558                                                 "meter not found "
9559                                                 "or invalid parameters");
9560                         }
9561                         dev_flow->dv.actions[actions_n++] =
9562                                 fm->mfts->meter_action;
9563                         action_flags |= MLX5_FLOW_ACTION_METER;
9564                         break;
9565                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
9566                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
9567                                                               actions, error))
9568                                 return -rte_errno;
9569                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
9570                         break;
9571                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
9572                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
9573                                                               actions, error))
9574                                 return -rte_errno;
9575                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
9576                         break;
9577                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
9578                         sample_act_pos = actions_n;
9579                         ret = flow_dv_translate_action_sample(dev,
9580                                                               actions,
9581                                                               dev_flow, attr,
9582                                                               &num_of_dest,
9583                                                               sample_actions,
9584                                                               &sample_res,
9585                                                               error);
9586                         if (ret < 0)
9587                                 return ret;
9588                         actions_n++;
9589                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
9590                         /* put encap action into group if work with port id */
9591                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
9592                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
9593                                 sample_act->action_flags |=
9594                                                         MLX5_FLOW_ACTION_ENCAP;
9595                         break;
9596                 case RTE_FLOW_ACTION_TYPE_END:
9597                         actions_end = true;
9598                         if (mhdr_res->actions_num) {
9599                                 /* create modify action if needed. */
9600                                 if (flow_dv_modify_hdr_resource_register
9601                                         (dev, mhdr_res, dev_flow, error))
9602                                         return -rte_errno;
9603                                 dev_flow->dv.actions[modify_action_position] =
9604                                         handle->dvh.modify_hdr->action;
9605                         }
9606                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
9607                                 flow->counter =
9608                                         flow_dv_translate_create_counter(dev,
9609                                                 dev_flow, count, age);
9610
9611                                 if (!flow->counter)
9612                                         return rte_flow_error_set
9613                                                 (error, rte_errno,
9614                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9615                                                 NULL,
9616                                                 "cannot create counter"
9617                                                 " object.");
9618                                 dev_flow->dv.actions[actions_n] =
9619                                           (flow_dv_counter_get_by_idx(dev,
9620                                           flow->counter, NULL))->action;
9621                                 actions_n++;
9622                         }
9623                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
9624                                 ret = flow_dv_create_action_sample(dev,
9625                                                           dev_flow, attr,
9626                                                           num_of_dest,
9627                                                           &sample_res,
9628                                                           &mdest_res,
9629                                                           sample_actions,
9630                                                           action_flags,
9631                                                           error);
9632                                 if (ret < 0)
9633                                         return rte_flow_error_set
9634                                                 (error, rte_errno,
9635                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9636                                                 NULL,
9637                                                 "cannot create sample action");
9638                                 if (num_of_dest > 1) {
9639                                         dev_flow->dv.actions[sample_act_pos] =
9640                                         dev_flow->dv.dest_array_res->action;
9641                                 } else {
9642                                         dev_flow->dv.actions[sample_act_pos] =
9643                                         dev_flow->dv.sample_res->verbs_action;
9644                                 }
9645                         }
9646                         break;
9647                 default:
9648                         break;
9649                 }
9650                 if (mhdr_res->actions_num &&
9651                     modify_action_position == UINT32_MAX)
9652                         modify_action_position = actions_n++;
9653         }
9654         /*
9655          * For multiple destination (sample action with ratio=1), the encap
9656          * action and port id action will be combined into group action.
9657          * So need remove the original these actions in the flow and only
9658          * use the sample action instead of.
9659          */
9660         if (num_of_dest > 1 && sample_act->dr_port_id_action) {
9661                 int i;
9662                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
9663
9664                 for (i = 0; i < actions_n; i++) {
9665                         if ((sample_act->dr_encap_action &&
9666                                 sample_act->dr_encap_action ==
9667                                 dev_flow->dv.actions[i]) ||
9668                                 (sample_act->dr_port_id_action &&
9669                                 sample_act->dr_port_id_action ==
9670                                 dev_flow->dv.actions[i]))
9671                                 continue;
9672                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
9673                 }
9674                 memcpy((void *)dev_flow->dv.actions,
9675                                 (void *)temp_actions,
9676                                 tmp_actions_n * sizeof(void *));
9677                 actions_n = tmp_actions_n;
9678         }
9679         dev_flow->dv.actions_n = actions_n;
9680         dev_flow->act_flags = action_flags;
9681         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
9682                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
9683                 int item_type = items->type;
9684
9685                 if (!mlx5_flow_os_item_supported(item_type))
9686                         return rte_flow_error_set(error, ENOTSUP,
9687                                                   RTE_FLOW_ERROR_TYPE_ITEM,
9688                                                   NULL, "item not supported");
9689                 switch (item_type) {
9690                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
9691                         flow_dv_translate_item_port_id(dev, match_mask,
9692                                                        match_value, items);
9693                         last_item = MLX5_FLOW_ITEM_PORT_ID;
9694                         break;
9695                 case RTE_FLOW_ITEM_TYPE_ETH:
9696                         flow_dv_translate_item_eth(match_mask, match_value,
9697                                                    items, tunnel,
9698                                                    dev_flow->dv.group);
9699                         matcher.priority = action_flags &
9700                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
9701                                         !dev_flow->external ?
9702                                         MLX5_PRIORITY_MAP_L3 :
9703                                         MLX5_PRIORITY_MAP_L2;
9704                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
9705                                              MLX5_FLOW_LAYER_OUTER_L2;
9706                         break;
9707                 case RTE_FLOW_ITEM_TYPE_VLAN:
9708                         flow_dv_translate_item_vlan(dev_flow,
9709                                                     match_mask, match_value,
9710                                                     items, tunnel,
9711                                                     dev_flow->dv.group);
9712                         matcher.priority = MLX5_PRIORITY_MAP_L2;
9713                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
9714                                               MLX5_FLOW_LAYER_INNER_VLAN) :
9715                                              (MLX5_FLOW_LAYER_OUTER_L2 |
9716                                               MLX5_FLOW_LAYER_OUTER_VLAN);
9717                         break;
9718                 case RTE_FLOW_ITEM_TYPE_IPV4:
9719                         mlx5_flow_tunnel_ip_check(items, next_protocol,
9720                                                   &item_flags, &tunnel);
9721                         flow_dv_translate_item_ipv4(match_mask, match_value,
9722                                                     items, item_flags, tunnel,
9723                                                     dev_flow->dv.group);
9724                         matcher.priority = MLX5_PRIORITY_MAP_L3;
9725                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
9726                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
9727                         if (items->mask != NULL &&
9728                             ((const struct rte_flow_item_ipv4 *)
9729                              items->mask)->hdr.next_proto_id) {
9730                                 next_protocol =
9731                                         ((const struct rte_flow_item_ipv4 *)
9732                                          (items->spec))->hdr.next_proto_id;
9733                                 next_protocol &=
9734                                         ((const struct rte_flow_item_ipv4 *)
9735                                          (items->mask))->hdr.next_proto_id;
9736                         } else {
9737                                 /* Reset for inner layer. */
9738                                 next_protocol = 0xff;
9739                         }
9740                         break;
9741                 case RTE_FLOW_ITEM_TYPE_IPV6:
9742                         mlx5_flow_tunnel_ip_check(items, next_protocol,
9743                                                   &item_flags, &tunnel);
9744                         flow_dv_translate_item_ipv6(match_mask, match_value,
9745                                                     items, item_flags, tunnel,
9746                                                     dev_flow->dv.group);
9747                         matcher.priority = MLX5_PRIORITY_MAP_L3;
9748                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
9749                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
9750                         if (items->mask != NULL &&
9751                             ((const struct rte_flow_item_ipv6 *)
9752                              items->mask)->hdr.proto) {
9753                                 next_protocol =
9754                                         ((const struct rte_flow_item_ipv6 *)
9755                                          items->spec)->hdr.proto;
9756                                 next_protocol &=
9757                                         ((const struct rte_flow_item_ipv6 *)
9758                                          items->mask)->hdr.proto;
9759                         } else {
9760                                 /* Reset for inner layer. */
9761                                 next_protocol = 0xff;
9762                         }
9763                         break;
9764                 case RTE_FLOW_ITEM_TYPE_IPV6_FRAG_EXT:
9765                         flow_dv_translate_item_ipv6_frag_ext(match_mask,
9766                                                              match_value,
9767                                                              items, tunnel);
9768                         last_item = tunnel ?
9769                                         MLX5_FLOW_LAYER_INNER_L3_IPV6_FRAG_EXT :
9770                                         MLX5_FLOW_LAYER_OUTER_L3_IPV6_FRAG_EXT;
9771                         if (items->mask != NULL &&
9772                             ((const struct rte_flow_item_ipv6_frag_ext *)
9773                              items->mask)->hdr.next_header) {
9774                                 next_protocol =
9775                                 ((const struct rte_flow_item_ipv6_frag_ext *)
9776                                  items->spec)->hdr.next_header;
9777                                 next_protocol &=
9778                                 ((const struct rte_flow_item_ipv6_frag_ext *)
9779                                  items->mask)->hdr.next_header;
9780                         } else {
9781                                 /* Reset for inner layer. */
9782                                 next_protocol = 0xff;
9783                         }
9784                         break;
9785                 case RTE_FLOW_ITEM_TYPE_TCP:
9786                         flow_dv_translate_item_tcp(match_mask, match_value,
9787                                                    items, tunnel);
9788                         matcher.priority = MLX5_PRIORITY_MAP_L4;
9789                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
9790                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
9791                         break;
9792                 case RTE_FLOW_ITEM_TYPE_UDP:
9793                         flow_dv_translate_item_udp(match_mask, match_value,
9794                                                    items, tunnel);
9795                         matcher.priority = MLX5_PRIORITY_MAP_L4;
9796                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
9797                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
9798                         break;
9799                 case RTE_FLOW_ITEM_TYPE_GRE:
9800                         flow_dv_translate_item_gre(match_mask, match_value,
9801                                                    items, tunnel);
9802                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9803                         last_item = MLX5_FLOW_LAYER_GRE;
9804                         break;
9805                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
9806                         flow_dv_translate_item_gre_key(match_mask,
9807                                                        match_value, items);
9808                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
9809                         break;
9810                 case RTE_FLOW_ITEM_TYPE_NVGRE:
9811                         flow_dv_translate_item_nvgre(match_mask, match_value,
9812                                                      items, tunnel);
9813                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9814                         last_item = MLX5_FLOW_LAYER_GRE;
9815                         break;
9816                 case RTE_FLOW_ITEM_TYPE_VXLAN:
9817                         flow_dv_translate_item_vxlan(match_mask, match_value,
9818                                                      items, tunnel);
9819                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9820                         last_item = MLX5_FLOW_LAYER_VXLAN;
9821                         break;
9822                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
9823                         flow_dv_translate_item_vxlan_gpe(match_mask,
9824                                                          match_value, items,
9825                                                          tunnel);
9826                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9827                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
9828                         break;
9829                 case RTE_FLOW_ITEM_TYPE_GENEVE:
9830                         flow_dv_translate_item_geneve(match_mask, match_value,
9831                                                       items, tunnel);
9832                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9833                         last_item = MLX5_FLOW_LAYER_GENEVE;
9834                         break;
9835                 case RTE_FLOW_ITEM_TYPE_MPLS:
9836                         flow_dv_translate_item_mpls(match_mask, match_value,
9837                                                     items, last_item, tunnel);
9838                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9839                         last_item = MLX5_FLOW_LAYER_MPLS;
9840                         break;
9841                 case RTE_FLOW_ITEM_TYPE_MARK:
9842                         flow_dv_translate_item_mark(dev, match_mask,
9843                                                     match_value, items);
9844                         last_item = MLX5_FLOW_ITEM_MARK;
9845                         break;
9846                 case RTE_FLOW_ITEM_TYPE_META:
9847                         flow_dv_translate_item_meta(dev, match_mask,
9848                                                     match_value, attr, items);
9849                         last_item = MLX5_FLOW_ITEM_METADATA;
9850                         break;
9851                 case RTE_FLOW_ITEM_TYPE_ICMP:
9852                         flow_dv_translate_item_icmp(match_mask, match_value,
9853                                                     items, tunnel);
9854                         last_item = MLX5_FLOW_LAYER_ICMP;
9855                         break;
9856                 case RTE_FLOW_ITEM_TYPE_ICMP6:
9857                         flow_dv_translate_item_icmp6(match_mask, match_value,
9858                                                       items, tunnel);
9859                         last_item = MLX5_FLOW_LAYER_ICMP6;
9860                         break;
9861                 case RTE_FLOW_ITEM_TYPE_TAG:
9862                         flow_dv_translate_item_tag(dev, match_mask,
9863                                                    match_value, items);
9864                         last_item = MLX5_FLOW_ITEM_TAG;
9865                         break;
9866                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
9867                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
9868                                                         match_value, items);
9869                         last_item = MLX5_FLOW_ITEM_TAG;
9870                         break;
9871                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
9872                         flow_dv_translate_item_tx_queue(dev, match_mask,
9873                                                         match_value,
9874                                                         items);
9875                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
9876                         break;
9877                 case RTE_FLOW_ITEM_TYPE_GTP:
9878                         flow_dv_translate_item_gtp(match_mask, match_value,
9879                                                    items, tunnel);
9880                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9881                         last_item = MLX5_FLOW_LAYER_GTP;
9882                         break;
9883                 case RTE_FLOW_ITEM_TYPE_ECPRI:
9884                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
9885                                 /* Create it only the first time to be used. */
9886                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
9887                                 if (ret)
9888                                         return rte_flow_error_set
9889                                                 (error, -ret,
9890                                                 RTE_FLOW_ERROR_TYPE_ITEM,
9891                                                 NULL,
9892                                                 "cannot create eCPRI parser");
9893                         }
9894                         /* Adjust the length matcher and device flow value. */
9895                         matcher.mask.size = MLX5_ST_SZ_BYTES(fte_match_param);
9896                         dev_flow->dv.value.size =
9897                                         MLX5_ST_SZ_BYTES(fte_match_param);
9898                         flow_dv_translate_item_ecpri(dev, match_mask,
9899                                                      match_value, items);
9900                         /* No other protocol should follow eCPRI layer. */
9901                         last_item = MLX5_FLOW_LAYER_ECPRI;
9902                         break;
9903                 default:
9904                         break;
9905                 }
9906                 item_flags |= last_item;
9907         }
9908         /*
9909          * When E-Switch mode is enabled, we have two cases where we need to
9910          * set the source port manually.
9911          * The first one, is in case of Nic steering rule, and the second is
9912          * E-Switch rule where no port_id item was found. In both cases
9913          * the source port is set according the current port in use.
9914          */
9915         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
9916             (priv->representor || priv->master)) {
9917                 if (flow_dv_translate_item_port_id(dev, match_mask,
9918                                                    match_value, NULL))
9919                         return -rte_errno;
9920         }
9921 #ifdef RTE_LIBRTE_MLX5_DEBUG
9922         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
9923                                               dev_flow->dv.value.buf));
9924 #endif
9925         /*
9926          * Layers may be already initialized from prefix flow if this dev_flow
9927          * is the suffix flow.
9928          */
9929         handle->layers |= item_flags;
9930         if (action_flags & MLX5_FLOW_ACTION_RSS)
9931                 flow_dv_hashfields_set(dev_flow, rss_desc);
9932         /* Register matcher. */
9933         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
9934                                     matcher.mask.size);
9935         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
9936                                                      matcher.priority);
9937         /* reserved field no needs to be set to 0 here. */
9938         tbl_key.domain = attr->transfer;
9939         tbl_key.direction = attr->egress;
9940         tbl_key.table_id = dev_flow->dv.group;
9941         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
9942                 return -rte_errno;
9943         return 0;
9944 }
9945
9946 /**
9947  * Set hash RX queue by hash fields (see enum ibv_rx_hash_fields)
9948  * and tunnel.
9949  *
9950  * @param[in, out] action
9951  *   Shred RSS action holding hash RX queue objects.
9952  * @param[in] hash_fields
9953  *   Defines combination of packet fields to participate in RX hash.
9954  * @param[in] tunnel
9955  *   Tunnel type
9956  * @param[in] hrxq_idx
9957  *   Hash RX queue index to set.
9958  *
9959  * @return
9960  *   0 on success, otherwise negative errno value.
9961  */
9962 static int
9963 __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action,
9964                               const uint64_t hash_fields,
9965                               const int tunnel,
9966                               uint32_t hrxq_idx)
9967 {
9968         uint32_t *hrxqs = tunnel ? action->hrxq : action->hrxq_tunnel;
9969
9970         switch (hash_fields & ~IBV_RX_HASH_INNER) {
9971         case MLX5_RSS_HASH_IPV4:
9972                 hrxqs[0] = hrxq_idx;
9973                 return 0;
9974         case MLX5_RSS_HASH_IPV4_TCP:
9975                 hrxqs[1] = hrxq_idx;
9976                 return 0;
9977         case MLX5_RSS_HASH_IPV4_UDP:
9978                 hrxqs[2] = hrxq_idx;
9979                 return 0;
9980         case MLX5_RSS_HASH_IPV6:
9981                 hrxqs[3] = hrxq_idx;
9982                 return 0;
9983         case MLX5_RSS_HASH_IPV6_TCP:
9984                 hrxqs[4] = hrxq_idx;
9985                 return 0;
9986         case MLX5_RSS_HASH_IPV6_UDP:
9987                 hrxqs[5] = hrxq_idx;
9988                 return 0;
9989         case MLX5_RSS_HASH_NONE:
9990                 hrxqs[6] = hrxq_idx;
9991                 return 0;
9992         default:
9993                 return -1;
9994         }
9995 }
9996
9997 /**
9998  * Look up for hash RX queue by hash fields (see enum ibv_rx_hash_fields)
9999  * and tunnel.
10000  *
10001  * @param[in] action
10002  *   Shred RSS action holding hash RX queue objects.
10003  * @param[in] hash_fields
10004  *   Defines combination of packet fields to participate in RX hash.
10005  * @param[in] tunnel
10006  *   Tunnel type
10007  *
10008  * @return
10009  *   Valid hash RX queue index, otherwise 0.
10010  */
10011 static uint32_t
10012 __flow_dv_action_rss_hrxq_lookup(const struct mlx5_shared_action_rss *action,
10013                                  const uint64_t hash_fields,
10014                                  const int tunnel)
10015 {
10016         const uint32_t *hrxqs = tunnel ? action->hrxq : action->hrxq_tunnel;
10017
10018         switch (hash_fields & ~IBV_RX_HASH_INNER) {
10019         case MLX5_RSS_HASH_IPV4:
10020                 return hrxqs[0];
10021         case MLX5_RSS_HASH_IPV4_TCP:
10022                 return hrxqs[1];
10023         case MLX5_RSS_HASH_IPV4_UDP:
10024                 return hrxqs[2];
10025         case MLX5_RSS_HASH_IPV6:
10026                 return hrxqs[3];
10027         case MLX5_RSS_HASH_IPV6_TCP:
10028                 return hrxqs[4];
10029         case MLX5_RSS_HASH_IPV6_UDP:
10030                 return hrxqs[5];
10031         case MLX5_RSS_HASH_NONE:
10032                 return hrxqs[6];
10033         default:
10034                 return 0;
10035         }
10036 }
10037
10038 /**
10039  * Retrieves hash RX queue suitable for the *flow*.
10040  * If shared action configured for *flow* suitable hash RX queue will be
10041  * retrieved from attached shared action.
10042  *
10043  * @param[in] flow
10044  *   Shred RSS action holding hash RX queue objects.
10045  * @param[in] dev_flow
10046  *   Pointer to the sub flow.
10047  * @param[out] hrxq
10048  *   Pointer to retrieved hash RX queue object.
10049  *
10050  * @return
10051  *   Valid hash RX queue index, otherwise 0 and rte_errno is set.
10052  */
10053 static uint32_t
10054 __flow_dv_rss_get_hrxq(struct rte_eth_dev *dev, struct rte_flow *flow,
10055                            struct mlx5_flow *dev_flow,
10056                            struct mlx5_hrxq **hrxq)
10057 {
10058         struct mlx5_priv *priv = dev->data->dev_private;
10059         uint32_t hrxq_idx;
10060
10061         if (flow->shared_rss) {
10062                 hrxq_idx = __flow_dv_action_rss_hrxq_lookup
10063                                 (flow->shared_rss, dev_flow->hash_fields,
10064                                  !!(dev_flow->handle->layers &
10065                                     MLX5_FLOW_LAYER_TUNNEL));
10066                 if (hrxq_idx) {
10067                         *hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
10068                                                hrxq_idx);
10069                         rte_atomic32_inc(&(*hrxq)->refcnt);
10070                 }
10071         } else {
10072                 struct mlx5_flow_rss_desc *rss_desc =
10073                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
10074                                 [!!priv->flow_nested_idx];
10075
10076                 MLX5_ASSERT(rss_desc->queue_num);
10077                 hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
10078                                          MLX5_RSS_HASH_KEY_LEN,
10079                                          dev_flow->hash_fields,
10080                                          rss_desc->queue, rss_desc->queue_num);
10081                 if (!hrxq_idx) {
10082                         hrxq_idx = mlx5_hrxq_new(dev,
10083                                                  rss_desc->key,
10084                                                  MLX5_RSS_HASH_KEY_LEN,
10085                                                  dev_flow->hash_fields,
10086                                                  rss_desc->queue,
10087                                                  rss_desc->queue_num,
10088                                                  !!(dev_flow->handle->layers &
10089                                                  MLX5_FLOW_LAYER_TUNNEL),
10090                                                  false);
10091                 }
10092                 *hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
10093                                        hrxq_idx);
10094         }
10095         return hrxq_idx;
10096 }
10097
10098 /**
10099  * Apply the flow to the NIC, lock free,
10100  * (mutex should be acquired by caller).
10101  *
10102  * @param[in] dev
10103  *   Pointer to the Ethernet device structure.
10104  * @param[in, out] flow
10105  *   Pointer to flow structure.
10106  * @param[out] error
10107  *   Pointer to error structure.
10108  *
10109  * @return
10110  *   0 on success, a negative errno value otherwise and rte_errno is set.
10111  */
10112 static int
10113 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
10114                 struct rte_flow_error *error)
10115 {
10116         struct mlx5_flow_dv_workspace *dv;
10117         struct mlx5_flow_handle *dh;
10118         struct mlx5_flow_handle_dv *dv_h;
10119         struct mlx5_flow *dev_flow;
10120         struct mlx5_priv *priv = dev->data->dev_private;
10121         uint32_t handle_idx;
10122         int n;
10123         int err;
10124         int idx;
10125
10126         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
10127                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
10128                 dv = &dev_flow->dv;
10129                 dh = dev_flow->handle;
10130                 dv_h = &dh->dvh;
10131                 n = dv->actions_n;
10132                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
10133                         if (dv->transfer) {
10134                                 dv->actions[n++] = priv->sh->esw_drop_action;
10135                         } else {
10136                                 struct mlx5_hrxq *drop_hrxq;
10137                                 drop_hrxq = mlx5_drop_action_create(dev);
10138                                 if (!drop_hrxq) {
10139                                         rte_flow_error_set
10140                                                 (error, errno,
10141                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10142                                                  NULL,
10143                                                  "cannot get drop hash queue");
10144                                         goto error;
10145                                 }
10146                                 /*
10147                                  * Drop queues will be released by the specify
10148                                  * mlx5_drop_action_destroy() function. Assign
10149                                  * the special index to hrxq to mark the queue
10150                                  * has been allocated.
10151                                  */
10152                                 dh->rix_hrxq = UINT32_MAX;
10153                                 dv->actions[n++] = drop_hrxq->action;
10154                         }
10155                 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
10156                            !dv_h->rix_sample && !dv_h->rix_dest_array) {
10157                         struct mlx5_hrxq *hrxq = NULL;
10158                         uint32_t hrxq_idx = __flow_dv_rss_get_hrxq
10159                                                 (dev, flow, dev_flow, &hrxq);
10160
10161                         if (!hrxq) {
10162                                 rte_flow_error_set
10163                                         (error, rte_errno,
10164                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10165                                          "cannot get hash queue");
10166                                 goto error;
10167                         }
10168                         dh->rix_hrxq = hrxq_idx;
10169                         dv->actions[n++] = hrxq->action;
10170                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
10171                         if (flow_dv_default_miss_resource_register
10172                                         (dev, error)) {
10173                                 rte_flow_error_set
10174                                         (error, rte_errno,
10175                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10176                                          "cannot create default miss resource");
10177                                 goto error_default_miss;
10178                         }
10179                         dh->rix_default_fate =  MLX5_FLOW_FATE_DEFAULT_MISS;
10180                         dv->actions[n++] = priv->sh->default_miss.action;
10181                 }
10182                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
10183                                                (void *)&dv->value, n,
10184                                                dv->actions, &dh->drv_flow);
10185                 if (err) {
10186                         rte_flow_error_set(error, errno,
10187                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10188                                            NULL,
10189                                            "hardware refuses to create flow");
10190                         goto error;
10191                 }
10192                 if (priv->vmwa_context &&
10193                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
10194                         /*
10195                          * The rule contains the VLAN pattern.
10196                          * For VF we are going to create VLAN
10197                          * interface to make hypervisor set correct
10198                          * e-Switch vport context.
10199                          */
10200                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
10201                 }
10202         }
10203         return 0;
10204 error:
10205         if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
10206                 flow_dv_default_miss_resource_release(dev);
10207 error_default_miss:
10208         err = rte_errno; /* Save rte_errno before cleanup. */
10209         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
10210                        handle_idx, dh, next) {
10211                 /* hrxq is union, don't clear it if the flag is not set. */
10212                 if (dh->rix_hrxq) {
10213                         if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
10214                                 mlx5_drop_action_destroy(dev);
10215                                 dh->rix_hrxq = 0;
10216                         } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
10217                                 mlx5_hrxq_release(dev, dh->rix_hrxq);
10218                                 dh->rix_hrxq = 0;
10219                         }
10220                 }
10221                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
10222                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
10223         }
10224         rte_errno = err; /* Restore rte_errno. */
10225         return -rte_errno;
10226 }
10227
10228 /**
10229  * Release the flow matcher.
10230  *
10231  * @param dev
10232  *   Pointer to Ethernet device.
10233  * @param handle
10234  *   Pointer to mlx5_flow_handle.
10235  *
10236  * @return
10237  *   1 while a reference on it exists, 0 when freed.
10238  */
10239 static int
10240 flow_dv_matcher_release(struct rte_eth_dev *dev,
10241                         struct mlx5_flow_handle *handle)
10242 {
10243         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
10244
10245         MLX5_ASSERT(matcher->matcher_object);
10246         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
10247                 dev->data->port_id, (void *)matcher,
10248                 rte_atomic32_read(&matcher->refcnt));
10249         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
10250                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10251                            (matcher->matcher_object));
10252                 LIST_REMOVE(matcher, next);
10253                 /* table ref-- in release interface. */
10254                 flow_dv_tbl_resource_release(dev, matcher->tbl);
10255                 mlx5_free(matcher);
10256                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
10257                         dev->data->port_id, (void *)matcher);
10258                 return 0;
10259         }
10260         return 1;
10261 }
10262
10263 /**
10264  * Release an encap/decap resource.
10265  *
10266  * @param dev
10267  *   Pointer to Ethernet device.
10268  * @param encap_decap_idx
10269  *   Index of encap decap resource.
10270  *
10271  * @return
10272  *   1 while a reference on it exists, 0 when freed.
10273  */
10274 static int
10275 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
10276                                      uint32_t encap_decap_idx)
10277 {
10278         struct mlx5_priv *priv = dev->data->dev_private;
10279         uint32_t idx = encap_decap_idx;
10280         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
10281
10282         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
10283                          idx);
10284         if (!cache_resource)
10285                 return 0;
10286         MLX5_ASSERT(cache_resource->action);
10287         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
10288                 (void *)cache_resource,
10289                 rte_atomic32_read(&cache_resource->refcnt));
10290         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10291                 claim_zero(mlx5_flow_os_destroy_flow_action
10292                                                 (cache_resource->action));
10293                 mlx5_hlist_remove(priv->sh->encaps_decaps,
10294                                   &cache_resource->entry);
10295                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
10296                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
10297                         (void *)cache_resource);
10298                 return 0;
10299         }
10300         return 1;
10301 }
10302
10303 /**
10304  * Release an jump to table action resource.
10305  *
10306  * @param dev
10307  *   Pointer to Ethernet device.
10308  * @param handle
10309  *   Pointer to mlx5_flow_handle.
10310  *
10311  * @return
10312  *   1 while a reference on it exists, 0 when freed.
10313  */
10314 static int
10315 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
10316                                   struct mlx5_flow_handle *handle)
10317 {
10318         struct mlx5_priv *priv = dev->data->dev_private;
10319         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
10320         struct mlx5_flow_tbl_data_entry *tbl_data;
10321
10322         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
10323                              handle->rix_jump);
10324         if (!tbl_data)
10325                 return 0;
10326         cache_resource = &tbl_data->jump;
10327         MLX5_ASSERT(cache_resource->action);
10328         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
10329                 (void *)cache_resource,
10330                 rte_atomic32_read(&cache_resource->refcnt));
10331         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10332                 claim_zero(mlx5_flow_os_destroy_flow_action
10333                                                 (cache_resource->action));
10334                 /* jump action memory free is inside the table release. */
10335                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
10336                 DRV_LOG(DEBUG, "jump table resource %p: removed",
10337                         (void *)cache_resource);
10338                 return 0;
10339         }
10340         return 1;
10341 }
10342
10343 /**
10344  * Release a default miss resource.
10345  *
10346  * @param dev
10347  *   Pointer to Ethernet device.
10348  * @return
10349  *   1 while a reference on it exists, 0 when freed.
10350  */
10351 static int
10352 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
10353 {
10354         struct mlx5_priv *priv = dev->data->dev_private;
10355         struct mlx5_dev_ctx_shared *sh = priv->sh;
10356         struct mlx5_flow_default_miss_resource *cache_resource =
10357                         &sh->default_miss;
10358
10359         MLX5_ASSERT(cache_resource->action);
10360         DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--",
10361                         (void *)cache_resource->action,
10362                         rte_atomic32_read(&cache_resource->refcnt));
10363         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10364                 claim_zero(mlx5_glue->destroy_flow_action
10365                                 (cache_resource->action));
10366                 DRV_LOG(DEBUG, "default miss resource %p: removed",
10367                                 (void *)cache_resource->action);
10368                 return 0;
10369         }
10370         return 1;
10371 }
10372
10373 /**
10374  * Release a modify-header resource.
10375  *
10376  * @param dev
10377  *   Pointer to Ethernet device.
10378  * @param handle
10379  *   Pointer to mlx5_flow_handle.
10380  *
10381  * @return
10382  *   1 while a reference on it exists, 0 when freed.
10383  */
10384 static int
10385 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
10386                                     struct mlx5_flow_handle *handle)
10387 {
10388         struct mlx5_priv *priv = dev->data->dev_private;
10389         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
10390                                                         handle->dvh.modify_hdr;
10391
10392         MLX5_ASSERT(cache_resource->action);
10393         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
10394                 (void *)cache_resource,
10395                 rte_atomic32_read(&cache_resource->refcnt));
10396         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10397                 claim_zero(mlx5_flow_os_destroy_flow_action
10398                                                 (cache_resource->action));
10399                 mlx5_hlist_remove(priv->sh->modify_cmds,
10400                                   &cache_resource->entry);
10401                 mlx5_free(cache_resource);
10402                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
10403                         (void *)cache_resource);
10404                 return 0;
10405         }
10406         return 1;
10407 }
10408
10409 /**
10410  * Release port ID action resource.
10411  *
10412  * @param dev
10413  *   Pointer to Ethernet device.
10414  * @param handle
10415  *   Pointer to mlx5_flow_handle.
10416  *
10417  * @return
10418  *   1 while a reference on it exists, 0 when freed.
10419  */
10420 static int
10421 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
10422                                         uint32_t port_id)
10423 {
10424         struct mlx5_priv *priv = dev->data->dev_private;
10425         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
10426         uint32_t idx = port_id;
10427
10428         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
10429                                         idx);
10430         if (!cache_resource)
10431                 return 0;
10432         MLX5_ASSERT(cache_resource->action);
10433         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
10434                 (void *)cache_resource,
10435                 rte_atomic32_read(&cache_resource->refcnt));
10436         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10437                 claim_zero(mlx5_flow_os_destroy_flow_action
10438                                                 (cache_resource->action));
10439                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
10440                              &priv->sh->port_id_action_list, idx,
10441                              cache_resource, next);
10442                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx);
10443                 DRV_LOG(DEBUG, "port id action resource %p: removed",
10444                         (void *)cache_resource);
10445                 return 0;
10446         }
10447         return 1;
10448 }
10449
10450 /**
10451  * Release push vlan action resource.
10452  *
10453  * @param dev
10454  *   Pointer to Ethernet device.
10455  * @param handle
10456  *   Pointer to mlx5_flow_handle.
10457  *
10458  * @return
10459  *   1 while a reference on it exists, 0 when freed.
10460  */
10461 static int
10462 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
10463                                           struct mlx5_flow_handle *handle)
10464 {
10465         struct mlx5_priv *priv = dev->data->dev_private;
10466         uint32_t idx = handle->dvh.rix_push_vlan;
10467         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
10468
10469         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
10470                                         idx);
10471         if (!cache_resource)
10472                 return 0;
10473         MLX5_ASSERT(cache_resource->action);
10474         DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
10475                 (void *)cache_resource,
10476                 rte_atomic32_read(&cache_resource->refcnt));
10477         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10478                 claim_zero(mlx5_flow_os_destroy_flow_action
10479                                                 (cache_resource->action));
10480                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
10481                              &priv->sh->push_vlan_action_list, idx,
10482                              cache_resource, next);
10483                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
10484                 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
10485                         (void *)cache_resource);
10486                 return 0;
10487         }
10488         return 1;
10489 }
10490
10491 /**
10492  * Release the fate resource.
10493  *
10494  * @param dev
10495  *   Pointer to Ethernet device.
10496  * @param handle
10497  *   Pointer to mlx5_flow_handle.
10498  */
10499 static void
10500 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
10501                                struct mlx5_flow_handle *handle)
10502 {
10503         if (!handle->rix_fate)
10504                 return;
10505         switch (handle->fate_action) {
10506         case MLX5_FLOW_FATE_DROP:
10507                 mlx5_drop_action_destroy(dev);
10508                 break;
10509         case MLX5_FLOW_FATE_QUEUE:
10510                 mlx5_hrxq_release(dev, handle->rix_hrxq);
10511                 break;
10512         case MLX5_FLOW_FATE_JUMP:
10513                 flow_dv_jump_tbl_resource_release(dev, handle);
10514                 break;
10515         case MLX5_FLOW_FATE_PORT_ID:
10516                 flow_dv_port_id_action_resource_release(dev,
10517                                 handle->rix_port_id_action);
10518                 break;
10519         case MLX5_FLOW_FATE_DEFAULT_MISS:
10520                 flow_dv_default_miss_resource_release(dev);
10521                 break;
10522         default:
10523                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
10524                 break;
10525         }
10526         handle->rix_fate = 0;
10527 }
10528
10529 /**
10530  * Release an sample resource.
10531  *
10532  * @param dev
10533  *   Pointer to Ethernet device.
10534  * @param handle
10535  *   Pointer to mlx5_flow_handle.
10536  *
10537  * @return
10538  *   1 while a reference on it exists, 0 when freed.
10539  */
10540 static int
10541 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
10542                                      struct mlx5_flow_handle *handle)
10543 {
10544         struct mlx5_priv *priv = dev->data->dev_private;
10545         uint32_t idx = handle->dvh.rix_sample;
10546         struct mlx5_flow_dv_sample_resource *cache_resource;
10547
10548         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
10549                          idx);
10550         if (!cache_resource)
10551                 return 0;
10552         MLX5_ASSERT(cache_resource->verbs_action);
10553         DRV_LOG(DEBUG, "sample resource %p: refcnt %d--",
10554                 (void *)cache_resource,
10555                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10556         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10557                                __ATOMIC_RELAXED) == 0) {
10558                 if (cache_resource->verbs_action)
10559                         claim_zero(mlx5_glue->destroy_flow_action
10560                                         (cache_resource->verbs_action));
10561                 if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
10562                         if (cache_resource->default_miss)
10563                                 claim_zero(mlx5_glue->destroy_flow_action
10564                                   (cache_resource->default_miss));
10565                 }
10566                 if (cache_resource->normal_path_tbl)
10567                         flow_dv_tbl_resource_release(dev,
10568                                 cache_resource->normal_path_tbl);
10569         }
10570         if (cache_resource->sample_idx.rix_hrxq &&
10571                 !mlx5_hrxq_release(dev,
10572                         cache_resource->sample_idx.rix_hrxq))
10573                 cache_resource->sample_idx.rix_hrxq = 0;
10574         if (cache_resource->sample_idx.rix_tag &&
10575                 !flow_dv_tag_release(dev,
10576                         cache_resource->sample_idx.rix_tag))
10577                 cache_resource->sample_idx.rix_tag = 0;
10578         if (cache_resource->sample_idx.cnt) {
10579                 flow_dv_counter_release(dev,
10580                         cache_resource->sample_idx.cnt);
10581                 cache_resource->sample_idx.cnt = 0;
10582         }
10583         if (!__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)) {
10584                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
10585                              &priv->sh->sample_action_list, idx,
10586                              cache_resource, next);
10587                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], idx);
10588                 DRV_LOG(DEBUG, "sample resource %p: removed",
10589                         (void *)cache_resource);
10590                 return 0;
10591         }
10592         return 1;
10593 }
10594
10595 /**
10596  * Release an destination array resource.
10597  *
10598  * @param dev
10599  *   Pointer to Ethernet device.
10600  * @param handle
10601  *   Pointer to mlx5_flow_handle.
10602  *
10603  * @return
10604  *   1 while a reference on it exists, 0 when freed.
10605  */
10606 static int
10607 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
10608                                      struct mlx5_flow_handle *handle)
10609 {
10610         struct mlx5_priv *priv = dev->data->dev_private;
10611         struct mlx5_flow_dv_dest_array_resource *cache_resource;
10612         struct mlx5_flow_sub_actions_idx *mdest_act_res;
10613         uint32_t idx = handle->dvh.rix_dest_array;
10614         uint32_t i = 0;
10615
10616         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10617                          idx);
10618         if (!cache_resource)
10619                 return 0;
10620         MLX5_ASSERT(cache_resource->action);
10621         DRV_LOG(DEBUG, "destination array resource %p: refcnt %d--",
10622                 (void *)cache_resource,
10623                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10624         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10625                                __ATOMIC_RELAXED) == 0) {
10626                 if (cache_resource->action)
10627                         claim_zero(mlx5_glue->destroy_flow_action
10628                                                 (cache_resource->action));
10629                 for (; i < cache_resource->num_of_dest; i++) {
10630                         mdest_act_res = &cache_resource->sample_idx[i];
10631                         if (mdest_act_res->rix_hrxq) {
10632                                 mlx5_hrxq_release(dev,
10633                                         mdest_act_res->rix_hrxq);
10634                                 mdest_act_res->rix_hrxq = 0;
10635                         }
10636                         if (mdest_act_res->rix_encap_decap) {
10637                                 flow_dv_encap_decap_resource_release(dev,
10638                                         mdest_act_res->rix_encap_decap);
10639                                 mdest_act_res->rix_encap_decap = 0;
10640                         }
10641                         if (mdest_act_res->rix_port_id_action) {
10642                                 flow_dv_port_id_action_resource_release(dev,
10643                                         mdest_act_res->rix_port_id_action);
10644                                 mdest_act_res->rix_port_id_action = 0;
10645                         }
10646                         if (mdest_act_res->rix_tag) {
10647                                 flow_dv_tag_release(dev,
10648                                         mdest_act_res->rix_tag);
10649                                 mdest_act_res->rix_tag = 0;
10650                         }
10651                 }
10652                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10653                              &priv->sh->dest_array_list, idx,
10654                              cache_resource, next);
10655                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], idx);
10656                 DRV_LOG(DEBUG, "destination array resource %p: removed",
10657                         (void *)cache_resource);
10658                 return 0;
10659         }
10660         return 1;
10661 }
10662
10663 /**
10664  * Remove the flow from the NIC but keeps it in memory.
10665  * Lock free, (mutex should be acquired by caller).
10666  *
10667  * @param[in] dev
10668  *   Pointer to Ethernet device.
10669  * @param[in, out] flow
10670  *   Pointer to flow structure.
10671  */
10672 static void
10673 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
10674 {
10675         struct mlx5_flow_handle *dh;
10676         uint32_t handle_idx;
10677         struct mlx5_priv *priv = dev->data->dev_private;
10678
10679         if (!flow)
10680                 return;
10681         handle_idx = flow->dev_handles;
10682         while (handle_idx) {
10683                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10684                                     handle_idx);
10685                 if (!dh)
10686                         return;
10687                 if (dh->drv_flow) {
10688                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
10689                         dh->drv_flow = NULL;
10690                 }
10691                 if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
10692                     dh->fate_action == MLX5_FLOW_FATE_QUEUE ||
10693                     dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
10694                         flow_dv_fate_resource_release(dev, dh);
10695                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
10696                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
10697                 handle_idx = dh->next.next;
10698         }
10699 }
10700
10701 /**
10702  * Remove the flow from the NIC and the memory.
10703  * Lock free, (mutex should be acquired by caller).
10704  *
10705  * @param[in] dev
10706  *   Pointer to the Ethernet device structure.
10707  * @param[in, out] flow
10708  *   Pointer to flow structure.
10709  */
10710 static void
10711 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
10712 {
10713         struct rte_flow_shared_action *shared;
10714         struct mlx5_flow_handle *dev_handle;
10715         struct mlx5_priv *priv = dev->data->dev_private;
10716
10717         if (!flow)
10718                 return;
10719         __flow_dv_remove(dev, flow);
10720         shared = mlx5_flow_get_shared_rss(flow);
10721         if (shared)
10722                 __atomic_sub_fetch(&shared->refcnt, 1, __ATOMIC_RELAXED);
10723         if (flow->counter) {
10724                 flow_dv_counter_release(dev, flow->counter);
10725                 flow->counter = 0;
10726         }
10727         if (flow->meter) {
10728                 struct mlx5_flow_meter *fm;
10729
10730                 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
10731                                     flow->meter);
10732                 if (fm)
10733                         mlx5_flow_meter_detach(fm);
10734                 flow->meter = 0;
10735         }
10736         while (flow->dev_handles) {
10737                 uint32_t tmp_idx = flow->dev_handles;
10738
10739                 dev_handle = mlx5_ipool_get(priv->sh->ipool
10740                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
10741                 if (!dev_handle)
10742                         return;
10743                 flow->dev_handles = dev_handle->next.next;
10744                 if (dev_handle->dvh.matcher)
10745                         flow_dv_matcher_release(dev, dev_handle);
10746                 if (dev_handle->dvh.rix_sample)
10747                         flow_dv_sample_resource_release(dev, dev_handle);
10748                 if (dev_handle->dvh.rix_dest_array)
10749                         flow_dv_dest_array_resource_release(dev, dev_handle);
10750                 if (dev_handle->dvh.rix_encap_decap)
10751                         flow_dv_encap_decap_resource_release(dev,
10752                                 dev_handle->dvh.rix_encap_decap);
10753                 if (dev_handle->dvh.modify_hdr)
10754                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
10755                 if (dev_handle->dvh.rix_push_vlan)
10756                         flow_dv_push_vlan_action_resource_release(dev,
10757                                                                   dev_handle);
10758                 if (dev_handle->dvh.rix_tag)
10759                         flow_dv_tag_release(dev,
10760                                             dev_handle->dvh.rix_tag);
10761                 flow_dv_fate_resource_release(dev, dev_handle);
10762                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10763                            tmp_idx);
10764         }
10765 }
10766
10767 /**
10768  * Release array of hash RX queue objects.
10769  * Helper function.
10770  *
10771  * @param[in] dev
10772  *   Pointer to the Ethernet device structure.
10773  * @param[in, out] hrxqs
10774  *   Array of hash RX queue objects.
10775  *
10776  * @return
10777  *   Total number of references to hash RX queue objects in *hrxqs* array
10778  *   after this operation.
10779  */
10780 static int
10781 __flow_dv_hrxqs_release(struct rte_eth_dev *dev,
10782                         uint32_t (*hrxqs)[MLX5_RSS_HASH_FIELDS_LEN])
10783 {
10784         size_t i;
10785         int remaining = 0;
10786
10787         for (i = 0; i < RTE_DIM(*hrxqs); i++) {
10788                 int ret = mlx5_hrxq_release(dev, (*hrxqs)[i]);
10789
10790                 if (!ret)
10791                         (*hrxqs)[i] = 0;
10792                 remaining += ret;
10793         }
10794         return remaining;
10795 }
10796
10797 /**
10798  * Release all hash RX queue objects representing shared RSS action.
10799  *
10800  * @param[in] dev
10801  *   Pointer to the Ethernet device structure.
10802  * @param[in, out] action
10803  *   Shared RSS action to remove hash RX queue objects from.
10804  *
10805  * @return
10806  *   Total number of references to hash RX queue objects stored in *action*
10807  *   after this operation.
10808  *   Expected to be 0 if no external references held.
10809  */
10810 static int
10811 __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev,
10812                                  struct mlx5_shared_action_rss *action)
10813 {
10814         return __flow_dv_hrxqs_release(dev, &action->hrxq) +
10815                 __flow_dv_hrxqs_release(dev, &action->hrxq_tunnel);
10816 }
10817
10818 /**
10819  * Setup shared RSS action.
10820  * Prepare set of hash RX queue objects sufficient to handle all valid
10821  * hash_fields combinations (see enum ibv_rx_hash_fields).
10822  *
10823  * @param[in] dev
10824  *   Pointer to the Ethernet device structure.
10825  * @param[in, out] action
10826  *   Partially initialized shared RSS action.
10827  * @param[out] error
10828  *   Perform verbose error reporting if not NULL. Initialized in case of
10829  *   error only.
10830  *
10831  * @return
10832  *   0 on success, otherwise negative errno value.
10833  */
10834 static int
10835 __flow_dv_action_rss_setup(struct rte_eth_dev *dev,
10836                         struct mlx5_shared_action_rss *action,
10837                         struct rte_flow_error *error)
10838 {
10839         size_t i;
10840         int err;
10841
10842         for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) {
10843                 uint32_t hrxq_idx;
10844                 uint64_t hash_fields = mlx5_rss_hash_fields[i];
10845                 int tunnel;
10846
10847                 for (tunnel = 0; tunnel < 2; tunnel++) {
10848                         hrxq_idx = mlx5_hrxq_new(dev, action->origin.key,
10849                                         MLX5_RSS_HASH_KEY_LEN,
10850                                         hash_fields,
10851                                         action->origin.queue,
10852                                         action->origin.queue_num,
10853                                         tunnel, true);
10854                         if (!hrxq_idx) {
10855                                 rte_flow_error_set
10856                                         (error, rte_errno,
10857                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10858                                          "cannot get hash queue");
10859                                 goto error_hrxq_new;
10860                         }
10861                         err = __flow_dv_action_rss_hrxq_set
10862                                 (action, hash_fields, tunnel, hrxq_idx);
10863                         MLX5_ASSERT(!err);
10864                 }
10865         }
10866         return 0;
10867 error_hrxq_new:
10868         err = rte_errno;
10869         __flow_dv_action_rss_hrxqs_release(dev, action);
10870         rte_errno = err;
10871         return -rte_errno;
10872 }
10873
10874 /**
10875  * Create shared RSS action.
10876  *
10877  * @param[in] dev
10878  *   Pointer to the Ethernet device structure.
10879  * @param[in] conf
10880  *   Shared action configuration.
10881  * @param[in] rss
10882  *   RSS action specification used to create shared action.
10883  * @param[out] error
10884  *   Perform verbose error reporting if not NULL. Initialized in case of
10885  *   error only.
10886  *
10887  * @return
10888  *   A valid shared action handle in case of success, NULL otherwise and
10889  *   rte_errno is set.
10890  */
10891 static struct rte_flow_shared_action *
10892 __flow_dv_action_rss_create(struct rte_eth_dev *dev,
10893                             const struct rte_flow_shared_action_conf *conf,
10894                             const struct rte_flow_action_rss *rss,
10895                             struct rte_flow_error *error)
10896 {
10897         struct rte_flow_shared_action *shared_action = NULL;
10898         void *queue = NULL;
10899         struct mlx5_shared_action_rss *shared_rss;
10900         struct rte_flow_action_rss *origin;
10901         const uint8_t *rss_key;
10902         uint32_t queue_size = rss->queue_num * sizeof(uint16_t);
10903
10904         RTE_SET_USED(conf);
10905         queue = mlx5_malloc(0, RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
10906                             0, SOCKET_ID_ANY);
10907         shared_action = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*shared_action), 0,
10908                                     SOCKET_ID_ANY);
10909         if (!shared_action || !queue) {
10910                 rte_flow_error_set(error, ENOMEM,
10911                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10912                                    "cannot allocate resource memory");
10913                 goto error_rss_init;
10914         }
10915         shared_rss = &shared_action->rss;
10916         shared_rss->queue = queue;
10917         origin = &shared_rss->origin;
10918         origin->func = rss->func;
10919         origin->level = rss->level;
10920         /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
10921         origin->types = !rss->types ? ETH_RSS_IP : rss->types;
10922         /* NULL RSS key indicates default RSS key. */
10923         rss_key = !rss->key ? rss_hash_default_key : rss->key;
10924         memcpy(shared_rss->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
10925         origin->key = &shared_rss->key[0];
10926         origin->key_len = MLX5_RSS_HASH_KEY_LEN;
10927         memcpy(shared_rss->queue, rss->queue, queue_size);
10928         origin->queue = shared_rss->queue;
10929         origin->queue_num = rss->queue_num;
10930         if (__flow_dv_action_rss_setup(dev, shared_rss, error))
10931                 goto error_rss_init;
10932         shared_action->type = MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS;
10933         return shared_action;
10934 error_rss_init:
10935         mlx5_free(shared_action);
10936         mlx5_free(queue);
10937         return NULL;
10938 }
10939
10940 /**
10941  * Destroy the shared RSS action.
10942  * Release related hash RX queue objects.
10943  *
10944  * @param[in] dev
10945  *   Pointer to the Ethernet device structure.
10946  * @param[in] shared_rss
10947  *   The shared RSS action object to be removed.
10948  * @param[out] error
10949  *   Perform verbose error reporting if not NULL. Initialized in case of
10950  *   error only.
10951  *
10952  * @return
10953  *   0 on success, otherwise negative errno value.
10954  */
10955 static int
10956 __flow_dv_action_rss_release(struct rte_eth_dev *dev,
10957                          struct mlx5_shared_action_rss *shared_rss,
10958                          struct rte_flow_error *error)
10959 {
10960         struct rte_flow_shared_action *shared_action = NULL;
10961         uint32_t old_refcnt = 1;
10962         int remaining = __flow_dv_action_rss_hrxqs_release(dev, shared_rss);
10963
10964         if (remaining) {
10965                 return rte_flow_error_set(error, ETOOMANYREFS,
10966                                           RTE_FLOW_ERROR_TYPE_ACTION,
10967                                           NULL,
10968                                           "shared rss hrxq has references");
10969         }
10970         shared_action = container_of(shared_rss,
10971                                      struct rte_flow_shared_action, rss);
10972         if (!__atomic_compare_exchange_n(&shared_action->refcnt, &old_refcnt,
10973                                          0, 0,
10974                                          __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
10975                 return rte_flow_error_set(error, ETOOMANYREFS,
10976                                           RTE_FLOW_ERROR_TYPE_ACTION,
10977                                           NULL,
10978                                           "shared rss has references");
10979         }
10980         rte_free(shared_rss->queue);
10981         return 0;
10982 }
10983
10984 /**
10985  * Create shared action, lock free,
10986  * (mutex should be acquired by caller).
10987  * Dispatcher for action type specific call.
10988  *
10989  * @param[in] dev
10990  *   Pointer to the Ethernet device structure.
10991  * @param[in] conf
10992  *   Shared action configuration.
10993  * @param[in] action
10994  *   Action specification used to create shared action.
10995  * @param[out] error
10996  *   Perform verbose error reporting if not NULL. Initialized in case of
10997  *   error only.
10998  *
10999  * @return
11000  *   A valid shared action handle in case of success, NULL otherwise and
11001  *   rte_errno is set.
11002  */
11003 static struct rte_flow_shared_action *
11004 __flow_dv_action_create(struct rte_eth_dev *dev,
11005                         const struct rte_flow_shared_action_conf *conf,
11006                         const struct rte_flow_action *action,
11007                         struct rte_flow_error *error)
11008 {
11009         struct rte_flow_shared_action *shared_action = NULL;
11010         struct mlx5_priv *priv = dev->data->dev_private;
11011
11012         switch (action->type) {
11013         case RTE_FLOW_ACTION_TYPE_RSS:
11014                 shared_action = __flow_dv_action_rss_create(dev, conf,
11015                                                             action->conf,
11016                                                             error);
11017                 break;
11018         default:
11019                 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
11020                                    NULL, "action type not supported");
11021                 break;
11022         }
11023         if (shared_action) {
11024                 __atomic_add_fetch(&shared_action->refcnt, 1,
11025                                    __ATOMIC_RELAXED);
11026                 LIST_INSERT_HEAD(&priv->shared_actions, shared_action, next);
11027         }
11028         return shared_action;
11029 }
11030
11031 /**
11032  * Destroy the shared action.
11033  * Release action related resources on the NIC and the memory.
11034  * Lock free, (mutex should be acquired by caller).
11035  * Dispatcher for action type specific call.
11036  *
11037  * @param[in] dev
11038  *   Pointer to the Ethernet device structure.
11039  * @param[in] action
11040  *   The shared action object to be removed.
11041  * @param[out] error
11042  *   Perform verbose error reporting if not NULL. Initialized in case of
11043  *   error only.
11044  *
11045  * @return
11046  *   0 on success, otherwise negative errno value.
11047  */
11048 static int
11049 __flow_dv_action_destroy(struct rte_eth_dev *dev,
11050                          struct rte_flow_shared_action *action,
11051                          struct rte_flow_error *error)
11052 {
11053         int ret;
11054
11055         switch (action->type) {
11056         case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS:
11057                 ret = __flow_dv_action_rss_release(dev, &action->rss, error);
11058                 break;
11059         default:
11060                 return rte_flow_error_set(error, ENOTSUP,
11061                                           RTE_FLOW_ERROR_TYPE_ACTION,
11062                                           NULL,
11063                                           "action type not supported");
11064         }
11065         if (ret)
11066                 return ret;
11067         LIST_REMOVE(action, next);
11068         rte_free(action);
11069         return 0;
11070 }
11071
11072 /**
11073  * Updates in place shared RSS action configuration.
11074  *
11075  * @param[in] dev
11076  *   Pointer to the Ethernet device structure.
11077  * @param[in] shared_rss
11078  *   The shared RSS action object to be updated.
11079  * @param[in] action_conf
11080  *   RSS action specification used to modify *shared_rss*.
11081  * @param[out] error
11082  *   Perform verbose error reporting if not NULL. Initialized in case of
11083  *   error only.
11084  *
11085  * @return
11086  *   0 on success, otherwise negative errno value.
11087  * @note: currently only support update of RSS queues.
11088  */
11089 static int
11090 __flow_dv_action_rss_update(struct rte_eth_dev *dev,
11091                             struct mlx5_shared_action_rss *shared_rss,
11092                             const struct rte_flow_action_rss *action_conf,
11093                             struct rte_flow_error *error)
11094 {
11095         size_t i;
11096         int ret;
11097         void *queue = NULL;
11098         const uint8_t *rss_key;
11099         uint32_t rss_key_len;
11100         uint32_t queue_size = action_conf->queue_num * sizeof(uint16_t);
11101
11102         queue = mlx5_malloc(MLX5_MEM_ZERO,
11103                             RTE_ALIGN_CEIL(queue_size, sizeof(void *)),
11104                             0, SOCKET_ID_ANY);
11105         if (!queue)
11106                 return rte_flow_error_set(error, ENOMEM,
11107                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11108                                           NULL,
11109                                           "cannot allocate resource memory");
11110         if (action_conf->key) {
11111                 rss_key = action_conf->key;
11112                 rss_key_len = action_conf->key_len;
11113         } else {
11114                 rss_key = rss_hash_default_key;
11115                 rss_key_len = MLX5_RSS_HASH_KEY_LEN;
11116         }
11117         for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) {
11118                 uint32_t hrxq_idx;
11119                 uint64_t hash_fields = mlx5_rss_hash_fields[i];
11120                 int tunnel;
11121
11122                 for (tunnel = 0; tunnel < 2; tunnel++) {
11123                         hrxq_idx = __flow_dv_action_rss_hrxq_lookup
11124                                         (shared_rss, hash_fields, tunnel);
11125                         MLX5_ASSERT(hrxq_idx);
11126                         ret = mlx5_hrxq_modify
11127                                 (dev, hrxq_idx,
11128                                  rss_key, rss_key_len,
11129                                  hash_fields,
11130                                  action_conf->queue, action_conf->queue_num);
11131                         if (ret) {
11132                                 mlx5_free(queue);
11133                                 return rte_flow_error_set
11134                                         (error, rte_errno,
11135                                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
11136                                          "cannot update hash queue");
11137                         }
11138                 }
11139         }
11140         mlx5_free(shared_rss->queue);
11141         shared_rss->queue = queue;
11142         memcpy(shared_rss->queue, action_conf->queue, queue_size);
11143         shared_rss->origin.queue = shared_rss->queue;
11144         shared_rss->origin.queue_num = action_conf->queue_num;
11145         return 0;
11146 }
11147
11148 /**
11149  * Updates in place shared action configuration, lock free,
11150  * (mutex should be acquired by caller).
11151  *
11152  * @param[in] dev
11153  *   Pointer to the Ethernet device structure.
11154  * @param[in] action
11155  *   The shared action object to be updated.
11156  * @param[in] action_conf
11157  *   Action specification used to modify *action*.
11158  *   *action_conf* should be of type correlating with type of the *action*,
11159  *   otherwise considered as invalid.
11160  * @param[out] error
11161  *   Perform verbose error reporting if not NULL. Initialized in case of
11162  *   error only.
11163  *
11164  * @return
11165  *   0 on success, otherwise negative errno value.
11166  */
11167 static int
11168 __flow_dv_action_update(struct rte_eth_dev *dev,
11169                         struct rte_flow_shared_action *action,
11170                         const void *action_conf,
11171                         struct rte_flow_error *error)
11172 {
11173         switch (action->type) {
11174         case MLX5_RTE_FLOW_ACTION_TYPE_SHARED_RSS:
11175                 return __flow_dv_action_rss_update(dev, &action->rss,
11176                                                    action_conf, error);
11177         default:
11178                 return rte_flow_error_set(error, ENOTSUP,
11179                                           RTE_FLOW_ERROR_TYPE_ACTION,
11180                                           NULL,
11181                                           "action type not supported");
11182         }
11183 }
11184 /**
11185  * Query a dv flow  rule for its statistics via devx.
11186  *
11187  * @param[in] dev
11188  *   Pointer to Ethernet device.
11189  * @param[in] flow
11190  *   Pointer to the sub flow.
11191  * @param[out] data
11192  *   data retrieved by the query.
11193  * @param[out] error
11194  *   Perform verbose error reporting if not NULL.
11195  *
11196  * @return
11197  *   0 on success, a negative errno value otherwise and rte_errno is set.
11198  */
11199 static int
11200 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
11201                     void *data, struct rte_flow_error *error)
11202 {
11203         struct mlx5_priv *priv = dev->data->dev_private;
11204         struct rte_flow_query_count *qc = data;
11205
11206         if (!priv->config.devx)
11207                 return rte_flow_error_set(error, ENOTSUP,
11208                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11209                                           NULL,
11210                                           "counters are not supported");
11211         if (flow->counter) {
11212                 uint64_t pkts, bytes;
11213                 struct mlx5_flow_counter *cnt;
11214
11215                 cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
11216                                                  NULL);
11217                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
11218                                                &bytes);
11219
11220                 if (err)
11221                         return rte_flow_error_set(error, -err,
11222                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11223                                         NULL, "cannot read counters");
11224                 qc->hits_set = 1;
11225                 qc->bytes_set = 1;
11226                 qc->hits = pkts - cnt->hits;
11227                 qc->bytes = bytes - cnt->bytes;
11228                 if (qc->reset) {
11229                         cnt->hits = pkts;
11230                         cnt->bytes = bytes;
11231                 }
11232                 return 0;
11233         }
11234         return rte_flow_error_set(error, EINVAL,
11235                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11236                                   NULL,
11237                                   "counters are not available");
11238 }
11239
11240 /**
11241  * Query a flow rule AGE action for aging information.
11242  *
11243  * @param[in] dev
11244  *   Pointer to Ethernet device.
11245  * @param[in] flow
11246  *   Pointer to the sub flow.
11247  * @param[out] data
11248  *   data retrieved by the query.
11249  * @param[out] error
11250  *   Perform verbose error reporting if not NULL.
11251  *
11252  * @return
11253  *   0 on success, a negative errno value otherwise and rte_errno is set.
11254  */
11255 static int
11256 flow_dv_query_age(struct rte_eth_dev *dev, struct rte_flow *flow,
11257                   void *data, struct rte_flow_error *error)
11258 {
11259         struct rte_flow_query_age *resp = data;
11260
11261         if (flow->counter) {
11262                 struct mlx5_age_param *age_param =
11263                                 flow_dv_counter_idx_get_age(dev, flow->counter);
11264
11265                 if (!age_param || !age_param->timeout)
11266                         return rte_flow_error_set
11267                                         (error, EINVAL,
11268                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11269                                          NULL, "cannot read age data");
11270                 resp->aged = __atomic_load_n(&age_param->state,
11271                                              __ATOMIC_RELAXED) ==
11272                                                         AGE_TMOUT ? 1 : 0;
11273                 resp->sec_since_last_hit_valid = !resp->aged;
11274                 if (resp->sec_since_last_hit_valid)
11275                         resp->sec_since_last_hit =
11276                                 __atomic_load_n(&age_param->sec_since_last_hit,
11277                                                 __ATOMIC_RELAXED);
11278                 return 0;
11279         }
11280         return rte_flow_error_set(error, EINVAL,
11281                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11282                                   NULL,
11283                                   "age data not available");
11284 }
11285
11286 /**
11287  * Query a flow.
11288  *
11289  * @see rte_flow_query()
11290  * @see rte_flow_ops
11291  */
11292 static int
11293 flow_dv_query(struct rte_eth_dev *dev,
11294               struct rte_flow *flow __rte_unused,
11295               const struct rte_flow_action *actions __rte_unused,
11296               void *data __rte_unused,
11297               struct rte_flow_error *error __rte_unused)
11298 {
11299         int ret = -EINVAL;
11300
11301         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
11302                 switch (actions->type) {
11303                 case RTE_FLOW_ACTION_TYPE_VOID:
11304                         break;
11305                 case RTE_FLOW_ACTION_TYPE_COUNT:
11306                         ret = flow_dv_query_count(dev, flow, data, error);
11307                         break;
11308                 case RTE_FLOW_ACTION_TYPE_AGE:
11309                         ret = flow_dv_query_age(dev, flow, data, error);
11310                         break;
11311                 default:
11312                         return rte_flow_error_set(error, ENOTSUP,
11313                                                   RTE_FLOW_ERROR_TYPE_ACTION,
11314                                                   actions,
11315                                                   "action not supported");
11316                 }
11317         }
11318         return ret;
11319 }
11320
11321 /**
11322  * Destroy the meter table set.
11323  * Lock free, (mutex should be acquired by caller).
11324  *
11325  * @param[in] dev
11326  *   Pointer to Ethernet device.
11327  * @param[in] tbl
11328  *   Pointer to the meter table set.
11329  *
11330  * @return
11331  *   Always 0.
11332  */
11333 static int
11334 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
11335                         struct mlx5_meter_domains_infos *tbl)
11336 {
11337         struct mlx5_priv *priv = dev->data->dev_private;
11338         struct mlx5_meter_domains_infos *mtd =
11339                                 (struct mlx5_meter_domains_infos *)tbl;
11340
11341         if (!mtd || !priv->config.dv_flow_en)
11342                 return 0;
11343         if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
11344                 claim_zero(mlx5_flow_os_destroy_flow
11345                            (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
11346         if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
11347                 claim_zero(mlx5_flow_os_destroy_flow
11348                            (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
11349         if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
11350                 claim_zero(mlx5_flow_os_destroy_flow
11351                            (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
11352         if (mtd->egress.color_matcher)
11353                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11354                            (mtd->egress.color_matcher));
11355         if (mtd->egress.any_matcher)
11356                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11357                            (mtd->egress.any_matcher));
11358         if (mtd->egress.tbl)
11359                 flow_dv_tbl_resource_release(dev, mtd->egress.tbl);
11360         if (mtd->egress.sfx_tbl)
11361                 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl);
11362         if (mtd->ingress.color_matcher)
11363                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11364                            (mtd->ingress.color_matcher));
11365         if (mtd->ingress.any_matcher)
11366                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11367                            (mtd->ingress.any_matcher));
11368         if (mtd->ingress.tbl)
11369                 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl);
11370         if (mtd->ingress.sfx_tbl)
11371                 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl);
11372         if (mtd->transfer.color_matcher)
11373                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11374                            (mtd->transfer.color_matcher));
11375         if (mtd->transfer.any_matcher)
11376                 claim_zero(mlx5_flow_os_destroy_flow_matcher
11377                            (mtd->transfer.any_matcher));
11378         if (mtd->transfer.tbl)
11379                 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl);
11380         if (mtd->transfer.sfx_tbl)
11381                 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl);
11382         if (mtd->drop_actn)
11383                 claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn));
11384         mlx5_free(mtd);
11385         return 0;
11386 }
11387
11388 /* Number of meter flow actions, count and jump or count and drop. */
11389 #define METER_ACTIONS 2
11390
11391 /**
11392  * Create specify domain meter table and suffix table.
11393  *
11394  * @param[in] dev
11395  *   Pointer to Ethernet device.
11396  * @param[in,out] mtb
11397  *   Pointer to DV meter table set.
11398  * @param[in] egress
11399  *   Table attribute.
11400  * @param[in] transfer
11401  *   Table attribute.
11402  * @param[in] color_reg_c_idx
11403  *   Reg C index for color match.
11404  *
11405  * @return
11406  *   0 on success, -1 otherwise and rte_errno is set.
11407  */
11408 static int
11409 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
11410                            struct mlx5_meter_domains_infos *mtb,
11411                            uint8_t egress, uint8_t transfer,
11412                            uint32_t color_reg_c_idx)
11413 {
11414         struct mlx5_priv *priv = dev->data->dev_private;
11415         struct mlx5_dev_ctx_shared *sh = priv->sh;
11416         struct mlx5_flow_dv_match_params mask = {
11417                 .size = sizeof(mask.buf),
11418         };
11419         struct mlx5_flow_dv_match_params value = {
11420                 .size = sizeof(value.buf),
11421         };
11422         struct mlx5dv_flow_matcher_attr dv_attr = {
11423                 .type = IBV_FLOW_ATTR_NORMAL,
11424                 .priority = 0,
11425                 .match_criteria_enable = 0,
11426                 .match_mask = (void *)&mask,
11427         };
11428         void *actions[METER_ACTIONS];
11429         struct mlx5_meter_domain_info *dtb;
11430         struct rte_flow_error error;
11431         int i = 0;
11432         int ret;
11433
11434         if (transfer)
11435                 dtb = &mtb->transfer;
11436         else if (egress)
11437                 dtb = &mtb->egress;
11438         else
11439                 dtb = &mtb->ingress;
11440         /* Create the meter table with METER level. */
11441         dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
11442                                             egress, transfer, &error);
11443         if (!dtb->tbl) {
11444                 DRV_LOG(ERR, "Failed to create meter policer table.");
11445                 return -1;
11446         }
11447         /* Create the meter suffix table with SUFFIX level. */
11448         dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
11449                                             MLX5_FLOW_TABLE_LEVEL_SUFFIX,
11450                                             egress, transfer, &error);
11451         if (!dtb->sfx_tbl) {
11452                 DRV_LOG(ERR, "Failed to create meter suffix table.");
11453                 return -1;
11454         }
11455         /* Create matchers, Any and Color. */
11456         dv_attr.priority = 3;
11457         dv_attr.match_criteria_enable = 0;
11458         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
11459                                                &dtb->any_matcher);
11460         if (ret) {
11461                 DRV_LOG(ERR, "Failed to create meter"
11462                              " policer default matcher.");
11463                 goto error_exit;
11464         }
11465         dv_attr.priority = 0;
11466         dv_attr.match_criteria_enable =
11467                                 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
11468         flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
11469                                rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
11470         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
11471                                                &dtb->color_matcher);
11472         if (ret) {
11473                 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
11474                 goto error_exit;
11475         }
11476         if (mtb->count_actns[RTE_MTR_DROPPED])
11477                 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
11478         actions[i++] = mtb->drop_actn;
11479         /* Default rule: lowest priority, match any, actions: drop. */
11480         ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i,
11481                                        actions,
11482                                        &dtb->policer_rules[RTE_MTR_DROPPED]);
11483         if (ret) {
11484                 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
11485                 goto error_exit;
11486         }
11487         return 0;
11488 error_exit:
11489         return -1;
11490 }
11491
11492 /**
11493  * Create the needed meter and suffix tables.
11494  * Lock free, (mutex should be acquired by caller).
11495  *
11496  * @param[in] dev
11497  *   Pointer to Ethernet device.
11498  * @param[in] fm
11499  *   Pointer to the flow meter.
11500  *
11501  * @return
11502  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
11503  */
11504 static struct mlx5_meter_domains_infos *
11505 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
11506                        const struct mlx5_flow_meter *fm)
11507 {
11508         struct mlx5_priv *priv = dev->data->dev_private;
11509         struct mlx5_meter_domains_infos *mtb;
11510         int ret;
11511         int i;
11512
11513         if (!priv->mtr_en) {
11514                 rte_errno = ENOTSUP;
11515                 return NULL;
11516         }
11517         mtb = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*mtb), 0, SOCKET_ID_ANY);
11518         if (!mtb) {
11519                 DRV_LOG(ERR, "Failed to allocate memory for meter.");
11520                 return NULL;
11521         }
11522         /* Create meter count actions */
11523         for (i = 0; i <= RTE_MTR_DROPPED; i++) {
11524                 struct mlx5_flow_counter *cnt;
11525                 if (!fm->policer_stats.cnt[i])
11526                         continue;
11527                 cnt = flow_dv_counter_get_by_idx(dev,
11528                       fm->policer_stats.cnt[i], NULL);
11529                 mtb->count_actns[i] = cnt->action;
11530         }
11531         /* Create drop action. */
11532         ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn);
11533         if (ret) {
11534                 DRV_LOG(ERR, "Failed to create drop action.");
11535                 goto error_exit;
11536         }
11537         /* Egress meter table. */
11538         ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
11539         if (ret) {
11540                 DRV_LOG(ERR, "Failed to prepare egress meter table.");
11541                 goto error_exit;
11542         }
11543         /* Ingress meter table. */
11544         ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
11545         if (ret) {
11546                 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
11547                 goto error_exit;
11548         }
11549         /* FDB meter table. */
11550         if (priv->config.dv_esw_en) {
11551                 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
11552                                                  priv->mtr_color_reg);
11553                 if (ret) {
11554                         DRV_LOG(ERR, "Failed to prepare fdb meter table.");
11555                         goto error_exit;
11556                 }
11557         }
11558         return mtb;
11559 error_exit:
11560         flow_dv_destroy_mtr_tbl(dev, mtb);
11561         return NULL;
11562 }
11563
11564 /**
11565  * Destroy domain policer rule.
11566  *
11567  * @param[in] dt
11568  *   Pointer to domain table.
11569  */
11570 static void
11571 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
11572 {
11573         int i;
11574
11575         for (i = 0; i < RTE_MTR_DROPPED; i++) {
11576                 if (dt->policer_rules[i]) {
11577                         claim_zero(mlx5_flow_os_destroy_flow
11578                                    (dt->policer_rules[i]));
11579                         dt->policer_rules[i] = NULL;
11580                 }
11581         }
11582         if (dt->jump_actn) {
11583                 claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn));
11584                 dt->jump_actn = NULL;
11585         }
11586 }
11587
11588 /**
11589  * Destroy policer rules.
11590  *
11591  * @param[in] dev
11592  *   Pointer to Ethernet device.
11593  * @param[in] fm
11594  *   Pointer to flow meter structure.
11595  * @param[in] attr
11596  *   Pointer to flow attributes.
11597  *
11598  * @return
11599  *   Always 0.
11600  */
11601 static int
11602 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
11603                               const struct mlx5_flow_meter *fm,
11604                               const struct rte_flow_attr *attr)
11605 {
11606         struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
11607
11608         if (!mtb)
11609                 return 0;
11610         if (attr->egress)
11611                 flow_dv_destroy_domain_policer_rule(&mtb->egress);
11612         if (attr->ingress)
11613                 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
11614         if (attr->transfer)
11615                 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
11616         return 0;
11617 }
11618
11619 /**
11620  * Create specify domain meter policer rule.
11621  *
11622  * @param[in] fm
11623  *   Pointer to flow meter structure.
11624  * @param[in] mtb
11625  *   Pointer to DV meter table set.
11626  * @param[in] mtr_reg_c
11627  *   Color match REG_C.
11628  *
11629  * @return
11630  *   0 on success, -1 otherwise.
11631  */
11632 static int
11633 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
11634                                     struct mlx5_meter_domain_info *dtb,
11635                                     uint8_t mtr_reg_c)
11636 {
11637         struct mlx5_flow_dv_match_params matcher = {
11638                 .size = sizeof(matcher.buf),
11639         };
11640         struct mlx5_flow_dv_match_params value = {
11641                 .size = sizeof(value.buf),
11642         };
11643         struct mlx5_meter_domains_infos *mtb = fm->mfts;
11644         void *actions[METER_ACTIONS];
11645         int i;
11646         int ret = 0;
11647
11648         /* Create jump action. */
11649         if (!dtb->jump_actn)
11650                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
11651                                 (dtb->sfx_tbl->obj, &dtb->jump_actn);
11652         if (ret) {
11653                 DRV_LOG(ERR, "Failed to create policer jump action.");
11654                 goto error;
11655         }
11656         for (i = 0; i < RTE_MTR_DROPPED; i++) {
11657                 int j = 0;
11658
11659                 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
11660                                        rte_col_2_mlx5_col(i), UINT8_MAX);
11661                 if (mtb->count_actns[i])
11662                         actions[j++] = mtb->count_actns[i];
11663                 if (fm->action[i] == MTR_POLICER_ACTION_DROP)
11664                         actions[j++] = mtb->drop_actn;
11665                 else
11666                         actions[j++] = dtb->jump_actn;
11667                 ret = mlx5_flow_os_create_flow(dtb->color_matcher,
11668                                                (void *)&value, j, actions,
11669                                                &dtb->policer_rules[i]);
11670                 if (ret) {
11671                         DRV_LOG(ERR, "Failed to create policer rule.");
11672                         goto error;
11673                 }
11674         }
11675         return 0;
11676 error:
11677         rte_errno = errno;
11678         return -1;
11679 }
11680
11681 /**
11682  * Create policer rules.
11683  *
11684  * @param[in] dev
11685  *   Pointer to Ethernet device.
11686  * @param[in] fm
11687  *   Pointer to flow meter structure.
11688  * @param[in] attr
11689  *   Pointer to flow attributes.
11690  *
11691  * @return
11692  *   0 on success, -1 otherwise.
11693  */
11694 static int
11695 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
11696                              struct mlx5_flow_meter *fm,
11697                              const struct rte_flow_attr *attr)
11698 {
11699         struct mlx5_priv *priv = dev->data->dev_private;
11700         struct mlx5_meter_domains_infos *mtb = fm->mfts;
11701         int ret;
11702
11703         if (attr->egress) {
11704                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
11705                                                 priv->mtr_color_reg);
11706                 if (ret) {
11707                         DRV_LOG(ERR, "Failed to create egress policer.");
11708                         goto error;
11709                 }
11710         }
11711         if (attr->ingress) {
11712                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
11713                                                 priv->mtr_color_reg);
11714                 if (ret) {
11715                         DRV_LOG(ERR, "Failed to create ingress policer.");
11716                         goto error;
11717                 }
11718         }
11719         if (attr->transfer) {
11720                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
11721                                                 priv->mtr_color_reg);
11722                 if (ret) {
11723                         DRV_LOG(ERR, "Failed to create transfer policer.");
11724                         goto error;
11725                 }
11726         }
11727         return 0;
11728 error:
11729         flow_dv_destroy_policer_rules(dev, fm, attr);
11730         return -1;
11731 }
11732
11733 /**
11734  * Validate the batch counter support in root table.
11735  *
11736  * Create a simple flow with invalid counter and drop action on root table to
11737  * validate if batch counter with offset on root table is supported or not.
11738  *
11739  * @param[in] dev
11740  *   Pointer to rte_eth_dev structure.
11741  *
11742  * @return
11743  *   0 on success, a negative errno value otherwise and rte_errno is set.
11744  */
11745 int
11746 mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev)
11747 {
11748         struct mlx5_priv *priv = dev->data->dev_private;
11749         struct mlx5_dev_ctx_shared *sh = priv->sh;
11750         struct mlx5_flow_dv_match_params mask = {
11751                 .size = sizeof(mask.buf),
11752         };
11753         struct mlx5_flow_dv_match_params value = {
11754                 .size = sizeof(value.buf),
11755         };
11756         struct mlx5dv_flow_matcher_attr dv_attr = {
11757                 .type = IBV_FLOW_ATTR_NORMAL,
11758                 .priority = 0,
11759                 .match_criteria_enable = 0,
11760                 .match_mask = (void *)&mask,
11761         };
11762         void *actions[2] = { 0 };
11763         struct mlx5_flow_tbl_resource *tbl = NULL, *dest_tbl = NULL;
11764         struct mlx5_devx_obj *dcs = NULL;
11765         void *matcher = NULL;
11766         void *flow = NULL;
11767         int i, ret = -1;
11768
11769         tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, NULL);
11770         if (!tbl)
11771                 goto err;
11772         dest_tbl = flow_dv_tbl_resource_get(dev, 1, 0, 0, NULL);
11773         if (!dest_tbl)
11774                 goto err;
11775         dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
11776         if (!dcs)
11777                 goto err;
11778         ret = mlx5_flow_os_create_flow_action_count(dcs->obj, UINT16_MAX,
11779                                                     &actions[0]);
11780         if (ret)
11781                 goto err;
11782         ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
11783                                 (dest_tbl->obj, &actions[1]);
11784         if (ret)
11785                 goto err;
11786         dv_attr.match_criteria_enable = flow_dv_matcher_enable(mask.buf);
11787         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
11788                                                &matcher);
11789         if (ret)
11790                 goto err;
11791         ret = mlx5_flow_os_create_flow(matcher, (void *)&value, 2,
11792                                        actions, &flow);
11793 err:
11794         /*
11795          * If batch counter with offset is not supported, the driver will not
11796          * validate the invalid offset value, flow create should success.
11797          * In this case, it means batch counter is not supported in root table.
11798          *
11799          * Otherwise, if flow create is failed, counter offset is supported.
11800          */
11801         if (flow) {
11802                 DRV_LOG(INFO, "Batch counter is not supported in root "
11803                               "table. Switch to fallback mode.");
11804                 rte_errno = ENOTSUP;
11805                 ret = -rte_errno;
11806                 claim_zero(mlx5_flow_os_destroy_flow(flow));
11807         } else {
11808                 /* Check matcher to make sure validate fail at flow create. */
11809                 if (!matcher || (matcher && errno != EINVAL))
11810                         DRV_LOG(ERR, "Unexpected error in counter offset "
11811                                      "support detection");
11812                 ret = 0;
11813         }
11814         for (i = 0; i < 2; i++) {
11815                 if (actions[i])
11816                         claim_zero(mlx5_flow_os_destroy_flow_action
11817                                    (actions[i]));
11818         }
11819         if (matcher)
11820                 claim_zero(mlx5_flow_os_destroy_flow_matcher(matcher));
11821         if (tbl)
11822                 flow_dv_tbl_resource_release(dev, tbl);
11823         if (dest_tbl)
11824                 flow_dv_tbl_resource_release(dev, dest_tbl);
11825         if (dcs)
11826                 claim_zero(mlx5_devx_cmd_destroy(dcs));
11827         return ret;
11828 }
11829
11830 /**
11831  * Query a devx counter.
11832  *
11833  * @param[in] dev
11834  *   Pointer to the Ethernet device structure.
11835  * @param[in] cnt
11836  *   Index to the flow counter.
11837  * @param[in] clear
11838  *   Set to clear the counter statistics.
11839  * @param[out] pkts
11840  *   The statistics value of packets.
11841  * @param[out] bytes
11842  *   The statistics value of bytes.
11843  *
11844  * @return
11845  *   0 on success, otherwise return -1.
11846  */
11847 static int
11848 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
11849                       uint64_t *pkts, uint64_t *bytes)
11850 {
11851         struct mlx5_priv *priv = dev->data->dev_private;
11852         struct mlx5_flow_counter *cnt;
11853         uint64_t inn_pkts, inn_bytes;
11854         int ret;
11855
11856         if (!priv->config.devx)
11857                 return -1;
11858
11859         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
11860         if (ret)
11861                 return -1;
11862         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
11863         *pkts = inn_pkts - cnt->hits;
11864         *bytes = inn_bytes - cnt->bytes;
11865         if (clear) {
11866                 cnt->hits = inn_pkts;
11867                 cnt->bytes = inn_bytes;
11868         }
11869         return 0;
11870 }
11871
11872 /**
11873  * Get aged-out flows.
11874  *
11875  * @param[in] dev
11876  *   Pointer to the Ethernet device structure.
11877  * @param[in] context
11878  *   The address of an array of pointers to the aged-out flows contexts.
11879  * @param[in] nb_contexts
11880  *   The length of context array pointers.
11881  * @param[out] error
11882  *   Perform verbose error reporting if not NULL. Initialized in case of
11883  *   error only.
11884  *
11885  * @return
11886  *   how many contexts get in success, otherwise negative errno value.
11887  *   if nb_contexts is 0, return the amount of all aged contexts.
11888  *   if nb_contexts is not 0 , return the amount of aged flows reported
11889  *   in the context array.
11890  * @note: only stub for now
11891  */
11892 static int
11893 flow_get_aged_flows(struct rte_eth_dev *dev,
11894                     void **context,
11895                     uint32_t nb_contexts,
11896                     struct rte_flow_error *error)
11897 {
11898         struct mlx5_priv *priv = dev->data->dev_private;
11899         struct mlx5_age_info *age_info;
11900         struct mlx5_age_param *age_param;
11901         struct mlx5_flow_counter *counter;
11902         int nb_flows = 0;
11903
11904         if (nb_contexts && !context)
11905                 return rte_flow_error_set(error, EINVAL,
11906                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11907                                           NULL,
11908                                           "Should assign at least one flow or"
11909                                           " context to get if nb_contexts != 0");
11910         age_info = GET_PORT_AGE_INFO(priv);
11911         rte_spinlock_lock(&age_info->aged_sl);
11912         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
11913                 nb_flows++;
11914                 if (nb_contexts) {
11915                         age_param = MLX5_CNT_TO_AGE(counter);
11916                         context[nb_flows - 1] = age_param->context;
11917                         if (!(--nb_contexts))
11918                                 break;
11919                 }
11920         }
11921         rte_spinlock_unlock(&age_info->aged_sl);
11922         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
11923         return nb_flows;
11924 }
11925
11926 /*
11927  * Mutex-protected thunk to lock-free  __flow_dv_translate().
11928  */
11929 static int
11930 flow_dv_translate(struct rte_eth_dev *dev,
11931                   struct mlx5_flow *dev_flow,
11932                   const struct rte_flow_attr *attr,
11933                   const struct rte_flow_item items[],
11934                   const struct rte_flow_action actions[],
11935                   struct rte_flow_error *error)
11936 {
11937         int ret;
11938
11939         flow_dv_shared_lock(dev);
11940         ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
11941         flow_dv_shared_unlock(dev);
11942         return ret;
11943 }
11944
11945 /*
11946  * Mutex-protected thunk to lock-free  __flow_dv_apply().
11947  */
11948 static int
11949 flow_dv_apply(struct rte_eth_dev *dev,
11950               struct rte_flow *flow,
11951               struct rte_flow_error *error)
11952 {
11953         int ret;
11954
11955         flow_dv_shared_lock(dev);
11956         ret = __flow_dv_apply(dev, flow, error);
11957         flow_dv_shared_unlock(dev);
11958         return ret;
11959 }
11960
11961 /*
11962  * Mutex-protected thunk to lock-free __flow_dv_remove().
11963  */
11964 static void
11965 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
11966 {
11967         flow_dv_shared_lock(dev);
11968         __flow_dv_remove(dev, flow);
11969         flow_dv_shared_unlock(dev);
11970 }
11971
11972 /*
11973  * Mutex-protected thunk to lock-free __flow_dv_destroy().
11974  */
11975 static void
11976 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
11977 {
11978         flow_dv_shared_lock(dev);
11979         __flow_dv_destroy(dev, flow);
11980         flow_dv_shared_unlock(dev);
11981 }
11982
11983 /*
11984  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
11985  */
11986 static uint32_t
11987 flow_dv_counter_allocate(struct rte_eth_dev *dev)
11988 {
11989         uint32_t cnt;
11990
11991         flow_dv_shared_lock(dev);
11992         cnt = flow_dv_counter_alloc(dev, 0);
11993         flow_dv_shared_unlock(dev);
11994         return cnt;
11995 }
11996
11997 /*
11998  * Mutex-protected thunk to lock-free flow_dv_counter_release().
11999  */
12000 static void
12001 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
12002 {
12003         flow_dv_shared_lock(dev);
12004         flow_dv_counter_release(dev, cnt);
12005         flow_dv_shared_unlock(dev);
12006 }
12007
12008 /**
12009  * Validate shared action.
12010  * Dispatcher for action type specific validation.
12011  *
12012  * @param[in] dev
12013  *   Pointer to the Ethernet device structure.
12014  * @param[in] conf
12015  *   Shared action configuration.
12016  * @param[in] action
12017  *   The shared action object to validate.
12018  * @param[out] error
12019  *   Perform verbose error reporting if not NULL. Initialized in case of
12020  *   error only.
12021  *
12022  * @return
12023  *   0 on success, otherwise negative errno value.
12024  */
12025 static int
12026 flow_dv_action_validate(struct rte_eth_dev *dev,
12027                         const struct rte_flow_shared_action_conf *conf,
12028                         const struct rte_flow_action *action,
12029                         struct rte_flow_error *error)
12030 {
12031         RTE_SET_USED(conf);
12032         switch (action->type) {
12033         case RTE_FLOW_ACTION_TYPE_RSS:
12034                 return mlx5_validate_action_rss(dev, action, error);
12035         default:
12036                 return rte_flow_error_set(error, ENOTSUP,
12037                                           RTE_FLOW_ERROR_TYPE_ACTION,
12038                                           NULL,
12039                                           "action type not supported");
12040         }
12041 }
12042
12043 /*
12044  * Mutex-protected thunk to lock-free  __flow_dv_action_create().
12045  */
12046 static struct rte_flow_shared_action *
12047 flow_dv_action_create(struct rte_eth_dev *dev,
12048                       const struct rte_flow_shared_action_conf *conf,
12049                       const struct rte_flow_action *action,
12050                       struct rte_flow_error *error)
12051 {
12052         struct rte_flow_shared_action *shared_action = NULL;
12053
12054         flow_dv_shared_lock(dev);
12055         shared_action = __flow_dv_action_create(dev, conf, action, error);
12056         flow_dv_shared_unlock(dev);
12057         return shared_action;
12058 }
12059
12060 /*
12061  * Mutex-protected thunk to lock-free  __flow_dv_action_destroy().
12062  */
12063 static int
12064 flow_dv_action_destroy(struct rte_eth_dev *dev,
12065                        struct rte_flow_shared_action *action,
12066                        struct rte_flow_error *error)
12067 {
12068         int ret;
12069
12070         flow_dv_shared_lock(dev);
12071         ret = __flow_dv_action_destroy(dev, action, error);
12072         flow_dv_shared_unlock(dev);
12073         return ret;
12074 }
12075
12076 /*
12077  * Mutex-protected thunk to lock-free  __flow_dv_action_update().
12078  */
12079 static int
12080 flow_dv_action_update(struct rte_eth_dev *dev,
12081                       struct rte_flow_shared_action *action,
12082                       const void *action_conf,
12083                       struct rte_flow_error *error)
12084 {
12085         int ret;
12086
12087         flow_dv_shared_lock(dev);
12088         ret = __flow_dv_action_update(dev, action, action_conf,
12089                                       error);
12090         flow_dv_shared_unlock(dev);
12091         return ret;
12092 }
12093
12094 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
12095         .validate = flow_dv_validate,
12096         .prepare = flow_dv_prepare,
12097         .translate = flow_dv_translate,
12098         .apply = flow_dv_apply,
12099         .remove = flow_dv_remove,
12100         .destroy = flow_dv_destroy,
12101         .query = flow_dv_query,
12102         .create_mtr_tbls = flow_dv_create_mtr_tbl,
12103         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
12104         .create_policer_rules = flow_dv_create_policer_rules,
12105         .destroy_policer_rules = flow_dv_destroy_policer_rules,
12106         .counter_alloc = flow_dv_counter_allocate,
12107         .counter_free = flow_dv_counter_free,
12108         .counter_query = flow_dv_counter_query,
12109         .get_aged_flows = flow_get_aged_flows,
12110         .action_validate = flow_dv_action_validate,
12111         .action_create = flow_dv_action_create,
12112         .action_destroy = flow_dv_action_destroy,
12113         .action_update = flow_dv_action_update,
12114 };
12115
12116 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
12117