drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 #include <rte_common.h>
  12 #include <rte_ether.h>
  13 #include <rte_ethdev_driver.h>
  14 #include <rte_flow.h>
  15 #include <rte_flow_driver.h>
  16 #include <rte_malloc.h>
  17 #include <rte_cycles.h>
  18 #include <rte_ip.h>
  19 #include <rte_gre.h>
  20 #include <rte_vxlan.h>
  21 #include <rte_gtp.h>
  22 #include <rte_eal_paging.h>
  23 #include <rte_mpls.h>
  24
  25 #include <mlx5_glue.h>
  26 #include <mlx5_devx_cmds.h>
  27 #include <mlx5_prm.h>
  28 #include <mlx5_malloc.h>
  29
  30 #include "mlx5_defs.h"
  31 #include "mlx5.h"
  32 #include "mlx5_common_os.h"
  33 #include "mlx5_flow.h"
  34 #include "mlx5_flow_os.h"
  35 #include "mlx5_rxtx.h"
  36
  37 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  38
  39 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  40 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  41 #endif
  42
  43 #ifndef HAVE_MLX5DV_DR_ESWITCH
  44 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  45 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  46 #endif
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR
  50 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  51 #endif
  52
  53 /* VLAN header definitions */
  54 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  55 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  56 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  57 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  58 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  59
  60 union flow_dv_attr {
  61         struct {
  62                 uint32_t valid:1;
  63                 uint32_t ipv4:1;
  64                 uint32_t ipv6:1;
  65                 uint32_t tcp:1;
  66                 uint32_t udp:1;
  67                 uint32_t reserved:27;
  68         };
  69         uint32_t attr;
  70 };
  71
  72 static int
  73 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
  74                              struct mlx5_flow_tbl_resource *tbl);
  75
  76 static int
  77 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev);
  78
  79 static int
  80 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
  81                                       uint32_t encap_decap_idx);
  82
  83 static int
  84 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
  85                                         uint32_t port_id);
  86
  87 /**
  88  * Initialize flow attributes structure according to flow items' types.
  89  *
  90  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  91  * mode. For tunnel mode, the items to be modified are the outermost ones.
  92  *
  93  * @param[in] item
  94  *   Pointer to item specification.
  95  * @param[out] attr
  96  *   Pointer to flow attributes structure.
  97  * @param[in] dev_flow
  98  *   Pointer to the sub flow.
  99  * @param[in] tunnel_decap
 100  *   Whether action is after tunnel decapsulation.
 101  */
 102 static void
 103 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 104                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 105 {
 106         uint64_t layers = dev_flow->handle->layers;
 107
 108         /*
 109          * If layers is already initialized, it means this dev_flow is the
 110          * suffix flow, the layers flags is set by the prefix flow. Need to
 111          * use the layer flags from prefix flow as the suffix flow may not
 112          * have the user defined items as the flow is split.
 113          */
 114         if (layers) {
 115                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 116                         attr->ipv4 = 1;
 117                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 118                         attr->ipv6 = 1;
 119                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 120                         attr->tcp = 1;
 121                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 122                         attr->udp = 1;
 123                 attr->valid = 1;
 124                 return;
 125         }
 126         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 127                 uint8_t next_protocol = 0xff;
 128                 switch (item->type) {
 129                 case RTE_FLOW_ITEM_TYPE_GRE:
 130                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 131                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 132                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 133                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 134                 case RTE_FLOW_ITEM_TYPE_MPLS:
 135                         if (tunnel_decap)
 136                                 attr->attr = 0;
 137                         break;
 138                 case RTE_FLOW_ITEM_TYPE_IPV4:
 139                         if (!attr->ipv6)
 140                                 attr->ipv4 = 1;
 141                         if (item->mask != NULL &&
 142                             ((const struct rte_flow_item_ipv4 *)
 143                             item->mask)->hdr.next_proto_id)
 144                                 next_protocol =
 145                                     ((const struct rte_flow_item_ipv4 *)
 146                                       (item->spec))->hdr.next_proto_id &
 147                                     ((const struct rte_flow_item_ipv4 *)
 148                                       (item->mask))->hdr.next_proto_id;
 149                         if ((next_protocol == IPPROTO_IPIP ||
 150                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 151                                 attr->attr = 0;
 152                         break;
 153                 case RTE_FLOW_ITEM_TYPE_IPV6:
 154                         if (!attr->ipv4)
 155                                 attr->ipv6 = 1;
 156                         if (item->mask != NULL &&
 157                             ((const struct rte_flow_item_ipv6 *)
 158                             item->mask)->hdr.proto)
 159                                 next_protocol =
 160                                     ((const struct rte_flow_item_ipv6 *)
 161                                       (item->spec))->hdr.proto &
 162                                     ((const struct rte_flow_item_ipv6 *)
 163                                       (item->mask))->hdr.proto;
 164                         if ((next_protocol == IPPROTO_IPIP ||
 165                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 166                                 attr->attr = 0;
 167                         break;
 168                 case RTE_FLOW_ITEM_TYPE_UDP:
 169                         if (!attr->tcp)
 170                                 attr->udp = 1;
 171                         break;
 172                 case RTE_FLOW_ITEM_TYPE_TCP:
 173                         if (!attr->udp)
 174                                 attr->tcp = 1;
 175                         break;
 176                 default:
 177                         break;
 178                 }
 179         }
 180         attr->valid = 1;
 181 }
 182
 183 /**
 184  * Convert rte_mtr_color to mlx5 color.
 185  *
 186  * @param[in] rcol
 187  *   rte_mtr_color.
 188  *
 189  * @return
 190  *   mlx5 color.
 191  */
 192 static int
 193 rte_col_2_mlx5_col(enum rte_color rcol)
 194 {
 195         switch (rcol) {
 196         case RTE_COLOR_GREEN:
 197                 return MLX5_FLOW_COLOR_GREEN;
 198         case RTE_COLOR_YELLOW:
 199                 return MLX5_FLOW_COLOR_YELLOW;
 200         case RTE_COLOR_RED:
 201                 return MLX5_FLOW_COLOR_RED;
 202         default:
 203                 break;
 204         }
 205         return MLX5_FLOW_COLOR_UNDEFINED;
 206 }
 207
 208 struct field_modify_info {
 209         uint32_t size; /* Size of field in protocol header, in bytes. */
 210         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 211         enum mlx5_modification_field id;
 212 };
 213
 214 struct field_modify_info modify_eth[] = {
 215         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 216         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 217         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 218         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 219         {0, 0, 0},
 220 };
 221
 222 struct field_modify_info modify_vlan_out_first_vid[] = {
 223         /* Size in bits !!! */
 224         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 225         {0, 0, 0},
 226 };
 227
 228 struct field_modify_info modify_ipv4[] = {
 229         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 230         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 231         {4, 12, MLX5_MODI_OUT_SIPV4},
 232         {4, 16, MLX5_MODI_OUT_DIPV4},
 233         {0, 0, 0},
 234 };
 235
 236 struct field_modify_info modify_ipv6[] = {
 237         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 238         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 239         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 240         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 241         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 242         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 243         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 244         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 245         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 246         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 247         {0, 0, 0},
 248 };
 249
 250 struct field_modify_info modify_udp[] = {
 251         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 252         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 253         {0, 0, 0},
 254 };
 255
 256 struct field_modify_info modify_tcp[] = {
 257         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 258         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 259         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 260         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 261         {0, 0, 0},
 262 };
 263
 264 static void
 265 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 266                           uint8_t next_protocol, uint64_t *item_flags,
 267                           int *tunnel)
 268 {
 269         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 270                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 271         if (next_protocol == IPPROTO_IPIP) {
 272                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 273                 *tunnel = 1;
 274         }
 275         if (next_protocol == IPPROTO_IPV6) {
 276                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 277                 *tunnel = 1;
 278         }
 279 }
 280
 281 /**
 282  * Acquire the synchronizing object to protect multithreaded access
 283  * to shared dv context. Lock occurs only if context is actually
 284  * shared, i.e. we have multiport IB device and representors are
 285  * created.
 286  *
 287  * @param[in] dev
 288  *   Pointer to the rte_eth_dev structure.
 289  */
 290 static void
 291 flow_dv_shared_lock(struct rte_eth_dev *dev)
 292 {
 293         struct mlx5_priv *priv = dev->data->dev_private;
 294         struct mlx5_dev_ctx_shared *sh = priv->sh;
 295
 296         if (sh->dv_refcnt > 1) {
 297                 int ret;
 298
 299                 ret = pthread_mutex_lock(&sh->dv_mutex);
 300                 MLX5_ASSERT(!ret);
 301                 (void)ret;
 302         }
 303 }
 304
 305 static void
 306 flow_dv_shared_unlock(struct rte_eth_dev *dev)
 307 {
 308         struct mlx5_priv *priv = dev->data->dev_private;
 309         struct mlx5_dev_ctx_shared *sh = priv->sh;
 310
 311         if (sh->dv_refcnt > 1) {
 312                 int ret;
 313
 314                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 315                 MLX5_ASSERT(!ret);
 316                 (void)ret;
 317         }
 318 }
 319
 320 /* Update VLAN's VID/PCP based on input rte_flow_action.
 321  *
 322  * @param[in] action
 323  *   Pointer to struct rte_flow_action.
 324  * @param[out] vlan
 325  *   Pointer to struct rte_vlan_hdr.
 326  */
 327 static void
 328 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 329                          struct rte_vlan_hdr *vlan)
 330 {
 331         uint16_t vlan_tci;
 332         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 333                 vlan_tci =
 334                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 335                                                action->conf)->vlan_pcp;
 336                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 337                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 338                 vlan->vlan_tci |= vlan_tci;
 339         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 340                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 341                 vlan->vlan_tci |= rte_be_to_cpu_16
 342                     (((const struct rte_flow_action_of_set_vlan_vid *)
 343                                              action->conf)->vlan_vid);
 344         }
 345 }
 346
 347 /**
 348  * Fetch 1, 2, 3 or 4 byte field from the byte array
 349  * and return as unsigned integer in host-endian format.
 350  *
 351  * @param[in] data
 352  *   Pointer to data array.
 353  * @param[in] size
 354  *   Size of field to extract.
 355  *
 356  * @return
 357  *   converted field in host endian format.
 358  */
 359 static inline uint32_t
 360 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 361 {
 362         uint32_t ret;
 363
 364         switch (size) {
 365         case 1:
 366                 ret = *data;
 367                 break;
 368         case 2:
 369                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 370                 break;
 371         case 3:
 372                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 373                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 374                 break;
 375         case 4:
 376                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 377                 break;
 378         default:
 379                 MLX5_ASSERT(false);
 380                 ret = 0;
 381                 break;
 382         }
 383         return ret;
 384 }
 385
 386 /**
 387  * Convert modify-header action to DV specification.
 388  *
 389  * Data length of each action is determined by provided field description
 390  * and the item mask. Data bit offset and width of each action is determined
 391  * by provided item mask.
 392  *
 393  * @param[in] item
 394  *   Pointer to item specification.
 395  * @param[in] field
 396  *   Pointer to field modification information.
 397  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 398  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 399  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 400  * @param[in] dcopy
 401  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 402  *   Negative offset value sets the same offset as source offset.
 403  *   size field is ignored, value is taken from source field.
 404  * @param[in,out] resource
 405  *   Pointer to the modify-header resource.
 406  * @param[in] type
 407  *   Type of modification.
 408  * @param[out] error
 409  *   Pointer to the error structure.
 410  *
 411  * @return
 412  *   0 on success, a negative errno value otherwise and rte_errno is set.
 413  */
 414 static int
 415 flow_dv_convert_modify_action(struct rte_flow_item *item,
 416                               struct field_modify_info *field,
 417                               struct field_modify_info *dcopy,
 418                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 419                               uint32_t type, struct rte_flow_error *error)
 420 {
 421         uint32_t i = resource->actions_num;
 422         struct mlx5_modification_cmd *actions = resource->actions;
 423
 424         /*
 425          * The item and mask are provided in big-endian format.
 426          * The fields should be presented as in big-endian format either.
 427          * Mask must be always present, it defines the actual field width.
 428          */
 429         MLX5_ASSERT(item->mask);
 430         MLX5_ASSERT(field->size);
 431         do {
 432                 unsigned int size_b;
 433                 unsigned int off_b;
 434                 uint32_t mask;
 435                 uint32_t data;
 436
 437                 if (i >= MLX5_MAX_MODIFY_NUM)
 438                         return rte_flow_error_set(error, EINVAL,
 439                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 440                                  "too many items to modify");
 441                 /* Fetch variable byte size mask from the array. */
 442                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 443                                            field->offset, field->size);
 444                 if (!mask) {
 445                         ++field;
 446                         continue;
 447                 }
 448                 /* Deduce actual data width in bits from mask value. */
 449                 off_b = rte_bsf32(mask);
 450                 size_b = sizeof(uint32_t) * CHAR_BIT -
 451                          off_b - __builtin_clz(mask);
 452                 MLX5_ASSERT(size_b);
 453                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 454                 actions[i] = (struct mlx5_modification_cmd) {
 455                         .action_type = type,
 456                         .field = field->id,
 457                         .offset = off_b,
 458                         .length = size_b,
 459                 };
 460                 /* Convert entire record to expected big-endian format. */
 461                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 462                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 463                         MLX5_ASSERT(dcopy);
 464                         actions[i].dst_field = dcopy->id;
 465                         actions[i].dst_offset =
 466                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 467                         /* Convert entire record to big-endian format. */
 468                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 469                 } else {
 470                         MLX5_ASSERT(item->spec);
 471                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 472                                                    field->offset, field->size);
 473                         /* Shift out the trailing masked bits from data. */
 474                         data = (data & mask) >> off_b;
 475                         actions[i].data1 = rte_cpu_to_be_32(data);
 476                 }
 477                 ++i;
 478                 ++field;
 479         } while (field->size);
 480         if (resource->actions_num == i)
 481                 return rte_flow_error_set(error, EINVAL,
 482                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 483                                           "invalid modification flow item");
 484         resource->actions_num = i;
 485         return 0;
 486 }
 487
 488 /**
 489  * Convert modify-header set IPv4 address action to DV specification.
 490  *
 491  * @param[in,out] resource
 492  *   Pointer to the modify-header resource.
 493  * @param[in] action
 494  *   Pointer to action specification.
 495  * @param[out] error
 496  *   Pointer to the error structure.
 497  *
 498  * @return
 499  *   0 on success, a negative errno value otherwise and rte_errno is set.
 500  */
 501 static int
 502 flow_dv_convert_action_modify_ipv4
 503                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 504                          const struct rte_flow_action *action,
 505                          struct rte_flow_error *error)
 506 {
 507         const struct rte_flow_action_set_ipv4 *conf =
 508                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 509         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 510         struct rte_flow_item_ipv4 ipv4;
 511         struct rte_flow_item_ipv4 ipv4_mask;
 512
 513         memset(&ipv4, 0, sizeof(ipv4));
 514         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 515         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 516                 ipv4.hdr.src_addr = conf->ipv4_addr;
 517                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 518         } else {
 519                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 520                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 521         }
 522         item.spec = &ipv4;
 523         item.mask = &ipv4_mask;
 524         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 525                                              MLX5_MODIFICATION_TYPE_SET, error);
 526 }
 527
 528 /**
 529  * Convert modify-header set IPv6 address action to DV specification.
 530  *
 531  * @param[in,out] resource
 532  *   Pointer to the modify-header resource.
 533  * @param[in] action
 534  *   Pointer to action specification.
 535  * @param[out] error
 536  *   Pointer to the error structure.
 537  *
 538  * @return
 539  *   0 on success, a negative errno value otherwise and rte_errno is set.
 540  */
 541 static int
 542 flow_dv_convert_action_modify_ipv6
 543                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 544                          const struct rte_flow_action *action,
 545                          struct rte_flow_error *error)
 546 {
 547         const struct rte_flow_action_set_ipv6 *conf =
 548                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 549         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 550         struct rte_flow_item_ipv6 ipv6;
 551         struct rte_flow_item_ipv6 ipv6_mask;
 552
 553         memset(&ipv6, 0, sizeof(ipv6));
 554         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 555         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 556                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 557                        sizeof(ipv6.hdr.src_addr));
 558                 memcpy(&ipv6_mask.hdr.src_addr,
 559                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 560                        sizeof(ipv6.hdr.src_addr));
 561         } else {
 562                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 563                        sizeof(ipv6.hdr.dst_addr));
 564                 memcpy(&ipv6_mask.hdr.dst_addr,
 565                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 566                        sizeof(ipv6.hdr.dst_addr));
 567         }
 568         item.spec = &ipv6;
 569         item.mask = &ipv6_mask;
 570         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 571                                              MLX5_MODIFICATION_TYPE_SET, error);
 572 }
 573
 574 /**
 575  * Convert modify-header set MAC address action to DV specification.
 576  *
 577  * @param[in,out] resource
 578  *   Pointer to the modify-header resource.
 579  * @param[in] action
 580  *   Pointer to action specification.
 581  * @param[out] error
 582  *   Pointer to the error structure.
 583  *
 584  * @return
 585  *   0 on success, a negative errno value otherwise and rte_errno is set.
 586  */
 587 static int
 588 flow_dv_convert_action_modify_mac
 589                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 590                          const struct rte_flow_action *action,
 591                          struct rte_flow_error *error)
 592 {
 593         const struct rte_flow_action_set_mac *conf =
 594                 (const struct rte_flow_action_set_mac *)(action->conf);
 595         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 596         struct rte_flow_item_eth eth;
 597         struct rte_flow_item_eth eth_mask;
 598
 599         memset(&eth, 0, sizeof(eth));
 600         memset(&eth_mask, 0, sizeof(eth_mask));
 601         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 602                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 603                        sizeof(eth.src.addr_bytes));
 604                 memcpy(&eth_mask.src.addr_bytes,
 605                        &rte_flow_item_eth_mask.src.addr_bytes,
 606                        sizeof(eth_mask.src.addr_bytes));
 607         } else {
 608                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 609                        sizeof(eth.dst.addr_bytes));
 610                 memcpy(&eth_mask.dst.addr_bytes,
 611                        &rte_flow_item_eth_mask.dst.addr_bytes,
 612                        sizeof(eth_mask.dst.addr_bytes));
 613         }
 614         item.spec = &eth;
 615         item.mask = &eth_mask;
 616         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 617                                              MLX5_MODIFICATION_TYPE_SET, error);
 618 }
 619
 620 /**
 621  * Convert modify-header set VLAN VID action to DV specification.
 622  *
 623  * @param[in,out] resource
 624  *   Pointer to the modify-header resource.
 625  * @param[in] action
 626  *   Pointer to action specification.
 627  * @param[out] error
 628  *   Pointer to the error structure.
 629  *
 630  * @return
 631  *   0 on success, a negative errno value otherwise and rte_errno is set.
 632  */
 633 static int
 634 flow_dv_convert_action_modify_vlan_vid
 635                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 636                          const struct rte_flow_action *action,
 637                          struct rte_flow_error *error)
 638 {
 639         const struct rte_flow_action_of_set_vlan_vid *conf =
 640                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 641         int i = resource->actions_num;
 642         struct mlx5_modification_cmd *actions = resource->actions;
 643         struct field_modify_info *field = modify_vlan_out_first_vid;
 644
 645         if (i >= MLX5_MAX_MODIFY_NUM)
 646                 return rte_flow_error_set(error, EINVAL,
 647                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 648                          "too many items to modify");
 649         actions[i] = (struct mlx5_modification_cmd) {
 650                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 651                 .field = field->id,
 652                 .length = field->size,
 653                 .offset = field->offset,
 654         };
 655         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 656         actions[i].data1 = conf->vlan_vid;
 657         actions[i].data1 = actions[i].data1 << 16;
 658         resource->actions_num = ++i;
 659         return 0;
 660 }
 661
 662 /**
 663  * Convert modify-header set TP action to DV specification.
 664  *
 665  * @param[in,out] resource
 666  *   Pointer to the modify-header resource.
 667  * @param[in] action
 668  *   Pointer to action specification.
 669  * @param[in] items
 670  *   Pointer to rte_flow_item objects list.
 671  * @param[in] attr
 672  *   Pointer to flow attributes structure.
 673  * @param[in] dev_flow
 674  *   Pointer to the sub flow.
 675  * @param[in] tunnel_decap
 676  *   Whether action is after tunnel decapsulation.
 677  * @param[out] error
 678  *   Pointer to the error structure.
 679  *
 680  * @return
 681  *   0 on success, a negative errno value otherwise and rte_errno is set.
 682  */
 683 static int
 684 flow_dv_convert_action_modify_tp
 685                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 686                          const struct rte_flow_action *action,
 687                          const struct rte_flow_item *items,
 688                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 689                          bool tunnel_decap, struct rte_flow_error *error)
 690 {
 691         const struct rte_flow_action_set_tp *conf =
 692                 (const struct rte_flow_action_set_tp *)(action->conf);
 693         struct rte_flow_item item;
 694         struct rte_flow_item_udp udp;
 695         struct rte_flow_item_udp udp_mask;
 696         struct rte_flow_item_tcp tcp;
 697         struct rte_flow_item_tcp tcp_mask;
 698         struct field_modify_info *field;
 699
 700         if (!attr->valid)
 701                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 702         if (attr->udp) {
 703                 memset(&udp, 0, sizeof(udp));
 704                 memset(&udp_mask, 0, sizeof(udp_mask));
 705                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 706                         udp.hdr.src_port = conf->port;
 707                         udp_mask.hdr.src_port =
 708                                         rte_flow_item_udp_mask.hdr.src_port;
 709                 } else {
 710                         udp.hdr.dst_port = conf->port;
 711                         udp_mask.hdr.dst_port =
 712                                         rte_flow_item_udp_mask.hdr.dst_port;
 713                 }
 714                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 715                 item.spec = &udp;
 716                 item.mask = &udp_mask;
 717                 field = modify_udp;
 718         } else {
 719                 MLX5_ASSERT(attr->tcp);
 720                 memset(&tcp, 0, sizeof(tcp));
 721                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 722                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 723                         tcp.hdr.src_port = conf->port;
 724                         tcp_mask.hdr.src_port =
 725                                         rte_flow_item_tcp_mask.hdr.src_port;
 726                 } else {
 727                         tcp.hdr.dst_port = conf->port;
 728                         tcp_mask.hdr.dst_port =
 729                                         rte_flow_item_tcp_mask.hdr.dst_port;
 730                 }
 731                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 732                 item.spec = &tcp;
 733                 item.mask = &tcp_mask;
 734                 field = modify_tcp;
 735         }
 736         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 737                                              MLX5_MODIFICATION_TYPE_SET, error);
 738 }
 739
 740 /**
 741  * Convert modify-header set TTL action to DV specification.
 742  *
 743  * @param[in,out] resource
 744  *   Pointer to the modify-header resource.
 745  * @param[in] action
 746  *   Pointer to action specification.
 747  * @param[in] items
 748  *   Pointer to rte_flow_item objects list.
 749  * @param[in] attr
 750  *   Pointer to flow attributes structure.
 751  * @param[in] dev_flow
 752  *   Pointer to the sub flow.
 753  * @param[in] tunnel_decap
 754  *   Whether action is after tunnel decapsulation.
 755  * @param[out] error
 756  *   Pointer to the error structure.
 757  *
 758  * @return
 759  *   0 on success, a negative errno value otherwise and rte_errno is set.
 760  */
 761 static int
 762 flow_dv_convert_action_modify_ttl
 763                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 764                          const struct rte_flow_action *action,
 765                          const struct rte_flow_item *items,
 766                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 767                          bool tunnel_decap, struct rte_flow_error *error)
 768 {
 769         const struct rte_flow_action_set_ttl *conf =
 770                 (const struct rte_flow_action_set_ttl *)(action->conf);
 771         struct rte_flow_item item;
 772         struct rte_flow_item_ipv4 ipv4;
 773         struct rte_flow_item_ipv4 ipv4_mask;
 774         struct rte_flow_item_ipv6 ipv6;
 775         struct rte_flow_item_ipv6 ipv6_mask;
 776         struct field_modify_info *field;
 777
 778         if (!attr->valid)
 779                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 780         if (attr->ipv4) {
 781                 memset(&ipv4, 0, sizeof(ipv4));
 782                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 783                 ipv4.hdr.time_to_live = conf->ttl_value;
 784                 ipv4_mask.hdr.time_to_live = 0xFF;
 785                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 786                 item.spec = &ipv4;
 787                 item.mask = &ipv4_mask;
 788                 field = modify_ipv4;
 789         } else {
 790                 MLX5_ASSERT(attr->ipv6);
 791                 memset(&ipv6, 0, sizeof(ipv6));
 792                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 793                 ipv6.hdr.hop_limits = conf->ttl_value;
 794                 ipv6_mask.hdr.hop_limits = 0xFF;
 795                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 796                 item.spec = &ipv6;
 797                 item.mask = &ipv6_mask;
 798                 field = modify_ipv6;
 799         }
 800         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 801                                              MLX5_MODIFICATION_TYPE_SET, error);
 802 }
 803
 804 /**
 805  * Convert modify-header decrement TTL action to DV specification.
 806  *
 807  * @param[in,out] resource
 808  *   Pointer to the modify-header resource.
 809  * @param[in] action
 810  *   Pointer to action specification.
 811  * @param[in] items
 812  *   Pointer to rte_flow_item objects list.
 813  * @param[in] attr
 814  *   Pointer to flow attributes structure.
 815  * @param[in] dev_flow
 816  *   Pointer to the sub flow.
 817  * @param[in] tunnel_decap
 818  *   Whether action is after tunnel decapsulation.
 819  * @param[out] error
 820  *   Pointer to the error structure.
 821  *
 822  * @return
 823  *   0 on success, a negative errno value otherwise and rte_errno is set.
 824  */
 825 static int
 826 flow_dv_convert_action_modify_dec_ttl
 827                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 828                          const struct rte_flow_item *items,
 829                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 830                          bool tunnel_decap, struct rte_flow_error *error)
 831 {
 832         struct rte_flow_item item;
 833         struct rte_flow_item_ipv4 ipv4;
 834         struct rte_flow_item_ipv4 ipv4_mask;
 835         struct rte_flow_item_ipv6 ipv6;
 836         struct rte_flow_item_ipv6 ipv6_mask;
 837         struct field_modify_info *field;
 838
 839         if (!attr->valid)
 840                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 841         if (attr->ipv4) {
 842                 memset(&ipv4, 0, sizeof(ipv4));
 843                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 844                 ipv4.hdr.time_to_live = 0xFF;
 845                 ipv4_mask.hdr.time_to_live = 0xFF;
 846                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 847                 item.spec = &ipv4;
 848                 item.mask = &ipv4_mask;
 849                 field = modify_ipv4;
 850         } else {
 851                 MLX5_ASSERT(attr->ipv6);
 852                 memset(&ipv6, 0, sizeof(ipv6));
 853                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 854                 ipv6.hdr.hop_limits = 0xFF;
 855                 ipv6_mask.hdr.hop_limits = 0xFF;
 856                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 857                 item.spec = &ipv6;
 858                 item.mask = &ipv6_mask;
 859                 field = modify_ipv6;
 860         }
 861         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 862                                              MLX5_MODIFICATION_TYPE_ADD, error);
 863 }
 864
 865 /**
 866  * Convert modify-header increment/decrement TCP Sequence number
 867  * to DV specification.
 868  *
 869  * @param[in,out] resource
 870  *   Pointer to the modify-header resource.
 871  * @param[in] action
 872  *   Pointer to action specification.
 873  * @param[out] error
 874  *   Pointer to the error structure.
 875  *
 876  * @return
 877  *   0 on success, a negative errno value otherwise and rte_errno is set.
 878  */
 879 static int
 880 flow_dv_convert_action_modify_tcp_seq
 881                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 882                          const struct rte_flow_action *action,
 883                          struct rte_flow_error *error)
 884 {
 885         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 886         uint64_t value = rte_be_to_cpu_32(*conf);
 887         struct rte_flow_item item;
 888         struct rte_flow_item_tcp tcp;
 889         struct rte_flow_item_tcp tcp_mask;
 890
 891         memset(&tcp, 0, sizeof(tcp));
 892         memset(&tcp_mask, 0, sizeof(tcp_mask));
 893         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 894                 /*
 895                  * The HW has no decrement operation, only increment operation.
 896                  * To simulate decrement X from Y using increment operation
 897                  * we need to add UINT32_MAX X times to Y.
 898                  * Each adding of UINT32_MAX decrements Y by 1.
 899                  */
 900                 value *= UINT32_MAX;
 901         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 902         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 903         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 904         item.spec = &tcp;
 905         item.mask = &tcp_mask;
 906         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 907                                              MLX5_MODIFICATION_TYPE_ADD, error);
 908 }
 909
 910 /**
 911  * Convert modify-header increment/decrement TCP Acknowledgment number
 912  * to DV specification.
 913  *
 914  * @param[in,out] resource
 915  *   Pointer to the modify-header resource.
 916  * @param[in] action
 917  *   Pointer to action specification.
 918  * @param[out] error
 919  *   Pointer to the error structure.
 920  *
 921  * @return
 922  *   0 on success, a negative errno value otherwise and rte_errno is set.
 923  */
 924 static int
 925 flow_dv_convert_action_modify_tcp_ack
 926                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 927                          const struct rte_flow_action *action,
 928                          struct rte_flow_error *error)
 929 {
 930         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 931         uint64_t value = rte_be_to_cpu_32(*conf);
 932         struct rte_flow_item item;
 933         struct rte_flow_item_tcp tcp;
 934         struct rte_flow_item_tcp tcp_mask;
 935
 936         memset(&tcp, 0, sizeof(tcp));
 937         memset(&tcp_mask, 0, sizeof(tcp_mask));
 938         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 939                 /*
 940                  * The HW has no decrement operation, only increment operation.
 941                  * To simulate decrement X from Y using increment operation
 942                  * we need to add UINT32_MAX X times to Y.
 943                  * Each adding of UINT32_MAX decrements Y by 1.
 944                  */
 945                 value *= UINT32_MAX;
 946         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 947         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 948         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 949         item.spec = &tcp;
 950         item.mask = &tcp_mask;
 951         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 952                                              MLX5_MODIFICATION_TYPE_ADD, error);
 953 }
 954
 955 static enum mlx5_modification_field reg_to_field[] = {
 956         [REG_NON] = MLX5_MODI_OUT_NONE,
 957         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 958         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 959         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 960         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 961         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 962         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 963         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 964         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 965         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 966         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 967 };
 968
 969 /**
 970  * Convert register set to DV specification.
 971  *
 972  * @param[in,out] resource
 973  *   Pointer to the modify-header resource.
 974  * @param[in] action
 975  *   Pointer to action specification.
 976  * @param[out] error
 977  *   Pointer to the error structure.
 978  *
 979  * @return
 980  *   0 on success, a negative errno value otherwise and rte_errno is set.
 981  */
 982 static int
 983 flow_dv_convert_action_set_reg
 984                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 985                          const struct rte_flow_action *action,
 986                          struct rte_flow_error *error)
 987 {
 988         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 989         struct mlx5_modification_cmd *actions = resource->actions;
 990         uint32_t i = resource->actions_num;
 991
 992         if (i >= MLX5_MAX_MODIFY_NUM)
 993                 return rte_flow_error_set(error, EINVAL,
 994                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 995                                           "too many items to modify");
 996         MLX5_ASSERT(conf->id != REG_NON);
 997         MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field));
 998         actions[i] = (struct mlx5_modification_cmd) {
 999                 .action_type = MLX5_MODIFICATION_TYPE_SET,
1000                 .field = reg_to_field[conf->id],
1001         };
1002         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1003         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1004         ++i;
1005         resource->actions_num = i;
1006         return 0;
1007 }
1008
1009 /**
1010  * Convert SET_TAG action to DV specification.
1011  *
1012  * @param[in] dev
1013  *   Pointer to the rte_eth_dev structure.
1014  * @param[in,out] resource
1015  *   Pointer to the modify-header resource.
1016  * @param[in] conf
1017  *   Pointer to action specification.
1018  * @param[out] error
1019  *   Pointer to the error structure.
1020  *
1021  * @return
1022  *   0 on success, a negative errno value otherwise and rte_errno is set.
1023  */
1024 static int
1025 flow_dv_convert_action_set_tag
1026                         (struct rte_eth_dev *dev,
1027                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1028                          const struct rte_flow_action_set_tag *conf,
1029                          struct rte_flow_error *error)
1030 {
1031         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1032         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1033         struct rte_flow_item item = {
1034                 .spec = &data,
1035                 .mask = &mask,
1036         };
1037         struct field_modify_info reg_c_x[] = {
1038                 [1] = {0, 0, 0},
1039         };
1040         enum mlx5_modification_field reg_type;
1041         int ret;
1042
1043         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1044         if (ret < 0)
1045                 return ret;
1046         MLX5_ASSERT(ret != REG_NON);
1047         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1048         reg_type = reg_to_field[ret];
1049         MLX5_ASSERT(reg_type > 0);
1050         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1051         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1052                                              MLX5_MODIFICATION_TYPE_SET, error);
1053 }
1054
1055 /**
1056  * Convert internal COPY_REG action to DV specification.
1057  *
1058  * @param[in] dev
1059  *   Pointer to the rte_eth_dev structure.
1060  * @param[in,out] res
1061  *   Pointer to the modify-header resource.
1062  * @param[in] action
1063  *   Pointer to action specification.
1064  * @param[out] error
1065  *   Pointer to the error structure.
1066  *
1067  * @return
1068  *   0 on success, a negative errno value otherwise and rte_errno is set.
1069  */
1070 static int
1071 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1072                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1073                                  const struct rte_flow_action *action,
1074                                  struct rte_flow_error *error)
1075 {
1076         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1077         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1078         struct rte_flow_item item = {
1079                 .spec = NULL,
1080                 .mask = &mask,
1081         };
1082         struct field_modify_info reg_src[] = {
1083                 {4, 0, reg_to_field[conf->src]},
1084                 {0, 0, 0},
1085         };
1086         struct field_modify_info reg_dst = {
1087                 .offset = 0,
1088                 .id = reg_to_field[conf->dst],
1089         };
1090         /* Adjust reg_c[0] usage according to reported mask. */
1091         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1092                 struct mlx5_priv *priv = dev->data->dev_private;
1093                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1094
1095                 MLX5_ASSERT(reg_c0);
1096                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1097                 if (conf->dst == REG_C_0) {
1098                         /* Copy to reg_c[0], within mask only. */
1099                         reg_dst.offset = rte_bsf32(reg_c0);
1100                         /*
1101                          * Mask is ignoring the enianness, because
1102                          * there is no conversion in datapath.
1103                          */
1104 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1105                         /* Copy from destination lower bits to reg_c[0]. */
1106                         mask = reg_c0 >> reg_dst.offset;
1107 #else
1108                         /* Copy from destination upper bits to reg_c[0]. */
1109                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1110                                           rte_fls_u32(reg_c0));
1111 #endif
1112                 } else {
1113                         mask = rte_cpu_to_be_32(reg_c0);
1114 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1115                         /* Copy from reg_c[0] to destination lower bits. */
1116                         reg_dst.offset = 0;
1117 #else
1118                         /* Copy from reg_c[0] to destination upper bits. */
1119                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1120                                          (rte_fls_u32(reg_c0) -
1121                                           rte_bsf32(reg_c0));
1122 #endif
1123                 }
1124         }
1125         return flow_dv_convert_modify_action(&item,
1126                                              reg_src, &reg_dst, res,
1127                                              MLX5_MODIFICATION_TYPE_COPY,
1128                                              error);
1129 }
1130
1131 /**
1132  * Convert MARK action to DV specification. This routine is used
1133  * in extensive metadata only and requires metadata register to be
1134  * handled. In legacy mode hardware tag resource is engaged.
1135  *
1136  * @param[in] dev
1137  *   Pointer to the rte_eth_dev structure.
1138  * @param[in] conf
1139  *   Pointer to MARK action specification.
1140  * @param[in,out] resource
1141  *   Pointer to the modify-header resource.
1142  * @param[out] error
1143  *   Pointer to the error structure.
1144  *
1145  * @return
1146  *   0 on success, a negative errno value otherwise and rte_errno is set.
1147  */
1148 static int
1149 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1150                             const struct rte_flow_action_mark *conf,
1151                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1152                             struct rte_flow_error *error)
1153 {
1154         struct mlx5_priv *priv = dev->data->dev_private;
1155         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1156                                            priv->sh->dv_mark_mask);
1157         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1158         struct rte_flow_item item = {
1159                 .spec = &data,
1160                 .mask = &mask,
1161         };
1162         struct field_modify_info reg_c_x[] = {
1163                 [1] = {0, 0, 0},
1164         };
1165         int reg;
1166
1167         if (!mask)
1168                 return rte_flow_error_set(error, EINVAL,
1169                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1170                                           NULL, "zero mark action mask");
1171         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1172         if (reg < 0)
1173                 return reg;
1174         MLX5_ASSERT(reg > 0);
1175         if (reg == REG_C_0) {
1176                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1177                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1178
1179                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1180                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1181                 mask = rte_cpu_to_be_32(mask << shl_c0);
1182         }
1183         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1184         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1185                                              MLX5_MODIFICATION_TYPE_SET, error);
1186 }
1187
1188 /**
1189  * Get metadata register index for specified steering domain.
1190  *
1191  * @param[in] dev
1192  *   Pointer to the rte_eth_dev structure.
1193  * @param[in] attr
1194  *   Attributes of flow to determine steering domain.
1195  * @param[out] error
1196  *   Pointer to the error structure.
1197  *
1198  * @return
1199  *   positive index on success, a negative errno value otherwise
1200  *   and rte_errno is set.
1201  */
1202 static enum modify_reg
1203 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1204                          const struct rte_flow_attr *attr,
1205                          struct rte_flow_error *error)
1206 {
1207         int reg =
1208                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1209                                           MLX5_METADATA_FDB :
1210                                             attr->egress ?
1211                                             MLX5_METADATA_TX :
1212                                             MLX5_METADATA_RX, 0, error);
1213         if (reg < 0)
1214                 return rte_flow_error_set(error,
1215                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1216                                           NULL, "unavailable "
1217                                           "metadata register");
1218         return reg;
1219 }
1220
1221 /**
1222  * Convert SET_META action to DV specification.
1223  *
1224  * @param[in] dev
1225  *   Pointer to the rte_eth_dev structure.
1226  * @param[in,out] resource
1227  *   Pointer to the modify-header resource.
1228  * @param[in] attr
1229  *   Attributes of flow that includes this item.
1230  * @param[in] conf
1231  *   Pointer to action specification.
1232  * @param[out] error
1233  *   Pointer to the error structure.
1234  *
1235  * @return
1236  *   0 on success, a negative errno value otherwise and rte_errno is set.
1237  */
1238 static int
1239 flow_dv_convert_action_set_meta
1240                         (struct rte_eth_dev *dev,
1241                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1242                          const struct rte_flow_attr *attr,
1243                          const struct rte_flow_action_set_meta *conf,
1244                          struct rte_flow_error *error)
1245 {
1246         uint32_t data = conf->data;
1247         uint32_t mask = conf->mask;
1248         struct rte_flow_item item = {
1249                 .spec = &data,
1250                 .mask = &mask,
1251         };
1252         struct field_modify_info reg_c_x[] = {
1253                 [1] = {0, 0, 0},
1254         };
1255         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1256
1257         if (reg < 0)
1258                 return reg;
1259         /*
1260          * In datapath code there is no endianness
1261          * coversions for perfromance reasons, all
1262          * pattern conversions are done in rte_flow.
1263          */
1264         if (reg == REG_C_0) {
1265                 struct mlx5_priv *priv = dev->data->dev_private;
1266                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1267                 uint32_t shl_c0;
1268
1269                 MLX5_ASSERT(msk_c0);
1270 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1271                 shl_c0 = rte_bsf32(msk_c0);
1272 #else
1273                 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1274 #endif
1275                 mask <<= shl_c0;
1276                 data <<= shl_c0;
1277                 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1278         }
1279         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1280         /* The routine expects parameters in memory as big-endian ones. */
1281         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1282                                              MLX5_MODIFICATION_TYPE_SET, error);
1283 }
1284
1285 /**
1286  * Convert modify-header set IPv4 DSCP action to DV specification.
1287  *
1288  * @param[in,out] resource
1289  *   Pointer to the modify-header resource.
1290  * @param[in] action
1291  *   Pointer to action specification.
1292  * @param[out] error
1293  *   Pointer to the error structure.
1294  *
1295  * @return
1296  *   0 on success, a negative errno value otherwise and rte_errno is set.
1297  */
1298 static int
1299 flow_dv_convert_action_modify_ipv4_dscp
1300                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1301                          const struct rte_flow_action *action,
1302                          struct rte_flow_error *error)
1303 {
1304         const struct rte_flow_action_set_dscp *conf =
1305                 (const struct rte_flow_action_set_dscp *)(action->conf);
1306         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1307         struct rte_flow_item_ipv4 ipv4;
1308         struct rte_flow_item_ipv4 ipv4_mask;
1309
1310         memset(&ipv4, 0, sizeof(ipv4));
1311         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1312         ipv4.hdr.type_of_service = conf->dscp;
1313         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1314         item.spec = &ipv4;
1315         item.mask = &ipv4_mask;
1316         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1317                                              MLX5_MODIFICATION_TYPE_SET, error);
1318 }
1319
1320 /**
1321  * Convert modify-header set IPv6 DSCP action to DV specification.
1322  *
1323  * @param[in,out] resource
1324  *   Pointer to the modify-header resource.
1325  * @param[in] action
1326  *   Pointer to action specification.
1327  * @param[out] error
1328  *   Pointer to the error structure.
1329  *
1330  * @return
1331  *   0 on success, a negative errno value otherwise and rte_errno is set.
1332  */
1333 static int
1334 flow_dv_convert_action_modify_ipv6_dscp
1335                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1336                          const struct rte_flow_action *action,
1337                          struct rte_flow_error *error)
1338 {
1339         const struct rte_flow_action_set_dscp *conf =
1340                 (const struct rte_flow_action_set_dscp *)(action->conf);
1341         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1342         struct rte_flow_item_ipv6 ipv6;
1343         struct rte_flow_item_ipv6 ipv6_mask;
1344
1345         memset(&ipv6, 0, sizeof(ipv6));
1346         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1347         /*
1348          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1349          * rdma-core only accept the DSCP bits byte aligned start from
1350          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1351          * bits in IPv6 case as rdma-core requires byte aligned value.
1352          */
1353         ipv6.hdr.vtc_flow = conf->dscp;
1354         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1355         item.spec = &ipv6;
1356         item.mask = &ipv6_mask;
1357         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1358                                              MLX5_MODIFICATION_TYPE_SET, error);
1359 }
1360
1361 /**
1362  * Validate MARK item.
1363  *
1364  * @param[in] dev
1365  *   Pointer to the rte_eth_dev structure.
1366  * @param[in] item
1367  *   Item specification.
1368  * @param[in] attr
1369  *   Attributes of flow that includes this item.
1370  * @param[out] error
1371  *   Pointer to error structure.
1372  *
1373  * @return
1374  *   0 on success, a negative errno value otherwise and rte_errno is set.
1375  */
1376 static int
1377 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1378                            const struct rte_flow_item *item,
1379                            const struct rte_flow_attr *attr __rte_unused,
1380                            struct rte_flow_error *error)
1381 {
1382         struct mlx5_priv *priv = dev->data->dev_private;
1383         struct mlx5_dev_config *config = &priv->config;
1384         const struct rte_flow_item_mark *spec = item->spec;
1385         const struct rte_flow_item_mark *mask = item->mask;
1386         const struct rte_flow_item_mark nic_mask = {
1387                 .id = priv->sh->dv_mark_mask,
1388         };
1389         int ret;
1390
1391         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1392                 return rte_flow_error_set(error, ENOTSUP,
1393                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1394                                           "extended metadata feature"
1395                                           " isn't enabled");
1396         if (!mlx5_flow_ext_mreg_supported(dev))
1397                 return rte_flow_error_set(error, ENOTSUP,
1398                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1399                                           "extended metadata register"
1400                                           " isn't supported");
1401         if (!nic_mask.id)
1402                 return rte_flow_error_set(error, ENOTSUP,
1403                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1404                                           "extended metadata register"
1405                                           " isn't available");
1406         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1407         if (ret < 0)
1408                 return ret;
1409         if (!spec)
1410                 return rte_flow_error_set(error, EINVAL,
1411                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1412                                           item->spec,
1413                                           "data cannot be empty");
1414         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1415                 return rte_flow_error_set(error, EINVAL,
1416                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1417                                           &spec->id,
1418                                           "mark id exceeds the limit");
1419         if (!mask)
1420                 mask = &nic_mask;
1421         if (!mask->id)
1422                 return rte_flow_error_set(error, EINVAL,
1423                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1424                                         "mask cannot be zero");
1425
1426         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1427                                         (const uint8_t *)&nic_mask,
1428                                         sizeof(struct rte_flow_item_mark),
1429                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1430         if (ret < 0)
1431                 return ret;
1432         return 0;
1433 }
1434
1435 /**
1436  * Validate META item.
1437  *
1438  * @param[in] dev
1439  *   Pointer to the rte_eth_dev structure.
1440  * @param[in] item
1441  *   Item specification.
1442  * @param[in] attr
1443  *   Attributes of flow that includes this item.
1444  * @param[out] error
1445  *   Pointer to error structure.
1446  *
1447  * @return
1448  *   0 on success, a negative errno value otherwise and rte_errno is set.
1449  */
1450 static int
1451 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1452                            const struct rte_flow_item *item,
1453                            const struct rte_flow_attr *attr,
1454                            struct rte_flow_error *error)
1455 {
1456         struct mlx5_priv *priv = dev->data->dev_private;
1457         struct mlx5_dev_config *config = &priv->config;
1458         const struct rte_flow_item_meta *spec = item->spec;
1459         const struct rte_flow_item_meta *mask = item->mask;
1460         struct rte_flow_item_meta nic_mask = {
1461                 .data = UINT32_MAX
1462         };
1463         int reg;
1464         int ret;
1465
1466         if (!spec)
1467                 return rte_flow_error_set(error, EINVAL,
1468                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1469                                           item->spec,
1470                                           "data cannot be empty");
1471         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1472                 if (!mlx5_flow_ext_mreg_supported(dev))
1473                         return rte_flow_error_set(error, ENOTSUP,
1474                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1475                                           "extended metadata register"
1476                                           " isn't supported");
1477                 reg = flow_dv_get_metadata_reg(dev, attr, error);
1478                 if (reg < 0)
1479                         return reg;
1480                 if (reg == REG_B)
1481                         return rte_flow_error_set(error, ENOTSUP,
1482                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1483                                           "match on reg_b "
1484                                           "isn't supported");
1485                 if (reg != REG_A)
1486                         nic_mask.data = priv->sh->dv_meta_mask;
1487         } else if (attr->transfer) {
1488                 return rte_flow_error_set(error, ENOTSUP,
1489                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1490                                         "extended metadata feature "
1491                                         "should be enabled when "
1492                                         "meta item is requested "
1493                                         "with e-switch mode ");
1494         }
1495         if (!mask)
1496                 mask = &rte_flow_item_meta_mask;
1497         if (!mask->data)
1498                 return rte_flow_error_set(error, EINVAL,
1499                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1500                                         "mask cannot be zero");
1501
1502         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1503                                         (const uint8_t *)&nic_mask,
1504                                         sizeof(struct rte_flow_item_meta),
1505                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1506         return ret;
1507 }
1508
1509 /**
1510  * Validate TAG item.
1511  *
1512  * @param[in] dev
1513  *   Pointer to the rte_eth_dev structure.
1514  * @param[in] item
1515  *   Item specification.
1516  * @param[in] attr
1517  *   Attributes of flow that includes this item.
1518  * @param[out] error
1519  *   Pointer to error structure.
1520  *
1521  * @return
1522  *   0 on success, a negative errno value otherwise and rte_errno is set.
1523  */
1524 static int
1525 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1526                           const struct rte_flow_item *item,
1527                           const struct rte_flow_attr *attr __rte_unused,
1528                           struct rte_flow_error *error)
1529 {
1530         const struct rte_flow_item_tag *spec = item->spec;
1531         const struct rte_flow_item_tag *mask = item->mask;
1532         const struct rte_flow_item_tag nic_mask = {
1533                 .data = RTE_BE32(UINT32_MAX),
1534                 .index = 0xff,
1535         };
1536         int ret;
1537
1538         if (!mlx5_flow_ext_mreg_supported(dev))
1539                 return rte_flow_error_set(error, ENOTSUP,
1540                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1541                                           "extensive metadata register"
1542                                           " isn't supported");
1543         if (!spec)
1544                 return rte_flow_error_set(error, EINVAL,
1545                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1546                                           item->spec,
1547                                           "data cannot be empty");
1548         if (!mask)
1549                 mask = &rte_flow_item_tag_mask;
1550         if (!mask->data)
1551                 return rte_flow_error_set(error, EINVAL,
1552                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1553                                         "mask cannot be zero");
1554
1555         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1556                                         (const uint8_t *)&nic_mask,
1557                                         sizeof(struct rte_flow_item_tag),
1558                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1559         if (ret < 0)
1560                 return ret;
1561         if (mask->index != 0xff)
1562                 return rte_flow_error_set(error, EINVAL,
1563                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1564                                           "partial mask for tag index"
1565                                           " is not supported");
1566         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1567         if (ret < 0)
1568                 return ret;
1569         MLX5_ASSERT(ret != REG_NON);
1570         return 0;
1571 }
1572
1573 /**
1574  * Validate vport item.
1575  *
1576  * @param[in] dev
1577  *   Pointer to the rte_eth_dev structure.
1578  * @param[in] item
1579  *   Item specification.
1580  * @param[in] attr
1581  *   Attributes of flow that includes this item.
1582  * @param[in] item_flags
1583  *   Bit-fields that holds the items detected until now.
1584  * @param[out] error
1585  *   Pointer to error structure.
1586  *
1587  * @return
1588  *   0 on success, a negative errno value otherwise and rte_errno is set.
1589  */
1590 static int
1591 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1592                               const struct rte_flow_item *item,
1593                               const struct rte_flow_attr *attr,
1594                               uint64_t item_flags,
1595                               struct rte_flow_error *error)
1596 {
1597         const struct rte_flow_item_port_id *spec = item->spec;
1598         const struct rte_flow_item_port_id *mask = item->mask;
1599         const struct rte_flow_item_port_id switch_mask = {
1600                         .id = 0xffffffff,
1601         };
1602         struct mlx5_priv *esw_priv;
1603         struct mlx5_priv *dev_priv;
1604         int ret;
1605
1606         if (!attr->transfer)
1607                 return rte_flow_error_set(error, EINVAL,
1608                                           RTE_FLOW_ERROR_TYPE_ITEM,
1609                                           NULL,
1610                                           "match on port id is valid only"
1611                                           " when transfer flag is enabled");
1612         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1613                 return rte_flow_error_set(error, ENOTSUP,
1614                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1615                                           "multiple source ports are not"
1616                                           " supported");
1617         if (!mask)
1618                 mask = &switch_mask;
1619         if (mask->id != 0xffffffff)
1620                 return rte_flow_error_set(error, ENOTSUP,
1621                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1622                                            mask,
1623                                            "no support for partial mask on"
1624                                            " \"id\" field");
1625         ret = mlx5_flow_item_acceptable
1626                                 (item, (const uint8_t *)mask,
1627                                  (const uint8_t *)&rte_flow_item_port_id_mask,
1628                                  sizeof(struct rte_flow_item_port_id),
1629                                  MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1630         if (ret)
1631                 return ret;
1632         if (!spec)
1633                 return 0;
1634         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1635         if (!esw_priv)
1636                 return rte_flow_error_set(error, rte_errno,
1637                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1638                                           "failed to obtain E-Switch info for"
1639                                           " port");
1640         dev_priv = mlx5_dev_to_eswitch_info(dev);
1641         if (!dev_priv)
1642                 return rte_flow_error_set(error, rte_errno,
1643                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1644                                           NULL,
1645                                           "failed to obtain E-Switch info");
1646         if (esw_priv->domain_id != dev_priv->domain_id)
1647                 return rte_flow_error_set(error, EINVAL,
1648                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1649                                           "cannot match on a port from a"
1650                                           " different E-Switch");
1651         return 0;
1652 }
1653
1654 /**
1655  * Validate VLAN item.
1656  *
1657  * @param[in] item
1658  *   Item specification.
1659  * @param[in] item_flags
1660  *   Bit-fields that holds the items detected until now.
1661  * @param[in] dev
1662  *   Ethernet device flow is being created on.
1663  * @param[out] error
1664  *   Pointer to error structure.
1665  *
1666  * @return
1667  *   0 on success, a negative errno value otherwise and rte_errno is set.
1668  */
1669 static int
1670 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
1671                            uint64_t item_flags,
1672                            struct rte_eth_dev *dev,
1673                            struct rte_flow_error *error)
1674 {
1675         const struct rte_flow_item_vlan *mask = item->mask;
1676         const struct rte_flow_item_vlan nic_mask = {
1677                 .tci = RTE_BE16(UINT16_MAX),
1678                 .inner_type = RTE_BE16(UINT16_MAX),
1679         };
1680         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1681         int ret;
1682         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1683                                         MLX5_FLOW_LAYER_INNER_L4) :
1684                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1685                                         MLX5_FLOW_LAYER_OUTER_L4);
1686         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1687                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1688
1689         if (item_flags & vlanm)
1690                 return rte_flow_error_set(error, EINVAL,
1691                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1692                                           "multiple VLAN layers not supported");
1693         else if ((item_flags & l34m) != 0)
1694                 return rte_flow_error_set(error, EINVAL,
1695                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1696                                           "VLAN cannot follow L3/L4 layer");
1697         if (!mask)
1698                 mask = &rte_flow_item_vlan_mask;
1699         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1700                                         (const uint8_t *)&nic_mask,
1701                                         sizeof(struct rte_flow_item_vlan),
1702                                         MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1703         if (ret)
1704                 return ret;
1705         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1706                 struct mlx5_priv *priv = dev->data->dev_private;
1707
1708                 if (priv->vmwa_context) {
1709                         /*
1710                          * Non-NULL context means we have a virtual machine
1711                          * and SR-IOV enabled, we have to create VLAN interface
1712                          * to make hypervisor to setup E-Switch vport
1713                          * context correctly. We avoid creating the multiple
1714                          * VLAN interfaces, so we cannot support VLAN tag mask.
1715                          */
1716                         return rte_flow_error_set(error, EINVAL,
1717                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1718                                                   item,
1719                                                   "VLAN tag mask is not"
1720                                                   " supported in virtual"
1721                                                   " environment");
1722                 }
1723         }
1724         return 0;
1725 }
1726
1727 /*
1728  * GTP flags are contained in 1 byte of the format:
1729  * -------------------------------------------
1730  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
1731  * |-----------------------------------------|
1732  * | value | Version | PT | Res | E | S | PN |
1733  * -------------------------------------------
1734  *
1735  * Matching is supported only for GTP flags E, S, PN.
1736  */
1737 #define MLX5_GTP_FLAGS_MASK     0x07
1738
1739 /**
1740  * Validate GTP item.
1741  *
1742  * @param[in] dev
1743  *   Pointer to the rte_eth_dev structure.
1744  * @param[in] item
1745  *   Item specification.
1746  * @param[in] item_flags
1747  *   Bit-fields that holds the items detected until now.
1748  * @param[out] error
1749  *   Pointer to error structure.
1750  *
1751  * @return
1752  *   0 on success, a negative errno value otherwise and rte_errno is set.
1753  */
1754 static int
1755 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
1756                           const struct rte_flow_item *item,
1757                           uint64_t item_flags,
1758                           struct rte_flow_error *error)
1759 {
1760         struct mlx5_priv *priv = dev->data->dev_private;
1761         const struct rte_flow_item_gtp *spec = item->spec;
1762         const struct rte_flow_item_gtp *mask = item->mask;
1763         const struct rte_flow_item_gtp nic_mask = {
1764                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
1765                 .msg_type = 0xff,
1766                 .teid = RTE_BE32(0xffffffff),
1767         };
1768
1769         if (!priv->config.hca_attr.tunnel_stateless_gtp)
1770                 return rte_flow_error_set(error, ENOTSUP,
1771                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1772                                           "GTP support is not enabled");
1773         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1774                 return rte_flow_error_set(error, ENOTSUP,
1775                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1776                                           "multiple tunnel layers not"
1777                                           " supported");
1778         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1779                 return rte_flow_error_set(error, EINVAL,
1780                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1781                                           "no outer UDP layer found");
1782         if (!mask)
1783                 mask = &rte_flow_item_gtp_mask;
1784         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
1785                 return rte_flow_error_set(error, ENOTSUP,
1786                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1787                                           "Match is supported for GTP"
1788                                           " flags only");
1789         return mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1790                                          (const uint8_t *)&nic_mask,
1791                                          sizeof(struct rte_flow_item_gtp),
1792                                          MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
1793 }
1794
1795 /**
1796  * Validate IPV4 item.
1797  * Use existing validation function mlx5_flow_validate_item_ipv4(), and
1798  * add specific validation of fragment_offset field,
1799  *
1800  * @param[in] item
1801  *   Item specification.
1802  * @param[in] item_flags
1803  *   Bit-fields that holds the items detected until now.
1804  * @param[out] error
1805  *   Pointer to error structure.
1806  *
1807  * @return
1808  *   0 on success, a negative errno value otherwise and rte_errno is set.
1809  */
1810 static int
1811 flow_dv_validate_item_ipv4(const struct rte_flow_item *item,
1812                            uint64_t item_flags,
1813                            uint64_t last_item,
1814                            uint16_t ether_type,
1815                            struct rte_flow_error *error)
1816 {
1817         int ret;
1818         const struct rte_flow_item_ipv4 *spec = item->spec;
1819         const struct rte_flow_item_ipv4 *last = item->last;
1820         const struct rte_flow_item_ipv4 *mask = item->mask;
1821         rte_be16_t fragment_offset_spec = 0;
1822         rte_be16_t fragment_offset_last = 0;
1823         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
1824                 .hdr = {
1825                         .src_addr = RTE_BE32(0xffffffff),
1826                         .dst_addr = RTE_BE32(0xffffffff),
1827                         .type_of_service = 0xff,
1828                         .fragment_offset = RTE_BE16(0xffff),
1829                         .next_proto_id = 0xff,
1830                         .time_to_live = 0xff,
1831                 },
1832         };
1833
1834         ret = mlx5_flow_validate_item_ipv4(item, item_flags, last_item,
1835                                            ether_type, &nic_ipv4_mask,
1836                                            MLX5_ITEM_RANGE_ACCEPTED, error);
1837         if (ret < 0)
1838                 return ret;
1839         if (spec && mask)
1840                 fragment_offset_spec = spec->hdr.fragment_offset &
1841                                        mask->hdr.fragment_offset;
1842         if (!fragment_offset_spec)
1843                 return 0;
1844         /*
1845          * spec and mask are valid, enforce using full mask to make sure the
1846          * complete value is used correctly.
1847          */
1848         if ((mask->hdr.fragment_offset & RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1849                         != RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1850                 return rte_flow_error_set(error, EINVAL,
1851                                           RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1852                                           item, "must use full mask for"
1853                                           " fragment_offset");
1854         /*
1855          * Match on fragment_offset 0x2000 means MF is 1 and frag-offset is 0,
1856          * indicating this is 1st fragment of fragmented packet.
1857          * This is not yet supported in MLX5, return appropriate error message.
1858          */
1859         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG))
1860                 return rte_flow_error_set(error, ENOTSUP,
1861                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1862                                           "match on first fragment not "
1863                                           "supported");
1864         if (fragment_offset_spec && !last)
1865                 return rte_flow_error_set(error, ENOTSUP,
1866                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1867                                           "specified value not supported");
1868         /* spec and last are valid, validate the specified range. */
1869         fragment_offset_last = last->hdr.fragment_offset &
1870                                mask->hdr.fragment_offset;
1871         /*
1872          * Match on fragment_offset spec 0x2001 and last 0x3fff
1873          * means MF is 1 and frag-offset is > 0.
1874          * This packet is fragment 2nd and onward, excluding last.
1875          * This is not yet supported in MLX5, return appropriate
1876          * error message.
1877          */
1878         if (fragment_offset_spec == RTE_BE16(RTE_IPV4_HDR_MF_FLAG + 1) &&
1879             fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK))
1880                 return rte_flow_error_set(error, ENOTSUP,
1881                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1882                                           last, "match on following "
1883                                           "fragments not supported");
1884         /*
1885          * Match on fragment_offset spec 0x0001 and last 0x1fff
1886          * means MF is 0 and frag-offset is > 0.
1887          * This packet is last fragment of fragmented packet.
1888          * This is not yet supported in MLX5, return appropriate
1889          * error message.
1890          */
1891         if (fragment_offset_spec == RTE_BE16(1) &&
1892             fragment_offset_last == RTE_BE16(RTE_IPV4_HDR_OFFSET_MASK))
1893                 return rte_flow_error_set(error, ENOTSUP,
1894                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST,
1895                                           last, "match on last "
1896                                           "fragment not supported");
1897         /*
1898          * Match on fragment_offset spec 0x0001 and last 0x3fff
1899          * means MF and/or frag-offset is not 0.
1900          * This is a fragmented packet.
1901          * Other range values are invalid and rejected.
1902          */
1903         if (!(fragment_offset_spec == RTE_BE16(1) &&
1904               fragment_offset_last == RTE_BE16(MLX5_IPV4_FRAG_OFFSET_MASK)))
1905                 return rte_flow_error_set(error, ENOTSUP,
1906                                           RTE_FLOW_ERROR_TYPE_ITEM_LAST, last,
1907                                           "specified range not supported");
1908         return 0;
1909 }
1910
1911 /**
1912  * Validate the pop VLAN action.
1913  *
1914  * @param[in] dev
1915  *   Pointer to the rte_eth_dev structure.
1916  * @param[in] action_flags
1917  *   Holds the actions detected until now.
1918  * @param[in] action
1919  *   Pointer to the pop vlan action.
1920  * @param[in] item_flags
1921  *   The items found in this flow rule.
1922  * @param[in] attr
1923  *   Pointer to flow attributes.
1924  * @param[out] error
1925  *   Pointer to error structure.
1926  *
1927  * @return
1928  *   0 on success, a negative errno value otherwise and rte_errno is set.
1929  */
1930 static int
1931 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
1932                                  uint64_t action_flags,
1933                                  const struct rte_flow_action *action,
1934                                  uint64_t item_flags,
1935                                  const struct rte_flow_attr *attr,
1936                                  struct rte_flow_error *error)
1937 {
1938         const struct mlx5_priv *priv = dev->data->dev_private;
1939
1940         (void)action;
1941         (void)attr;
1942         if (!priv->sh->pop_vlan_action)
1943                 return rte_flow_error_set(error, ENOTSUP,
1944                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1945                                           NULL,
1946                                           "pop vlan action is not supported");
1947         if (attr->egress)
1948                 return rte_flow_error_set(error, ENOTSUP,
1949                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1950                                           NULL,
1951                                           "pop vlan action not supported for "
1952                                           "egress");
1953         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
1954                 return rte_flow_error_set(error, ENOTSUP,
1955                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1956                                           "no support for multiple VLAN "
1957                                           "actions");
1958         /* Pop VLAN with preceding Decap requires inner header with VLAN. */
1959         if ((action_flags & MLX5_FLOW_ACTION_DECAP) &&
1960             !(item_flags & MLX5_FLOW_LAYER_INNER_VLAN))
1961                 return rte_flow_error_set(error, ENOTSUP,
1962                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1963                                           NULL,
1964                                           "cannot pop vlan after decap without "
1965                                           "match on inner vlan in the flow");
1966         /* Pop VLAN without preceding Decap requires outer header with VLAN. */
1967         if (!(action_flags & MLX5_FLOW_ACTION_DECAP) &&
1968             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
1969                 return rte_flow_error_set(error, ENOTSUP,
1970                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1971                                           NULL,
1972                                           "cannot pop vlan without a "
1973                                           "match on (outer) vlan in the flow");
1974         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1975                 return rte_flow_error_set(error, EINVAL,
1976                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1977                                           "wrong action order, port_id should "
1978                                           "be after pop VLAN action");
1979         if (!attr->transfer && priv->representor)
1980                 return rte_flow_error_set(error, ENOTSUP,
1981                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1982                                           "pop vlan action for VF representor "
1983                                           "not supported on NIC table");
1984         return 0;
1985 }
1986
1987 /**
1988  * Get VLAN default info from vlan match info.
1989  *
1990  * @param[in] items
1991  *   the list of item specifications.
1992  * @param[out] vlan
1993  *   pointer VLAN info to fill to.
1994  *
1995  * @return
1996  *   0 on success, a negative errno value otherwise and rte_errno is set.
1997  */
1998 static void
1999 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
2000                                   struct rte_vlan_hdr *vlan)
2001 {
2002         const struct rte_flow_item_vlan nic_mask = {
2003                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
2004                                 MLX5DV_FLOW_VLAN_VID_MASK),
2005                 .inner_type = RTE_BE16(0xffff),
2006         };
2007
2008         if (items == NULL)
2009                 return;
2010         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2011                 int type = items->type;
2012
2013                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
2014                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
2015                         break;
2016         }
2017         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
2018                 const struct rte_flow_item_vlan *vlan_m = items->mask;
2019                 const struct rte_flow_item_vlan *vlan_v = items->spec;
2020
2021                 /* If VLAN item in pattern doesn't contain data, return here. */
2022                 if (!vlan_v)
2023                         return;
2024                 if (!vlan_m)
2025                         vlan_m = &nic_mask;
2026                 /* Only full match values are accepted */
2027                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
2028                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
2029                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
2030                         vlan->vlan_tci |=
2031                                 rte_be_to_cpu_16(vlan_v->tci &
2032                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
2033                 }
2034                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
2035                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
2036                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
2037                         vlan->vlan_tci |=
2038                                 rte_be_to_cpu_16(vlan_v->tci &
2039                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
2040                 }
2041                 if (vlan_m->inner_type == nic_mask.inner_type)
2042                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
2043                                                            vlan_m->inner_type);
2044         }
2045 }
2046
2047 /**
2048  * Validate the push VLAN action.
2049  *
2050  * @param[in] dev
2051  *   Pointer to the rte_eth_dev structure.
2052  * @param[in] action_flags
2053  *   Holds the actions detected until now.
2054  * @param[in] item_flags
2055  *   The items found in this flow rule.
2056  * @param[in] action
2057  *   Pointer to the action structure.
2058  * @param[in] attr
2059  *   Pointer to flow attributes
2060  * @param[out] error
2061  *   Pointer to error structure.
2062  *
2063  * @return
2064  *   0 on success, a negative errno value otherwise and rte_errno is set.
2065  */
2066 static int
2067 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
2068                                   uint64_t action_flags,
2069                                   const struct rte_flow_item_vlan *vlan_m,
2070                                   const struct rte_flow_action *action,
2071                                   const struct rte_flow_attr *attr,
2072                                   struct rte_flow_error *error)
2073 {
2074         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
2075         const struct mlx5_priv *priv = dev->data->dev_private;
2076
2077         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
2078             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
2079                 return rte_flow_error_set(error, EINVAL,
2080                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2081                                           "invalid vlan ethertype");
2082         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2083                 return rte_flow_error_set(error, EINVAL,
2084                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2085                                           "wrong action order, port_id should "
2086                                           "be after push VLAN");
2087         if (!attr->transfer && priv->representor)
2088                 return rte_flow_error_set(error, ENOTSUP,
2089                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2090                                           "push vlan action for VF representor "
2091                                           "not supported on NIC table");
2092         if (vlan_m &&
2093             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
2094             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
2095                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
2096             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
2097             !(mlx5_flow_find_action
2098                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
2099                 return rte_flow_error_set(error, EINVAL,
2100                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2101                                           "not full match mask on VLAN PCP and "
2102                                           "there is no of_set_vlan_pcp action, "
2103                                           "push VLAN action cannot figure out "
2104                                           "PCP value");
2105         if (vlan_m &&
2106             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
2107             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
2108                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
2109             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
2110             !(mlx5_flow_find_action
2111                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
2112                 return rte_flow_error_set(error, EINVAL,
2113                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2114                                           "not full match mask on VLAN VID and "
2115                                           "there is no of_set_vlan_vid action, "
2116                                           "push VLAN action cannot figure out "
2117                                           "VID value");
2118         (void)attr;
2119         return 0;
2120 }
2121
2122 /**
2123  * Validate the set VLAN PCP.
2124  *
2125  * @param[in] action_flags
2126  *   Holds the actions detected until now.
2127  * @param[in] actions
2128  *   Pointer to the list of actions remaining in the flow rule.
2129  * @param[out] error
2130  *   Pointer to error structure.
2131  *
2132  * @return
2133  *   0 on success, a negative errno value otherwise and rte_errno is set.
2134  */
2135 static int
2136 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2137                                      const struct rte_flow_action actions[],
2138                                      struct rte_flow_error *error)
2139 {
2140         const struct rte_flow_action *action = actions;
2141         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2142
2143         if (conf->vlan_pcp > 7)
2144                 return rte_flow_error_set(error, EINVAL,
2145                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2146                                           "VLAN PCP value is too big");
2147         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2148                 return rte_flow_error_set(error, ENOTSUP,
2149                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2150                                           "set VLAN PCP action must follow "
2151                                           "the push VLAN action");
2152         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2153                 return rte_flow_error_set(error, ENOTSUP,
2154                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2155                                           "Multiple VLAN PCP modification are "
2156                                           "not supported");
2157         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2158                 return rte_flow_error_set(error, EINVAL,
2159                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2160                                           "wrong action order, port_id should "
2161                                           "be after set VLAN PCP");
2162         return 0;
2163 }
2164
2165 /**
2166  * Validate the set VLAN VID.
2167  *
2168  * @param[in] item_flags
2169  *   Holds the items detected in this rule.
2170  * @param[in] action_flags
2171  *   Holds the actions detected until now.
2172  * @param[in] actions
2173  *   Pointer to the list of actions remaining in the flow rule.
2174  * @param[out] error
2175  *   Pointer to error structure.
2176  *
2177  * @return
2178  *   0 on success, a negative errno value otherwise and rte_errno is set.
2179  */
2180 static int
2181 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2182                                      uint64_t action_flags,
2183                                      const struct rte_flow_action actions[],
2184                                      struct rte_flow_error *error)
2185 {
2186         const struct rte_flow_action *action = actions;
2187         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2188
2189         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2190                 return rte_flow_error_set(error, EINVAL,
2191                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2192                                           "VLAN VID value is too big");
2193         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2194             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2195                 return rte_flow_error_set(error, ENOTSUP,
2196                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2197                                           "set VLAN VID action must follow push"
2198                                           " VLAN action or match on VLAN item");
2199         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2200                 return rte_flow_error_set(error, ENOTSUP,
2201                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2202                                           "Multiple VLAN VID modifications are "
2203                                           "not supported");
2204         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2205                 return rte_flow_error_set(error, EINVAL,
2206                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2207                                           "wrong action order, port_id should "
2208                                           "be after set VLAN VID");
2209         return 0;
2210 }
2211
2212 /*
2213  * Validate the FLAG action.
2214  *
2215  * @param[in] dev
2216  *   Pointer to the rte_eth_dev structure.
2217  * @param[in] action_flags
2218  *   Holds the actions detected until now.
2219  * @param[in] attr
2220  *   Pointer to flow attributes
2221  * @param[out] error
2222  *   Pointer to error structure.
2223  *
2224  * @return
2225  *   0 on success, a negative errno value otherwise and rte_errno is set.
2226  */
2227 static int
2228 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
2229                              uint64_t action_flags,
2230                              const struct rte_flow_attr *attr,
2231                              struct rte_flow_error *error)
2232 {
2233         struct mlx5_priv *priv = dev->data->dev_private;
2234         struct mlx5_dev_config *config = &priv->config;
2235         int ret;
2236
2237         /* Fall back if no extended metadata register support. */
2238         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2239                 return mlx5_flow_validate_action_flag(action_flags, attr,
2240                                                       error);
2241         /* Extensive metadata mode requires registers. */
2242         if (!mlx5_flow_ext_mreg_supported(dev))
2243                 return rte_flow_error_set(error, ENOTSUP,
2244                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2245                                           "no metadata registers "
2246                                           "to support flag action");
2247         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
2248                 return rte_flow_error_set(error, ENOTSUP,
2249                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2250                                           "extended metadata register"
2251                                           " isn't available");
2252         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2253         if (ret < 0)
2254                 return ret;
2255         MLX5_ASSERT(ret > 0);
2256         if (action_flags & MLX5_FLOW_ACTION_MARK)
2257                 return rte_flow_error_set(error, EINVAL,
2258                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2259                                           "can't mark and flag in same flow");
2260         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2261                 return rte_flow_error_set(error, EINVAL,
2262                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2263                                           "can't have 2 flag"
2264                                           " actions in same flow");
2265         return 0;
2266 }
2267
2268 /**
2269  * Validate MARK action.
2270  *
2271  * @param[in] dev
2272  *   Pointer to the rte_eth_dev structure.
2273  * @param[in] action
2274  *   Pointer to action.
2275  * @param[in] action_flags
2276  *   Holds the actions detected until now.
2277  * @param[in] attr
2278  *   Pointer to flow attributes
2279  * @param[out] error
2280  *   Pointer to error structure.
2281  *
2282  * @return
2283  *   0 on success, a negative errno value otherwise and rte_errno is set.
2284  */
2285 static int
2286 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
2287                              const struct rte_flow_action *action,
2288                              uint64_t action_flags,
2289                              const struct rte_flow_attr *attr,
2290                              struct rte_flow_error *error)
2291 {
2292         struct mlx5_priv *priv = dev->data->dev_private;
2293         struct mlx5_dev_config *config = &priv->config;
2294         const struct rte_flow_action_mark *mark = action->conf;
2295         int ret;
2296
2297         /* Fall back if no extended metadata register support. */
2298         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2299                 return mlx5_flow_validate_action_mark(action, action_flags,
2300                                                       attr, error);
2301         /* Extensive metadata mode requires registers. */
2302         if (!mlx5_flow_ext_mreg_supported(dev))
2303                 return rte_flow_error_set(error, ENOTSUP,
2304                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2305                                           "no metadata registers "
2306                                           "to support mark action");
2307         if (!priv->sh->dv_mark_mask)
2308                 return rte_flow_error_set(error, ENOTSUP,
2309                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2310                                           "extended metadata register"
2311                                           " isn't available");
2312         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2313         if (ret < 0)
2314                 return ret;
2315         MLX5_ASSERT(ret > 0);
2316         if (!mark)
2317                 return rte_flow_error_set(error, EINVAL,
2318                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2319                                           "configuration cannot be null");
2320         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
2321                 return rte_flow_error_set(error, EINVAL,
2322                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2323                                           &mark->id,
2324                                           "mark id exceeds the limit");
2325         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2326                 return rte_flow_error_set(error, EINVAL,
2327                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2328                                           "can't flag and mark in same flow");
2329         if (action_flags & MLX5_FLOW_ACTION_MARK)
2330                 return rte_flow_error_set(error, EINVAL,
2331                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2332                                           "can't have 2 mark actions in same"
2333                                           " flow");
2334         return 0;
2335 }
2336
2337 /**
2338  * Validate SET_META action.
2339  *
2340  * @param[in] dev
2341  *   Pointer to the rte_eth_dev structure.
2342  * @param[in] action
2343  *   Pointer to the action structure.
2344  * @param[in] action_flags
2345  *   Holds the actions detected until now.
2346  * @param[in] attr
2347  *   Pointer to flow attributes
2348  * @param[out] error
2349  *   Pointer to error structure.
2350  *
2351  * @return
2352  *   0 on success, a negative errno value otherwise and rte_errno is set.
2353  */
2354 static int
2355 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
2356                                  const struct rte_flow_action *action,
2357                                  uint64_t action_flags __rte_unused,
2358                                  const struct rte_flow_attr *attr,
2359                                  struct rte_flow_error *error)
2360 {
2361         const struct rte_flow_action_set_meta *conf;
2362         uint32_t nic_mask = UINT32_MAX;
2363         int reg;
2364
2365         if (!mlx5_flow_ext_mreg_supported(dev))
2366                 return rte_flow_error_set(error, ENOTSUP,
2367                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2368                                           "extended metadata register"
2369                                           " isn't supported");
2370         reg = flow_dv_get_metadata_reg(dev, attr, error);
2371         if (reg < 0)
2372                 return reg;
2373         if (reg != REG_A && reg != REG_B) {
2374                 struct mlx5_priv *priv = dev->data->dev_private;
2375
2376                 nic_mask = priv->sh->dv_meta_mask;
2377         }
2378         if (!(action->conf))
2379                 return rte_flow_error_set(error, EINVAL,
2380                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2381                                           "configuration cannot be null");
2382         conf = (const struct rte_flow_action_set_meta *)action->conf;
2383         if (!conf->mask)
2384                 return rte_flow_error_set(error, EINVAL,
2385                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2386                                           "zero mask doesn't have any effect");
2387         if (conf->mask & ~nic_mask)
2388                 return rte_flow_error_set(error, EINVAL,
2389                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2390                                           "meta data must be within reg C0");
2391         return 0;
2392 }
2393
2394 /**
2395  * Validate SET_TAG action.
2396  *
2397  * @param[in] dev
2398  *   Pointer to the rte_eth_dev structure.
2399  * @param[in] action
2400  *   Pointer to the action structure.
2401  * @param[in] action_flags
2402  *   Holds the actions detected until now.
2403  * @param[in] attr
2404  *   Pointer to flow attributes
2405  * @param[out] error
2406  *   Pointer to error structure.
2407  *
2408  * @return
2409  *   0 on success, a negative errno value otherwise and rte_errno is set.
2410  */
2411 static int
2412 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
2413                                 const struct rte_flow_action *action,
2414                                 uint64_t action_flags,
2415                                 const struct rte_flow_attr *attr,
2416                                 struct rte_flow_error *error)
2417 {
2418         const struct rte_flow_action_set_tag *conf;
2419         const uint64_t terminal_action_flags =
2420                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
2421                 MLX5_FLOW_ACTION_RSS;
2422         int ret;
2423
2424         if (!mlx5_flow_ext_mreg_supported(dev))
2425                 return rte_flow_error_set(error, ENOTSUP,
2426                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2427                                           "extensive metadata register"
2428                                           " isn't supported");
2429         if (!(action->conf))
2430                 return rte_flow_error_set(error, EINVAL,
2431                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2432                                           "configuration cannot be null");
2433         conf = (const struct rte_flow_action_set_tag *)action->conf;
2434         if (!conf->mask)
2435                 return rte_flow_error_set(error, EINVAL,
2436                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2437                                           "zero mask doesn't have any effect");
2438         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
2439         if (ret < 0)
2440                 return ret;
2441         if (!attr->transfer && attr->ingress &&
2442             (action_flags & terminal_action_flags))
2443                 return rte_flow_error_set(error, EINVAL,
2444                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2445                                           "set_tag has no effect"
2446                                           " with terminal actions");
2447         return 0;
2448 }
2449
2450 /**
2451  * Validate count action.
2452  *
2453  * @param[in] dev
2454  *   Pointer to rte_eth_dev structure.
2455  * @param[out] error
2456  *   Pointer to error structure.
2457  *
2458  * @return
2459  *   0 on success, a negative errno value otherwise and rte_errno is set.
2460  */
2461 static int
2462 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2463                               struct rte_flow_error *error)
2464 {
2465         struct mlx5_priv *priv = dev->data->dev_private;
2466
2467         if (!priv->config.devx)
2468                 goto notsup_err;
2469 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2470         return 0;
2471 #endif
2472 notsup_err:
2473         return rte_flow_error_set
2474                       (error, ENOTSUP,
2475                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2476                        NULL,
2477                        "count action not supported");
2478 }
2479
2480 /**
2481  * Validate the L2 encap action.
2482  *
2483  * @param[in] dev
2484  *   Pointer to the rte_eth_dev structure.
2485  * @param[in] action_flags
2486  *   Holds the actions detected until now.
2487  * @param[in] action
2488  *   Pointer to the action structure.
2489  * @param[in] attr
2490  *   Pointer to flow attributes.
2491  * @param[out] error
2492  *   Pointer to error structure.
2493  *
2494  * @return
2495  *   0 on success, a negative errno value otherwise and rte_errno is set.
2496  */
2497 static int
2498 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
2499                                  uint64_t action_flags,
2500                                  const struct rte_flow_action *action,
2501                                  const struct rte_flow_attr *attr,
2502                                  struct rte_flow_error *error)
2503 {
2504         const struct mlx5_priv *priv = dev->data->dev_private;
2505
2506         if (!(action->conf))
2507                 return rte_flow_error_set(error, EINVAL,
2508                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2509                                           "configuration cannot be null");
2510         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
2511                 return rte_flow_error_set(error, EINVAL,
2512                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2513                                           "can only have a single encap action "
2514                                           "in a flow");
2515         if (!attr->transfer && priv->representor)
2516                 return rte_flow_error_set(error, ENOTSUP,
2517                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2518                                           "encap action for VF representor "
2519                                           "not supported on NIC table");
2520         return 0;
2521 }
2522
2523 /**
2524  * Validate a decap action.
2525  *
2526  * @param[in] dev
2527  *   Pointer to the rte_eth_dev structure.
2528  * @param[in] action_flags
2529  *   Holds the actions detected until now.
2530  * @param[in] attr
2531  *   Pointer to flow attributes
2532  * @param[out] error
2533  *   Pointer to error structure.
2534  *
2535  * @return
2536  *   0 on success, a negative errno value otherwise and rte_errno is set.
2537  */
2538 static int
2539 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
2540                               uint64_t action_flags,
2541                               const struct rte_flow_attr *attr,
2542                               struct rte_flow_error *error)
2543 {
2544         const struct mlx5_priv *priv = dev->data->dev_private;
2545
2546         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
2547             !priv->config.decap_en)
2548                 return rte_flow_error_set(error, ENOTSUP,
2549                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2550                                           "decap is not enabled");
2551         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
2552                 return rte_flow_error_set(error, ENOTSUP,
2553                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2554                                           action_flags &
2555                                           MLX5_FLOW_ACTION_DECAP ? "can only "
2556                                           "have a single decap action" : "decap "
2557                                           "after encap is not supported");
2558         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2559                 return rte_flow_error_set(error, EINVAL,
2560                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2561                                           "can't have decap action after"
2562                                           " modify action");
2563         if (attr->egress)
2564                 return rte_flow_error_set(error, ENOTSUP,
2565                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2566                                           NULL,
2567                                           "decap action not supported for "
2568                                           "egress");
2569         if (!attr->transfer && priv->representor)
2570                 return rte_flow_error_set(error, ENOTSUP,
2571                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2572                                           "decap action for VF representor "
2573                                           "not supported on NIC table");
2574         return 0;
2575 }
2576
2577 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
2578
2579 /**
2580  * Validate the raw encap and decap actions.
2581  *
2582  * @param[in] dev
2583  *   Pointer to the rte_eth_dev structure.
2584  * @param[in] decap
2585  *   Pointer to the decap action.
2586  * @param[in] encap
2587  *   Pointer to the encap action.
2588  * @param[in] attr
2589  *   Pointer to flow attributes
2590  * @param[in/out] action_flags
2591  *   Holds the actions detected until now.
2592  * @param[out] actions_n
2593  *   pointer to the number of actions counter.
2594  * @param[out] error
2595  *   Pointer to error structure.
2596  *
2597  * @return
2598  *   0 on success, a negative errno value otherwise and rte_errno is set.
2599  */
2600 static int
2601 flow_dv_validate_action_raw_encap_decap
2602         (struct rte_eth_dev *dev,
2603          const struct rte_flow_action_raw_decap *decap,
2604          const struct rte_flow_action_raw_encap *encap,
2605          const struct rte_flow_attr *attr, uint64_t *action_flags,
2606          int *actions_n, struct rte_flow_error *error)
2607 {
2608         const struct mlx5_priv *priv = dev->data->dev_private;
2609         int ret;
2610
2611         if (encap && (!encap->size || !encap->data))
2612                 return rte_flow_error_set(error, EINVAL,
2613                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2614                                           "raw encap data cannot be empty");
2615         if (decap && encap) {
2616                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
2617                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2618                         /* L3 encap. */
2619                         decap = NULL;
2620                 else if (encap->size <=
2621                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2622                            decap->size >
2623                            MLX5_ENCAPSULATION_DECISION_SIZE)
2624                         /* L3 decap. */
2625                         encap = NULL;
2626                 else if (encap->size >
2627                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2628                            decap->size >
2629                            MLX5_ENCAPSULATION_DECISION_SIZE)
2630                         /* 2 L2 actions: encap and decap. */
2631                         ;
2632                 else
2633                         return rte_flow_error_set(error,
2634                                 ENOTSUP,
2635                                 RTE_FLOW_ERROR_TYPE_ACTION,
2636                                 NULL, "unsupported too small "
2637                                 "raw decap and too small raw "
2638                                 "encap combination");
2639         }
2640         if (decap) {
2641                 ret = flow_dv_validate_action_decap(dev, *action_flags, attr,
2642                                                     error);
2643                 if (ret < 0)
2644                         return ret;
2645                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
2646                 ++(*actions_n);
2647         }
2648         if (encap) {
2649                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
2650                         return rte_flow_error_set(error, ENOTSUP,
2651                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2652                                                   NULL,
2653                                                   "small raw encap size");
2654                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
2655                         return rte_flow_error_set(error, EINVAL,
2656                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2657                                                   NULL,
2658                                                   "more than one encap action");
2659                 if (!attr->transfer && priv->representor)
2660                         return rte_flow_error_set
2661                                         (error, ENOTSUP,
2662                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2663                                          "encap action for VF representor "
2664                                          "not supported on NIC table");
2665                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
2666                 ++(*actions_n);
2667         }
2668         return 0;
2669 }
2670
2671 /**
2672  * Match encap_decap resource.
2673  *
2674  * @param entry
2675  *   Pointer to exist resource entry object.
2676  * @param ctx
2677  *   Pointer to new encap_decap resource.
2678  *
2679  * @return
2680  *   0 on matching, -1 otherwise.
2681  */
2682 static int
2683 flow_dv_encap_decap_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
2684 {
2685         struct mlx5_flow_dv_encap_decap_resource *resource;
2686         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2687
2688         resource = (struct mlx5_flow_dv_encap_decap_resource *)ctx;
2689         cache_resource = container_of(entry,
2690                                       struct mlx5_flow_dv_encap_decap_resource,
2691                                       entry);
2692         if (resource->entry.key == cache_resource->entry.key &&
2693             resource->reformat_type == cache_resource->reformat_type &&
2694             resource->ft_type == cache_resource->ft_type &&
2695             resource->flags == cache_resource->flags &&
2696             resource->size == cache_resource->size &&
2697             !memcmp((const void *)resource->buf,
2698                     (const void *)cache_resource->buf,
2699                     resource->size))
2700                 return 0;
2701         return -1;
2702 }
2703
2704 /**
2705  * Find existing encap/decap resource or create and register a new one.
2706  *
2707  * @param[in, out] dev
2708  *   Pointer to rte_eth_dev structure.
2709  * @param[in, out] resource
2710  *   Pointer to encap/decap resource.
2711  * @parm[in, out] dev_flow
2712  *   Pointer to the dev_flow.
2713  * @param[out] error
2714  *   pointer to error structure.
2715  *
2716  * @return
2717  *   0 on success otherwise -errno and errno is set.
2718  */
2719 static int
2720 flow_dv_encap_decap_resource_register
2721                         (struct rte_eth_dev *dev,
2722                          struct mlx5_flow_dv_encap_decap_resource *resource,
2723                          struct mlx5_flow *dev_flow,
2724                          struct rte_flow_error *error)
2725 {
2726         struct mlx5_priv *priv = dev->data->dev_private;
2727         struct mlx5_dev_ctx_shared *sh = priv->sh;
2728         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2729         struct mlx5dv_dr_domain *domain;
2730         struct mlx5_hlist_entry *entry;
2731         union mlx5_flow_encap_decap_key encap_decap_key = {
2732                 {
2733                         .ft_type = resource->ft_type,
2734                         .refmt_type = resource->reformat_type,
2735                         .buf_size = resource->size,
2736                         .table_level = !!dev_flow->dv.group,
2737                         .cksum = 0,
2738                 }
2739         };
2740         int ret;
2741
2742         resource->flags = dev_flow->dv.group ? 0 : 1;
2743         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2744                 domain = sh->fdb_domain;
2745         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2746                 domain = sh->rx_domain;
2747         else
2748                 domain = sh->tx_domain;
2749         encap_decap_key.cksum = __rte_raw_cksum(resource->buf,
2750                                                 resource->size, 0);
2751         resource->entry.key = encap_decap_key.v64;
2752         /* Lookup a matching resource from cache. */
2753         entry = mlx5_hlist_lookup_ex(sh->encaps_decaps, resource->entry.key,
2754                                      flow_dv_encap_decap_resource_match,
2755                                      (void *)resource);
2756         if (entry) {
2757                 cache_resource = container_of(entry,
2758                         struct mlx5_flow_dv_encap_decap_resource, entry);
2759                 DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2760                         (void *)cache_resource,
2761                         rte_atomic32_read(&cache_resource->refcnt));
2762                 rte_atomic32_inc(&cache_resource->refcnt);
2763                 dev_flow->handle->dvh.rix_encap_decap = cache_resource->idx;
2764                 dev_flow->dv.encap_decap = cache_resource;
2765                 return 0;
2766         }
2767         /* Register new encap/decap resource. */
2768         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2769                                        &dev_flow->handle->dvh.rix_encap_decap);
2770         if (!cache_resource)
2771                 return rte_flow_error_set(error, ENOMEM,
2772                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2773                                           "cannot allocate resource memory");
2774         *cache_resource = *resource;
2775         cache_resource->idx = dev_flow->handle->dvh.rix_encap_decap;
2776         ret = mlx5_flow_os_create_flow_action_packet_reformat
2777                                         (sh->ctx, domain, cache_resource,
2778                                          &cache_resource->action);
2779         if (ret) {
2780                 mlx5_free(cache_resource);
2781                 return rte_flow_error_set(error, ENOMEM,
2782                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2783                                           NULL, "cannot create action");
2784         }
2785         rte_atomic32_init(&cache_resource->refcnt);
2786         rte_atomic32_inc(&cache_resource->refcnt);
2787         if (mlx5_hlist_insert_ex(sh->encaps_decaps, &cache_resource->entry,
2788                                  flow_dv_encap_decap_resource_match,
2789                                  (void *)cache_resource)) {
2790                 claim_zero(mlx5_flow_os_destroy_flow_action
2791                                                 (cache_resource->action));
2792                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2793                                 cache_resource->idx);
2794                 return rte_flow_error_set(error, EEXIST,
2795                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2796                                           NULL, "action exist");
2797         }
2798         dev_flow->dv.encap_decap = cache_resource;
2799         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2800                 (void *)cache_resource,
2801                 rte_atomic32_read(&cache_resource->refcnt));
2802         return 0;
2803 }
2804
2805 /**
2806  * Find existing table jump resource or create and register a new one.
2807  *
2808  * @param[in, out] dev
2809  *   Pointer to rte_eth_dev structure.
2810  * @param[in, out] tbl
2811  *   Pointer to flow table resource.
2812  * @parm[in, out] dev_flow
2813  *   Pointer to the dev_flow.
2814  * @param[out] error
2815  *   pointer to error structure.
2816  *
2817  * @return
2818  *   0 on success otherwise -errno and errno is set.
2819  */
2820 static int
2821 flow_dv_jump_tbl_resource_register
2822                         (struct rte_eth_dev *dev __rte_unused,
2823                          struct mlx5_flow_tbl_resource *tbl,
2824                          struct mlx5_flow *dev_flow,
2825                          struct rte_flow_error *error)
2826 {
2827         struct mlx5_flow_tbl_data_entry *tbl_data =
2828                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2829         int cnt, ret;
2830
2831         MLX5_ASSERT(tbl);
2832         cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
2833         if (!cnt) {
2834                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
2835                                 (tbl->obj, &tbl_data->jump.action);
2836                 if (ret)
2837                         return rte_flow_error_set(error, ENOMEM,
2838                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2839                                         NULL, "cannot create jump action");
2840                 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2841                         (void *)&tbl_data->jump, cnt);
2842         } else {
2843                 /* old jump should not make the table ref++. */
2844                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
2845                 MLX5_ASSERT(tbl_data->jump.action);
2846                 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2847                         (void *)&tbl_data->jump, cnt);
2848         }
2849         rte_atomic32_inc(&tbl_data->jump.refcnt);
2850         dev_flow->handle->rix_jump = tbl_data->idx;
2851         dev_flow->dv.jump = &tbl_data->jump;
2852         return 0;
2853 }
2854
2855 /**
2856  * Find existing default miss resource or create and register a new one.
2857  *
2858  * @param[in, out] dev
2859  *   Pointer to rte_eth_dev structure.
2860  * @param[out] error
2861  *   pointer to error structure.
2862  *
2863  * @return
2864  *   0 on success otherwise -errno and errno is set.
2865  */
2866 static int
2867 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
2868                 struct rte_flow_error *error)
2869 {
2870         struct mlx5_priv *priv = dev->data->dev_private;
2871         struct mlx5_dev_ctx_shared *sh = priv->sh;
2872         struct mlx5_flow_default_miss_resource *cache_resource =
2873                         &sh->default_miss;
2874         int cnt = rte_atomic32_read(&cache_resource->refcnt);
2875
2876         if (!cnt) {
2877                 MLX5_ASSERT(cache_resource->action);
2878                 cache_resource->action =
2879                 mlx5_glue->dr_create_flow_action_default_miss();
2880                 if (!cache_resource->action)
2881                         return rte_flow_error_set(error, ENOMEM,
2882                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2883                                         "cannot create default miss action");
2884                 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++",
2885                                 (void *)cache_resource->action, cnt);
2886         }
2887         rte_atomic32_inc(&cache_resource->refcnt);
2888         return 0;
2889 }
2890
2891 /**
2892  * Find existing table port ID resource or create and register a new one.
2893  *
2894  * @param[in, out] dev
2895  *   Pointer to rte_eth_dev structure.
2896  * @param[in, out] resource
2897  *   Pointer to port ID action resource.
2898  * @parm[in, out] dev_flow
2899  *   Pointer to the dev_flow.
2900  * @param[out] error
2901  *   pointer to error structure.
2902  *
2903  * @return
2904  *   0 on success otherwise -errno and errno is set.
2905  */
2906 static int
2907 flow_dv_port_id_action_resource_register
2908                         (struct rte_eth_dev *dev,
2909                          struct mlx5_flow_dv_port_id_action_resource *resource,
2910                          struct mlx5_flow *dev_flow,
2911                          struct rte_flow_error *error)
2912 {
2913         struct mlx5_priv *priv = dev->data->dev_private;
2914         struct mlx5_dev_ctx_shared *sh = priv->sh;
2915         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
2916         uint32_t idx = 0;
2917         int ret;
2918
2919         /* Lookup a matching resource from cache. */
2920         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list,
2921                       idx, cache_resource, next) {
2922                 if (resource->port_id == cache_resource->port_id) {
2923                         DRV_LOG(DEBUG, "port id action resource resource %p: "
2924                                 "refcnt %d++",
2925                                 (void *)cache_resource,
2926                                 rte_atomic32_read(&cache_resource->refcnt));
2927                         rte_atomic32_inc(&cache_resource->refcnt);
2928                         dev_flow->handle->rix_port_id_action = idx;
2929                         dev_flow->dv.port_id_action = cache_resource;
2930                         return 0;
2931                 }
2932         }
2933         /* Register new port id action resource. */
2934         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID],
2935                                        &dev_flow->handle->rix_port_id_action);
2936         if (!cache_resource)
2937                 return rte_flow_error_set(error, ENOMEM,
2938                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2939                                           "cannot allocate resource memory");
2940         *cache_resource = *resource;
2941         ret = mlx5_flow_os_create_flow_action_dest_port
2942                                 (priv->sh->fdb_domain, resource->port_id,
2943                                  &cache_resource->action);
2944         if (ret) {
2945                 mlx5_free(cache_resource);
2946                 return rte_flow_error_set(error, ENOMEM,
2947                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2948                                           NULL, "cannot create action");
2949         }
2950         rte_atomic32_init(&cache_resource->refcnt);
2951         rte_atomic32_inc(&cache_resource->refcnt);
2952         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
2953                      dev_flow->handle->rix_port_id_action, cache_resource,
2954                      next);
2955         dev_flow->dv.port_id_action = cache_resource;
2956         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
2957                 (void *)cache_resource,
2958                 rte_atomic32_read(&cache_resource->refcnt));
2959         return 0;
2960 }
2961
2962 /**
2963  * Find existing push vlan resource or create and register a new one.
2964  *
2965  * @param [in, out] dev
2966  *   Pointer to rte_eth_dev structure.
2967  * @param[in, out] resource
2968  *   Pointer to port ID action resource.
2969  * @parm[in, out] dev_flow
2970  *   Pointer to the dev_flow.
2971  * @param[out] error
2972  *   pointer to error structure.
2973  *
2974  * @return
2975  *   0 on success otherwise -errno and errno is set.
2976  */
2977 static int
2978 flow_dv_push_vlan_action_resource_register
2979                        (struct rte_eth_dev *dev,
2980                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
2981                         struct mlx5_flow *dev_flow,
2982                         struct rte_flow_error *error)
2983 {
2984         struct mlx5_priv *priv = dev->data->dev_private;
2985         struct mlx5_dev_ctx_shared *sh = priv->sh;
2986         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
2987         struct mlx5dv_dr_domain *domain;
2988         uint32_t idx = 0;
2989         int ret;
2990
2991         /* Lookup a matching resource from cache. */
2992         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2993                       sh->push_vlan_action_list, idx, cache_resource, next) {
2994                 if (resource->vlan_tag == cache_resource->vlan_tag &&
2995                     resource->ft_type == cache_resource->ft_type) {
2996                         DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
2997                                 "refcnt %d++",
2998                                 (void *)cache_resource,
2999                                 rte_atomic32_read(&cache_resource->refcnt));
3000                         rte_atomic32_inc(&cache_resource->refcnt);
3001                         dev_flow->handle->dvh.rix_push_vlan = idx;
3002                         dev_flow->dv.push_vlan_res = cache_resource;
3003                         return 0;
3004                 }
3005         }
3006         /* Register new push_vlan action resource. */
3007         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3008                                        &dev_flow->handle->dvh.rix_push_vlan);
3009         if (!cache_resource)
3010                 return rte_flow_error_set(error, ENOMEM,
3011                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3012                                           "cannot allocate resource memory");
3013         *cache_resource = *resource;
3014         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3015                 domain = sh->fdb_domain;
3016         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
3017                 domain = sh->rx_domain;
3018         else
3019                 domain = sh->tx_domain;
3020         ret = mlx5_flow_os_create_flow_action_push_vlan
3021                                         (domain, resource->vlan_tag,
3022                                          &cache_resource->action);
3023         if (ret) {
3024                 mlx5_free(cache_resource);
3025                 return rte_flow_error_set(error, ENOMEM,
3026                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3027                                           NULL, "cannot create action");
3028         }
3029         rte_atomic32_init(&cache_resource->refcnt);
3030         rte_atomic32_inc(&cache_resource->refcnt);
3031         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
3032                      &sh->push_vlan_action_list,
3033                      dev_flow->handle->dvh.rix_push_vlan,
3034                      cache_resource, next);
3035         dev_flow->dv.push_vlan_res = cache_resource;
3036         DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
3037                 (void *)cache_resource,
3038                 rte_atomic32_read(&cache_resource->refcnt));
3039         return 0;
3040 }
3041 /**
3042  * Get the size of specific rte_flow_item_type hdr size
3043  *
3044  * @param[in] item_type
3045  *   Tested rte_flow_item_type.
3046  *
3047  * @return
3048  *   sizeof struct item_type, 0 if void or irrelevant.
3049  */
3050 static size_t
3051 flow_dv_get_item_hdr_len(const enum rte_flow_item_type item_type)
3052 {
3053         size_t retval;
3054
3055         switch (item_type) {
3056         case RTE_FLOW_ITEM_TYPE_ETH:
3057                 retval = sizeof(struct rte_ether_hdr);
3058                 break;
3059         case RTE_FLOW_ITEM_TYPE_VLAN:
3060                 retval = sizeof(struct rte_vlan_hdr);
3061                 break;
3062         case RTE_FLOW_ITEM_TYPE_IPV4:
3063                 retval = sizeof(struct rte_ipv4_hdr);
3064                 break;
3065         case RTE_FLOW_ITEM_TYPE_IPV6:
3066                 retval = sizeof(struct rte_ipv6_hdr);
3067                 break;
3068         case RTE_FLOW_ITEM_TYPE_UDP:
3069                 retval = sizeof(struct rte_udp_hdr);
3070                 break;
3071         case RTE_FLOW_ITEM_TYPE_TCP:
3072                 retval = sizeof(struct rte_tcp_hdr);
3073                 break;
3074         case RTE_FLOW_ITEM_TYPE_VXLAN:
3075         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3076                 retval = sizeof(struct rte_vxlan_hdr);
3077                 break;
3078         case RTE_FLOW_ITEM_TYPE_GRE:
3079         case RTE_FLOW_ITEM_TYPE_NVGRE:
3080                 retval = sizeof(struct rte_gre_hdr);
3081                 break;
3082         case RTE_FLOW_ITEM_TYPE_MPLS:
3083                 retval = sizeof(struct rte_mpls_hdr);
3084                 break;
3085         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
3086         default:
3087                 retval = 0;
3088                 break;
3089         }
3090         return retval;
3091 }
3092
3093 #define MLX5_ENCAP_IPV4_VERSION         0x40
3094 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
3095 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
3096 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
3097 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
3098 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
3099 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
3100
3101 /**
3102  * Convert the encap action data from list of rte_flow_item to raw buffer
3103  *
3104  * @param[in] items
3105  *   Pointer to rte_flow_item objects list.
3106  * @param[out] buf
3107  *   Pointer to the output buffer.
3108  * @param[out] size
3109  *   Pointer to the output buffer size.
3110  * @param[out] error
3111  *   Pointer to the error structure.
3112  *
3113  * @return
3114  *   0 on success, a negative errno value otherwise and rte_errno is set.
3115  */
3116 static int
3117 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
3118                            size_t *size, struct rte_flow_error *error)
3119 {
3120         struct rte_ether_hdr *eth = NULL;
3121         struct rte_vlan_hdr *vlan = NULL;
3122         struct rte_ipv4_hdr *ipv4 = NULL;
3123         struct rte_ipv6_hdr *ipv6 = NULL;
3124         struct rte_udp_hdr *udp = NULL;
3125         struct rte_vxlan_hdr *vxlan = NULL;
3126         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
3127         struct rte_gre_hdr *gre = NULL;
3128         size_t len;
3129         size_t temp_size = 0;
3130
3131         if (!items)
3132                 return rte_flow_error_set(error, EINVAL,
3133                                           RTE_FLOW_ERROR_TYPE_ACTION,
3134                                           NULL, "invalid empty data");
3135         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
3136                 len = flow_dv_get_item_hdr_len(items->type);
3137                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
3138                         return rte_flow_error_set(error, EINVAL,
3139                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3140                                                   (void *)items->type,
3141                                                   "items total size is too big"
3142                                                   " for encap action");
3143                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
3144                 switch (items->type) {
3145                 case RTE_FLOW_ITEM_TYPE_ETH:
3146                         eth = (struct rte_ether_hdr *)&buf[temp_size];
3147                         break;
3148                 case RTE_FLOW_ITEM_TYPE_VLAN:
3149                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
3150                         if (!eth)
3151                                 return rte_flow_error_set(error, EINVAL,
3152                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3153                                                 (void *)items->type,
3154                                                 "eth header not found");
3155                         if (!eth->ether_type)
3156                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
3157                         break;
3158                 case RTE_FLOW_ITEM_TYPE_IPV4:
3159                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
3160                         if (!vlan && !eth)
3161                                 return rte_flow_error_set(error, EINVAL,
3162                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3163                                                 (void *)items->type,
3164                                                 "neither eth nor vlan"
3165                                                 " header found");
3166                         if (vlan && !vlan->eth_proto)
3167                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3168                         else if (eth && !eth->ether_type)
3169                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3170                         if (!ipv4->version_ihl)
3171                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
3172                                                     MLX5_ENCAP_IPV4_IHL_MIN;
3173                         if (!ipv4->time_to_live)
3174                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
3175                         break;
3176                 case RTE_FLOW_ITEM_TYPE_IPV6:
3177                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
3178                         if (!vlan && !eth)
3179                                 return rte_flow_error_set(error, EINVAL,
3180                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3181                                                 (void *)items->type,
3182                                                 "neither eth nor vlan"
3183                                                 " header found");
3184                         if (vlan && !vlan->eth_proto)
3185                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3186                         else if (eth && !eth->ether_type)
3187                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3188                         if (!ipv6->vtc_flow)
3189                                 ipv6->vtc_flow =
3190                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
3191                         if (!ipv6->hop_limits)
3192                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
3193                         break;
3194                 case RTE_FLOW_ITEM_TYPE_UDP:
3195                         udp = (struct rte_udp_hdr *)&buf[temp_size];
3196                         if (!ipv4 && !ipv6)
3197                                 return rte_flow_error_set(error, EINVAL,
3198                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3199                                                 (void *)items->type,
3200                                                 "ip header not found");
3201                         if (ipv4 && !ipv4->next_proto_id)
3202                                 ipv4->next_proto_id = IPPROTO_UDP;
3203                         else if (ipv6 && !ipv6->proto)
3204                                 ipv6->proto = IPPROTO_UDP;
3205                         break;
3206                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3207                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
3208                         if (!udp)
3209                                 return rte_flow_error_set(error, EINVAL,
3210                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3211                                                 (void *)items->type,
3212                                                 "udp header not found");
3213                         if (!udp->dst_port)
3214                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
3215                         if (!vxlan->vx_flags)
3216                                 vxlan->vx_flags =
3217                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
3218                         break;
3219                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3220                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
3221                         if (!udp)
3222                                 return rte_flow_error_set(error, EINVAL,
3223                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3224                                                 (void *)items->type,
3225                                                 "udp header not found");
3226                         if (!vxlan_gpe->proto)
3227                                 return rte_flow_error_set(error, EINVAL,
3228                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3229                                                 (void *)items->type,
3230                                                 "next protocol not found");
3231                         if (!udp->dst_port)
3232                                 udp->dst_port =
3233                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
3234                         if (!vxlan_gpe->vx_flags)
3235                                 vxlan_gpe->vx_flags =
3236                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
3237                         break;
3238                 case RTE_FLOW_ITEM_TYPE_GRE:
3239                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3240                         gre = (struct rte_gre_hdr *)&buf[temp_size];
3241                         if (!gre->proto)
3242                                 return rte_flow_error_set(error, EINVAL,
3243                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3244                                                 (void *)items->type,
3245                                                 "next protocol not found");
3246                         if (!ipv4 && !ipv6)
3247                                 return rte_flow_error_set(error, EINVAL,
3248                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3249                                                 (void *)items->type,
3250                                                 "ip header not found");
3251                         if (ipv4 && !ipv4->next_proto_id)
3252                                 ipv4->next_proto_id = IPPROTO_GRE;
3253                         else if (ipv6 && !ipv6->proto)
3254                                 ipv6->proto = IPPROTO_GRE;
3255                         break;
3256                 case RTE_FLOW_ITEM_TYPE_VOID:
3257                         break;
3258                 default:
3259                         return rte_flow_error_set(error, EINVAL,
3260                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3261                                                   (void *)items->type,
3262                                                   "unsupported item type");
3263                         break;
3264                 }
3265                 temp_size += len;
3266         }
3267         *size = temp_size;
3268         return 0;
3269 }
3270
3271 static int
3272 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
3273 {
3274         struct rte_ether_hdr *eth = NULL;
3275         struct rte_vlan_hdr *vlan = NULL;
3276         struct rte_ipv6_hdr *ipv6 = NULL;
3277         struct rte_udp_hdr *udp = NULL;
3278         char *next_hdr;
3279         uint16_t proto;
3280
3281         eth = (struct rte_ether_hdr *)data;
3282         next_hdr = (char *)(eth + 1);
3283         proto = RTE_BE16(eth->ether_type);
3284
3285         /* VLAN skipping */
3286         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
3287                 vlan = (struct rte_vlan_hdr *)next_hdr;
3288                 proto = RTE_BE16(vlan->eth_proto);
3289                 next_hdr += sizeof(struct rte_vlan_hdr);
3290         }
3291
3292         /* HW calculates IPv4 csum. no need to proceed */
3293         if (proto == RTE_ETHER_TYPE_IPV4)
3294                 return 0;
3295
3296         /* non IPv4/IPv6 header. not supported */
3297         if (proto != RTE_ETHER_TYPE_IPV6) {
3298                 return rte_flow_error_set(error, ENOTSUP,
3299                                           RTE_FLOW_ERROR_TYPE_ACTION,
3300                                           NULL, "Cannot offload non IPv4/IPv6");
3301         }
3302
3303         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
3304
3305         /* ignore non UDP */
3306         if (ipv6->proto != IPPROTO_UDP)
3307                 return 0;
3308
3309         udp = (struct rte_udp_hdr *)(ipv6 + 1);
3310         udp->dgram_cksum = 0;
3311
3312         return 0;
3313 }
3314
3315 /**
3316  * Convert L2 encap action to DV specification.
3317  *
3318  * @param[in] dev
3319  *   Pointer to rte_eth_dev structure.
3320  * @param[in] action
3321  *   Pointer to action structure.
3322  * @param[in, out] dev_flow
3323  *   Pointer to the mlx5_flow.
3324  * @param[in] transfer
3325  *   Mark if the flow is E-Switch flow.
3326  * @param[out] error
3327  *   Pointer to the error structure.
3328  *
3329  * @return
3330  *   0 on success, a negative errno value otherwise and rte_errno is set.
3331  */
3332 static int
3333 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
3334                                const struct rte_flow_action *action,
3335                                struct mlx5_flow *dev_flow,
3336                                uint8_t transfer,
3337                                struct rte_flow_error *error)
3338 {
3339         const struct rte_flow_item *encap_data;
3340         const struct rte_flow_action_raw_encap *raw_encap_data;
3341         struct mlx5_flow_dv_encap_decap_resource res = {
3342                 .reformat_type =
3343                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
3344                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3345                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
3346         };
3347
3348         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3349                 raw_encap_data =
3350                         (const struct rte_flow_action_raw_encap *)action->conf;
3351                 res.size = raw_encap_data->size;
3352                 memcpy(res.buf, raw_encap_data->data, res.size);
3353         } else {
3354                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
3355                         encap_data =
3356                                 ((const struct rte_flow_action_vxlan_encap *)
3357                                                 action->conf)->definition;
3358                 else
3359                         encap_data =
3360                                 ((const struct rte_flow_action_nvgre_encap *)
3361                                                 action->conf)->definition;
3362                 if (flow_dv_convert_encap_data(encap_data, res.buf,
3363                                                &res.size, error))
3364                         return -rte_errno;
3365         }
3366         if (flow_dv_zero_encap_udp_csum(res.buf, error))
3367                 return -rte_errno;
3368         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3369                 return rte_flow_error_set(error, EINVAL,
3370                                           RTE_FLOW_ERROR_TYPE_ACTION,
3371                                           NULL, "can't create L2 encap action");
3372         return 0;
3373 }
3374
3375 /**
3376  * Convert L2 decap action to DV specification.
3377  *
3378  * @param[in] dev
3379  *   Pointer to rte_eth_dev structure.
3380  * @param[in, out] dev_flow
3381  *   Pointer to the mlx5_flow.
3382  * @param[in] transfer
3383  *   Mark if the flow is E-Switch flow.
3384  * @param[out] error
3385  *   Pointer to the error structure.
3386  *
3387  * @return
3388  *   0 on success, a negative errno value otherwise and rte_errno is set.
3389  */
3390 static int
3391 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
3392                                struct mlx5_flow *dev_flow,
3393                                uint8_t transfer,
3394                                struct rte_flow_error *error)
3395 {
3396         struct mlx5_flow_dv_encap_decap_resource res = {
3397                 .size = 0,
3398                 .reformat_type =
3399                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
3400                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3401                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
3402         };
3403
3404         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3405                 return rte_flow_error_set(error, EINVAL,
3406                                           RTE_FLOW_ERROR_TYPE_ACTION,
3407                                           NULL, "can't create L2 decap action");
3408         return 0;
3409 }
3410
3411 /**
3412  * Convert raw decap/encap (L3 tunnel) action to DV specification.
3413  *
3414  * @param[in] dev
3415  *   Pointer to rte_eth_dev structure.
3416  * @param[in] action
3417  *   Pointer to action structure.
3418  * @param[in, out] dev_flow
3419  *   Pointer to the mlx5_flow.
3420  * @param[in] attr
3421  *   Pointer to the flow attributes.
3422  * @param[out] error
3423  *   Pointer to the error structure.
3424  *
3425  * @return
3426  *   0 on success, a negative errno value otherwise and rte_errno is set.
3427  */
3428 static int
3429 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
3430                                 const struct rte_flow_action *action,
3431                                 struct mlx5_flow *dev_flow,
3432                                 const struct rte_flow_attr *attr,
3433                                 struct rte_flow_error *error)
3434 {
3435         const struct rte_flow_action_raw_encap *encap_data;
3436         struct mlx5_flow_dv_encap_decap_resource res;
3437
3438         memset(&res, 0, sizeof(res));
3439         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
3440         res.size = encap_data->size;
3441         memcpy(res.buf, encap_data->data, res.size);
3442         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
3443                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
3444                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3445         if (attr->transfer)
3446                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3447         else
3448                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3449                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3450         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3451                 return rte_flow_error_set(error, EINVAL,
3452                                           RTE_FLOW_ERROR_TYPE_ACTION,
3453                                           NULL, "can't create encap action");
3454         return 0;
3455 }
3456
3457 /**
3458  * Create action push VLAN.
3459  *
3460  * @param[in] dev
3461  *   Pointer to rte_eth_dev structure.
3462  * @param[in] attr
3463  *   Pointer to the flow attributes.
3464  * @param[in] vlan
3465  *   Pointer to the vlan to push to the Ethernet header.
3466  * @param[in, out] dev_flow
3467  *   Pointer to the mlx5_flow.
3468  * @param[out] error
3469  *   Pointer to the error structure.
3470  *
3471  * @return
3472  *   0 on success, a negative errno value otherwise and rte_errno is set.
3473  */
3474 static int
3475 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
3476                                 const struct rte_flow_attr *attr,
3477                                 const struct rte_vlan_hdr *vlan,
3478                                 struct mlx5_flow *dev_flow,
3479                                 struct rte_flow_error *error)
3480 {
3481         struct mlx5_flow_dv_push_vlan_action_resource res;
3482
3483         memset(&res, 0, sizeof(res));
3484         res.vlan_tag =
3485                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
3486                                  vlan->vlan_tci);
3487         if (attr->transfer)
3488                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3489         else
3490                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3491                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3492         return flow_dv_push_vlan_action_resource_register
3493                                             (dev, &res, dev_flow, error);
3494 }
3495
3496 static int fdb_mirror;
3497
3498 /**
3499  * Validate the modify-header actions.
3500  *
3501  * @param[in] action_flags
3502  *   Holds the actions detected until now.
3503  * @param[in] action
3504  *   Pointer to the modify action.
3505  * @param[out] error
3506  *   Pointer to error structure.
3507  *
3508  * @return
3509  *   0 on success, a negative errno value otherwise and rte_errno is set.
3510  */
3511 static int
3512 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
3513                                    const struct rte_flow_action *action,
3514                                    struct rte_flow_error *error)
3515 {
3516         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
3517                 return rte_flow_error_set(error, EINVAL,
3518                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3519                                           NULL, "action configuration not set");
3520         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3521                 return rte_flow_error_set(error, EINVAL,
3522                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3523                                           "can't have encap action before"
3524                                           " modify action");
3525         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror)
3526                 return rte_flow_error_set(error, EINVAL,
3527                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3528                                           "can't support sample action before"
3529                                           " modify action for E-Switch"
3530                                           " mirroring");
3531         return 0;
3532 }
3533
3534 /**
3535  * Validate the modify-header MAC address actions.
3536  *
3537  * @param[in] action_flags
3538  *   Holds the actions detected until now.
3539  * @param[in] action
3540  *   Pointer to the modify action.
3541  * @param[in] item_flags
3542  *   Holds the items detected.
3543  * @param[out] error
3544  *   Pointer to error structure.
3545  *
3546  * @return
3547  *   0 on success, a negative errno value otherwise and rte_errno is set.
3548  */
3549 static int
3550 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
3551                                    const struct rte_flow_action *action,
3552                                    const uint64_t item_flags,
3553                                    struct rte_flow_error *error)
3554 {
3555         int ret = 0;
3556
3557         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3558         if (!ret) {
3559                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
3560                         return rte_flow_error_set(error, EINVAL,
3561                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3562                                                   NULL,
3563                                                   "no L2 item in pattern");
3564         }
3565         return ret;
3566 }
3567
3568 /**
3569  * Validate the modify-header IPv4 address actions.
3570  *
3571  * @param[in] action_flags
3572  *   Holds the actions detected until now.
3573  * @param[in] action
3574  *   Pointer to the modify action.
3575  * @param[in] item_flags
3576  *   Holds the items detected.
3577  * @param[out] error
3578  *   Pointer to error structure.
3579  *
3580  * @return
3581  *   0 on success, a negative errno value otherwise and rte_errno is set.
3582  */
3583 static int
3584 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3585                                     const struct rte_flow_action *action,
3586                                     const uint64_t item_flags,
3587                                     struct rte_flow_error *error)
3588 {
3589         int ret = 0;
3590         uint64_t layer;
3591
3592         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3593         if (!ret) {
3594                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3595                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3596                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3597                 if (!(item_flags & layer))
3598                         return rte_flow_error_set(error, EINVAL,
3599                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3600                                                   NULL,
3601                                                   "no ipv4 item in pattern");
3602         }
3603         return ret;
3604 }
3605
3606 /**
3607  * Validate the modify-header IPv6 address actions.
3608  *
3609  * @param[in] action_flags
3610  *   Holds the actions detected until now.
3611  * @param[in] action
3612  *   Pointer to the modify action.
3613  * @param[in] item_flags
3614  *   Holds the items detected.
3615  * @param[out] error
3616  *   Pointer to error structure.
3617  *
3618  * @return
3619  *   0 on success, a negative errno value otherwise and rte_errno is set.
3620  */
3621 static int
3622 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3623                                     const struct rte_flow_action *action,
3624                                     const uint64_t item_flags,
3625                                     struct rte_flow_error *error)
3626 {
3627         int ret = 0;
3628         uint64_t layer;
3629
3630         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3631         if (!ret) {
3632                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3633                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3634                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3635                 if (!(item_flags & layer))
3636                         return rte_flow_error_set(error, EINVAL,
3637                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3638                                                   NULL,
3639                                                   "no ipv6 item in pattern");
3640         }
3641         return ret;
3642 }
3643
3644 /**
3645  * Validate the modify-header TP actions.
3646  *
3647  * @param[in] action_flags
3648  *   Holds the actions detected until now.
3649  * @param[in] action
3650  *   Pointer to the modify action.
3651  * @param[in] item_flags
3652  *   Holds the items detected.
3653  * @param[out] error
3654  *   Pointer to error structure.
3655  *
3656  * @return
3657  *   0 on success, a negative errno value otherwise and rte_errno is set.
3658  */
3659 static int
3660 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3661                                   const struct rte_flow_action *action,
3662                                   const uint64_t item_flags,
3663                                   struct rte_flow_error *error)
3664 {
3665         int ret = 0;
3666         uint64_t layer;
3667
3668         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3669         if (!ret) {
3670                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3671                                  MLX5_FLOW_LAYER_INNER_L4 :
3672                                  MLX5_FLOW_LAYER_OUTER_L4;
3673                 if (!(item_flags & layer))
3674                         return rte_flow_error_set(error, EINVAL,
3675                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3676                                                   NULL, "no transport layer "
3677                                                   "in pattern");
3678         }
3679         return ret;
3680 }
3681
3682 /**
3683  * Validate the modify-header actions of increment/decrement
3684  * TCP Sequence-number.
3685  *
3686  * @param[in] action_flags
3687  *   Holds the actions detected until now.
3688  * @param[in] action
3689  *   Pointer to the modify action.
3690  * @param[in] item_flags
3691  *   Holds the items detected.
3692  * @param[out] error
3693  *   Pointer to error structure.
3694  *
3695  * @return
3696  *   0 on success, a negative errno value otherwise and rte_errno is set.
3697  */
3698 static int
3699 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3700                                        const struct rte_flow_action *action,
3701                                        const uint64_t item_flags,
3702                                        struct rte_flow_error *error)
3703 {
3704         int ret = 0;
3705         uint64_t layer;
3706
3707         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3708         if (!ret) {
3709                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3710                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3711                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3712                 if (!(item_flags & layer))
3713                         return rte_flow_error_set(error, EINVAL,
3714                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3715                                                   NULL, "no TCP item in"
3716                                                   " pattern");
3717                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3718                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3719                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3720                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3721                         return rte_flow_error_set(error, EINVAL,
3722                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3723                                                   NULL,
3724                                                   "cannot decrease and increase"
3725                                                   " TCP sequence number"
3726                                                   " at the same time");
3727         }
3728         return ret;
3729 }
3730
3731 /**
3732  * Validate the modify-header actions of increment/decrement
3733  * TCP Acknowledgment number.
3734  *
3735  * @param[in] action_flags
3736  *   Holds the actions detected until now.
3737  * @param[in] action
3738  *   Pointer to the modify action.
3739  * @param[in] item_flags
3740  *   Holds the items detected.
3741  * @param[out] error
3742  *   Pointer to error structure.
3743  *
3744  * @return
3745  *   0 on success, a negative errno value otherwise and rte_errno is set.
3746  */
3747 static int
3748 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3749                                        const struct rte_flow_action *action,
3750                                        const uint64_t item_flags,
3751                                        struct rte_flow_error *error)
3752 {
3753         int ret = 0;
3754         uint64_t layer;
3755
3756         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3757         if (!ret) {
3758                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3759                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3760                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3761                 if (!(item_flags & layer))
3762                         return rte_flow_error_set(error, EINVAL,
3763                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3764                                                   NULL, "no TCP item in"
3765                                                   " pattern");
3766                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3767                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3768                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3769                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3770                         return rte_flow_error_set(error, EINVAL,
3771                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3772                                                   NULL,
3773                                                   "cannot decrease and increase"
3774                                                   " TCP acknowledgment number"
3775                                                   " at the same time");
3776         }
3777         return ret;
3778 }
3779
3780 /**
3781  * Validate the modify-header TTL actions.
3782  *
3783  * @param[in] action_flags
3784  *   Holds the actions detected until now.
3785  * @param[in] action
3786  *   Pointer to the modify action.
3787  * @param[in] item_flags
3788  *   Holds the items detected.
3789  * @param[out] error
3790  *   Pointer to error structure.
3791  *
3792  * @return
3793  *   0 on success, a negative errno value otherwise and rte_errno is set.
3794  */
3795 static int
3796 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3797                                    const struct rte_flow_action *action,
3798                                    const uint64_t item_flags,
3799                                    struct rte_flow_error *error)
3800 {
3801         int ret = 0;
3802         uint64_t layer;
3803
3804         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3805         if (!ret) {
3806                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3807                                  MLX5_FLOW_LAYER_INNER_L3 :
3808                                  MLX5_FLOW_LAYER_OUTER_L3;
3809                 if (!(item_flags & layer))
3810                         return rte_flow_error_set(error, EINVAL,
3811                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3812                                                   NULL,
3813                                                   "no IP protocol in pattern");
3814         }
3815         return ret;
3816 }
3817
3818 /**
3819  * Validate jump action.
3820  *
3821  * @param[in] action
3822  *   Pointer to the jump action.
3823  * @param[in] action_flags
3824  *   Holds the actions detected until now.
3825  * @param[in] attributes
3826  *   Pointer to flow attributes
3827  * @param[in] external
3828  *   Action belongs to flow rule created by request external to PMD.
3829  * @param[out] error
3830  *   Pointer to error structure.
3831  *
3832  * @return
3833  *   0 on success, a negative errno value otherwise and rte_errno is set.
3834  */
3835 static int
3836 flow_dv_validate_action_jump(const struct rte_flow_action *action,
3837                              uint64_t action_flags,
3838                              const struct rte_flow_attr *attributes,
3839                              bool external, struct rte_flow_error *error)
3840 {
3841         uint32_t target_group, table;
3842         int ret = 0;
3843
3844         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3845                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3846                 return rte_flow_error_set(error, EINVAL,
3847                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3848                                           "can't have 2 fate actions in"
3849                                           " same flow");
3850         if (action_flags & MLX5_FLOW_ACTION_METER)
3851                 return rte_flow_error_set(error, ENOTSUP,
3852                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3853                                           "jump with meter not support");
3854         if ((action_flags & MLX5_FLOW_ACTION_SAMPLE) && fdb_mirror)
3855                 return rte_flow_error_set(error, EINVAL,
3856                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3857                                           "E-Switch mirroring can't support"
3858                                           " Sample action and jump action in"
3859                                           " same flow now");
3860         if (!action->conf)
3861                 return rte_flow_error_set(error, EINVAL,
3862                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3863                                           NULL, "action configuration not set");
3864         target_group =
3865                 ((const struct rte_flow_action_jump *)action->conf)->group;
3866         ret = mlx5_flow_group_to_table(attributes, external, target_group,
3867                                        true, &table, error);
3868         if (ret)
3869                 return ret;
3870         if (attributes->group == target_group)
3871                 return rte_flow_error_set(error, EINVAL,
3872                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3873                                           "target group must be other than"
3874                                           " the current flow group");
3875         return 0;
3876 }
3877
3878 /*
3879  * Validate the port_id action.
3880  *
3881  * @param[in] dev
3882  *   Pointer to rte_eth_dev structure.
3883  * @param[in] action_flags
3884  *   Bit-fields that holds the actions detected until now.
3885  * @param[in] action
3886  *   Port_id RTE action structure.
3887  * @param[in] attr
3888  *   Attributes of flow that includes this action.
3889  * @param[out] error
3890  *   Pointer to error structure.
3891  *
3892  * @return
3893  *   0 on success, a negative errno value otherwise and rte_errno is set.
3894  */
3895 static int
3896 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
3897                                 uint64_t action_flags,
3898                                 const struct rte_flow_action *action,
3899                                 const struct rte_flow_attr *attr,
3900                                 struct rte_flow_error *error)
3901 {
3902         const struct rte_flow_action_port_id *port_id;
3903         struct mlx5_priv *act_priv;
3904         struct mlx5_priv *dev_priv;
3905         uint16_t port;
3906
3907         if (!attr->transfer)
3908                 return rte_flow_error_set(error, ENOTSUP,
3909                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3910                                           NULL,
3911                                           "port id action is valid in transfer"
3912                                           " mode only");
3913         if (!action || !action->conf)
3914                 return rte_flow_error_set(error, ENOTSUP,
3915                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3916                                           NULL,
3917                                           "port id action parameters must be"
3918                                           " specified");
3919         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3920                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3921                 return rte_flow_error_set(error, EINVAL,
3922                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3923                                           "can have only one fate actions in"
3924                                           " a flow");
3925         dev_priv = mlx5_dev_to_eswitch_info(dev);
3926         if (!dev_priv)
3927                 return rte_flow_error_set(error, rte_errno,
3928                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3929                                           NULL,
3930                                           "failed to obtain E-Switch info");
3931         port_id = action->conf;
3932         port = port_id->original ? dev->data->port_id : port_id->id;
3933         act_priv = mlx5_port_to_eswitch_info(port, false);
3934         if (!act_priv)
3935                 return rte_flow_error_set
3936                                 (error, rte_errno,
3937                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
3938                                  "failed to obtain E-Switch port id for port");
3939         if (act_priv->domain_id != dev_priv->domain_id)
3940                 return rte_flow_error_set
3941                                 (error, EINVAL,
3942                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3943                                  "port does not belong to"
3944                                  " E-Switch being configured");
3945         return 0;
3946 }
3947
3948 /**
3949  * Get the maximum number of modify header actions.
3950  *
3951  * @param dev
3952  *   Pointer to rte_eth_dev structure.
3953  * @param flags
3954  *   Flags bits to check if root level.
3955  *
3956  * @return
3957  *   Max number of modify header actions device can support.
3958  */
3959 static inline unsigned int
3960 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
3961                               uint64_t flags)
3962 {
3963         /*
3964          * There's no way to directly query the max capacity from FW.
3965          * The maximal value on root table should be assumed to be supported.
3966          */
3967         if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
3968                 return MLX5_MAX_MODIFY_NUM;
3969         else
3970                 return MLX5_ROOT_TBL_MODIFY_NUM;
3971 }
3972
3973 /**
3974  * Validate the meter action.
3975  *
3976  * @param[in] dev
3977  *   Pointer to rte_eth_dev structure.
3978  * @param[in] action_flags
3979  *   Bit-fields that holds the actions detected until now.
3980  * @param[in] action
3981  *   Pointer to the meter action.
3982  * @param[in] attr
3983  *   Attributes of flow that includes this action.
3984  * @param[out] error
3985  *   Pointer to error structure.
3986  *
3987  * @return
3988  *   0 on success, a negative errno value otherwise and rte_ernno is set.
3989  */
3990 static int
3991 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
3992                                 uint64_t action_flags,
3993                                 const struct rte_flow_action *action,
3994                                 const struct rte_flow_attr *attr,
3995                                 struct rte_flow_error *error)
3996 {
3997         struct mlx5_priv *priv = dev->data->dev_private;
3998         const struct rte_flow_action_meter *am = action->conf;
3999         struct mlx5_flow_meter *fm;
4000
4001         if (!am)
4002                 return rte_flow_error_set(error, EINVAL,
4003                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4004                                           "meter action conf is NULL");
4005
4006         if (action_flags & MLX5_FLOW_ACTION_METER)
4007                 return rte_flow_error_set(error, ENOTSUP,
4008                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4009                                           "meter chaining not support");
4010         if (action_flags & MLX5_FLOW_ACTION_JUMP)
4011                 return rte_flow_error_set(error, ENOTSUP,
4012                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4013                                           "meter with jump not support");
4014         if (!priv->mtr_en)
4015                 return rte_flow_error_set(error, ENOTSUP,
4016                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4017                                           NULL,
4018                                           "meter action not supported");
4019         fm = mlx5_flow_meter_find(priv, am->mtr_id);
4020         if (!fm)
4021                 return rte_flow_error_set(error, EINVAL,
4022                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4023                                           "Meter not found");
4024         if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
4025               (!fm->ingress && !attr->ingress && attr->egress) ||
4026               (!fm->egress && !attr->egress && attr->ingress))))
4027                 return rte_flow_error_set(error, EINVAL,
4028                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4029                                           "Flow attributes are either invalid "
4030                                           "or have a conflict with current "
4031                                           "meter attributes");
4032         return 0;
4033 }
4034
4035 /**
4036  * Validate the age action.
4037  *
4038  * @param[in] action_flags
4039  *   Holds the actions detected until now.
4040  * @param[in] action
4041  *   Pointer to the age action.
4042  * @param[in] dev
4043  *   Pointer to the Ethernet device structure.
4044  * @param[out] error
4045  *   Pointer to error structure.
4046  *
4047  * @return
4048  *   0 on success, a negative errno value otherwise and rte_errno is set.
4049  */
4050 static int
4051 flow_dv_validate_action_age(uint64_t action_flags,
4052                             const struct rte_flow_action *action,
4053                             struct rte_eth_dev *dev,
4054                             struct rte_flow_error *error)
4055 {
4056         struct mlx5_priv *priv = dev->data->dev_private;
4057         const struct rte_flow_action_age *age = action->conf;
4058
4059         if (!priv->config.devx || priv->counter_fallback)
4060                 return rte_flow_error_set(error, ENOTSUP,
4061                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4062                                           NULL,
4063                                           "age action not supported");
4064         if (!(action->conf))
4065                 return rte_flow_error_set(error, EINVAL,
4066                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4067                                           "configuration cannot be null");
4068         if (age->timeout >= UINT16_MAX / 2 / 10)
4069                 return rte_flow_error_set(error, ENOTSUP,
4070                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4071                                           "Max age time: 3275 seconds");
4072         if (action_flags & MLX5_FLOW_ACTION_AGE)
4073                 return rte_flow_error_set(error, EINVAL,
4074                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4075                                           "Duplicate age ctions set");
4076         return 0;
4077 }
4078
4079 /**
4080  * Validate the modify-header IPv4 DSCP actions.
4081  *
4082  * @param[in] action_flags
4083  *   Holds the actions detected until now.
4084  * @param[in] action
4085  *   Pointer to the modify action.
4086  * @param[in] item_flags
4087  *   Holds the items detected.
4088  * @param[out] error
4089  *   Pointer to error structure.
4090  *
4091  * @return
4092  *   0 on success, a negative errno value otherwise and rte_errno is set.
4093  */
4094 static int
4095 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
4096                                          const struct rte_flow_action *action,
4097                                          const uint64_t item_flags,
4098                                          struct rte_flow_error *error)
4099 {
4100         int ret = 0;
4101
4102         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4103         if (!ret) {
4104                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
4105                         return rte_flow_error_set(error, EINVAL,
4106                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4107                                                   NULL,
4108                                                   "no ipv4 item in pattern");
4109         }
4110         return ret;
4111 }
4112
4113 /**
4114  * Validate the modify-header IPv6 DSCP actions.
4115  *
4116  * @param[in] action_flags
4117  *   Holds the actions detected until now.
4118  * @param[in] action
4119  *   Pointer to the modify action.
4120  * @param[in] item_flags
4121  *   Holds the items detected.
4122  * @param[out] error
4123  *   Pointer to error structure.
4124  *
4125  * @return
4126  *   0 on success, a negative errno value otherwise and rte_errno is set.
4127  */
4128 static int
4129 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
4130                                          const struct rte_flow_action *action,
4131                                          const uint64_t item_flags,
4132                                          struct rte_flow_error *error)
4133 {
4134         int ret = 0;
4135
4136         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
4137         if (!ret) {
4138                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
4139                         return rte_flow_error_set(error, EINVAL,
4140                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4141                                                   NULL,
4142                                                   "no ipv6 item in pattern");
4143         }
4144         return ret;
4145 }
4146
4147 /**
4148  * Match modify-header resource.
4149  *
4150  * @param entry
4151  *   Pointer to exist resource entry object.
4152  * @param ctx
4153  *   Pointer to new modify-header resource.
4154  *
4155  * @return
4156  *   0 on matching, -1 otherwise.
4157  */
4158 static int
4159 flow_dv_modify_hdr_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
4160 {
4161         struct mlx5_flow_dv_modify_hdr_resource *resource;
4162         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4163         uint32_t actions_len;
4164
4165         resource = (struct mlx5_flow_dv_modify_hdr_resource *)ctx;
4166         cache_resource = container_of(entry,
4167                                       struct mlx5_flow_dv_modify_hdr_resource,
4168                                       entry);
4169         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4170         if (resource->entry.key == cache_resource->entry.key &&
4171             resource->ft_type == cache_resource->ft_type &&
4172             resource->actions_num == cache_resource->actions_num &&
4173             resource->flags == cache_resource->flags &&
4174             !memcmp((const void *)resource->actions,
4175                     (const void *)cache_resource->actions,
4176                     actions_len))
4177                 return 0;
4178         return -1;
4179 }
4180
4181 /**
4182  * Validate the sample action.
4183  *
4184  * @param[in] action_flags
4185  *   Holds the actions detected until now.
4186  * @param[in] action
4187  *   Pointer to the sample action.
4188  * @param[in] dev
4189  *   Pointer to the Ethernet device structure.
4190  * @param[in] attr
4191  *   Attributes of flow that includes this action.
4192  * @param[out] error
4193  *   Pointer to error structure.
4194  *
4195  * @return
4196  *   0 on success, a negative errno value otherwise and rte_errno is set.
4197  */
4198 static int
4199 flow_dv_validate_action_sample(uint64_t action_flags,
4200                                const struct rte_flow_action *action,
4201                                struct rte_eth_dev *dev,
4202                                const struct rte_flow_attr *attr,
4203                                struct rte_flow_error *error)
4204 {
4205         struct mlx5_priv *priv = dev->data->dev_private;
4206         struct mlx5_dev_config *dev_conf = &priv->config;
4207         const struct rte_flow_action_sample *sample = action->conf;
4208         const struct rte_flow_action *act;
4209         uint64_t sub_action_flags = 0;
4210         uint16_t queue_index = 0xFFFF;
4211         int actions_n = 0;
4212         int ret;
4213         fdb_mirror = 0;
4214
4215         if (!sample)
4216                 return rte_flow_error_set(error, EINVAL,
4217                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4218                                           "configuration cannot be NULL");
4219         if (sample->ratio == 0)
4220                 return rte_flow_error_set(error, EINVAL,
4221                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4222                                           "ratio value starts from 1");
4223         if (!priv->config.devx || (sample->ratio > 0 && !priv->sampler_en))
4224                 return rte_flow_error_set(error, ENOTSUP,
4225                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4226                                           NULL,
4227                                           "sample action not supported");
4228         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
4229                 return rte_flow_error_set(error, EINVAL,
4230                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4231                                           "Multiple sample actions not "
4232                                           "supported");
4233         if (action_flags & MLX5_FLOW_ACTION_METER)
4234                 return rte_flow_error_set(error, EINVAL,
4235                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4236                                           "wrong action order, meter should "
4237                                           "be after sample action");
4238         if (action_flags & MLX5_FLOW_ACTION_JUMP)
4239                 return rte_flow_error_set(error, EINVAL,
4240                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
4241                                           "wrong action order, jump should "
4242                                           "be after sample action");
4243         act = sample->actions;
4244         for (; act->type != RTE_FLOW_ACTION_TYPE_END; act++) {
4245                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
4246                         return rte_flow_error_set(error, ENOTSUP,
4247                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4248                                                   act, "too many actions");
4249                 switch (act->type) {
4250                 case RTE_FLOW_ACTION_TYPE_QUEUE:
4251                         ret = mlx5_flow_validate_action_queue(act,
4252                                                               sub_action_flags,
4253                                                               dev,
4254                                                               attr, error);
4255                         if (ret < 0)
4256                                 return ret;
4257                         queue_index = ((const struct rte_flow_action_queue *)
4258                                                         (act->conf))->index;
4259                         sub_action_flags |= MLX5_FLOW_ACTION_QUEUE;
4260                         ++actions_n;
4261                         break;
4262                 case RTE_FLOW_ACTION_TYPE_MARK:
4263                         ret = flow_dv_validate_action_mark(dev, act,
4264                                                            sub_action_flags,
4265                                                            attr, error);
4266                         if (ret < 0)
4267                                 return ret;
4268                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
4269                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK |
4270                                                 MLX5_FLOW_ACTION_MARK_EXT;
4271                         else
4272                                 sub_action_flags |= MLX5_FLOW_ACTION_MARK;
4273                         ++actions_n;
4274                         break;
4275                 case RTE_FLOW_ACTION_TYPE_COUNT:
4276                         ret = flow_dv_validate_action_count(dev, error);
4277                         if (ret < 0)
4278                                 return ret;
4279                         sub_action_flags |= MLX5_FLOW_ACTION_COUNT;
4280                         ++actions_n;
4281                         break;
4282                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
4283                         ret = flow_dv_validate_action_port_id(dev,
4284                                                               sub_action_flags,
4285                                                               act,
4286                                                               attr,
4287                                                               error);
4288                         if (ret)
4289                                 return ret;
4290                         sub_action_flags |= MLX5_FLOW_ACTION_PORT_ID;
4291                         ++actions_n;
4292                         break;
4293                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
4294                         ret = flow_dv_validate_action_raw_encap_decap
4295                                 (dev, NULL, act->conf, attr, &sub_action_flags,
4296                                  &actions_n, error);
4297                         if (ret < 0)
4298                                 return ret;
4299                         ++actions_n;
4300                         break;
4301                 default:
4302                         return rte_flow_error_set(error, ENOTSUP,
4303                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4304                                                   NULL,
4305                                                   "Doesn't support optional "
4306                                                   "action");
4307                 }
4308         }
4309         if (attr->ingress && !attr->transfer) {
4310                 if (!(sub_action_flags & MLX5_FLOW_ACTION_QUEUE))
4311                         return rte_flow_error_set(error, EINVAL,
4312                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4313                                                   NULL,
4314                                                   "Ingress must has a dest "
4315                                                   "QUEUE for Sample");
4316         } else if (attr->egress && !attr->transfer) {
4317                 return rte_flow_error_set(error, ENOTSUP,
4318                                           RTE_FLOW_ERROR_TYPE_ACTION,
4319                                           NULL,
4320                                           "Sample Only support Ingress "
4321                                           "or E-Switch");
4322         } else if (sample->actions->type != RTE_FLOW_ACTION_TYPE_END) {
4323                 MLX5_ASSERT(attr->transfer);
4324                 if (sample->ratio > 1)
4325                         return rte_flow_error_set(error, ENOTSUP,
4326                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4327                                                   NULL,
4328                                                   "E-Switch doesn't support "
4329                                                   "any optional action "
4330                                                   "for sampling");
4331                 fdb_mirror = 1;
4332                 if (sub_action_flags & MLX5_FLOW_ACTION_QUEUE)
4333                         return rte_flow_error_set(error, ENOTSUP,
4334                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4335                                                   NULL,
4336                                                   "unsupported action QUEUE");
4337                 if (!(sub_action_flags & MLX5_FLOW_ACTION_PORT_ID))
4338                         return rte_flow_error_set(error, EINVAL,
4339                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4340                                                   NULL,
4341                                                   "E-Switch must has a dest "
4342                                                   "port for mirroring");
4343         }
4344         /* Continue validation for Xcap actions.*/
4345         if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
4346             (queue_index == 0xFFFF ||
4347              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
4348                 if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
4349                      MLX5_FLOW_XCAP_ACTIONS)
4350                         return rte_flow_error_set(error, ENOTSUP,
4351                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4352                                                   NULL, "encap and decap "
4353                                                   "combination aren't "
4354                                                   "supported");
4355                 if (!attr->transfer && attr->ingress && (sub_action_flags &
4356                                                         MLX5_FLOW_ACTION_ENCAP))
4357                         return rte_flow_error_set(error, ENOTSUP,
4358                                                   RTE_FLOW_ERROR_TYPE_ACTION,
4359                                                   NULL, "encap is not supported"
4360                                                   " for ingress traffic");
4361         }
4362         return 0;
4363 }
4364
4365 /**
4366  * Find existing modify-header resource or create and register a new one.
4367  *
4368  * @param dev[in, out]
4369  *   Pointer to rte_eth_dev structure.
4370  * @param[in, out] resource
4371  *   Pointer to modify-header resource.
4372  * @parm[in, out] dev_flow
4373  *   Pointer to the dev_flow.
4374  * @param[out] error
4375  *   pointer to error structure.
4376  *
4377  * @return
4378  *   0 on success otherwise -errno and errno is set.
4379  */
4380 static int
4381 flow_dv_modify_hdr_resource_register
4382                         (struct rte_eth_dev *dev,
4383                          struct mlx5_flow_dv_modify_hdr_resource *resource,
4384                          struct mlx5_flow *dev_flow,
4385                          struct rte_flow_error *error)
4386 {
4387         struct mlx5_priv *priv = dev->data->dev_private;
4388         struct mlx5_dev_ctx_shared *sh = priv->sh;
4389         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
4390         struct mlx5dv_dr_domain *ns;
4391         uint32_t actions_len;
4392         struct mlx5_hlist_entry *entry;
4393         union mlx5_flow_modify_hdr_key hdr_mod_key = {
4394                 {
4395                         .ft_type = resource->ft_type,
4396                         .actions_num = resource->actions_num,
4397                         .group = dev_flow->dv.group,
4398                         .cksum = 0,
4399                 }
4400         };
4401         int ret;
4402
4403         resource->flags = dev_flow->dv.group ? 0 :
4404                           MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4405         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
4406                                     resource->flags))
4407                 return rte_flow_error_set(error, EOVERFLOW,
4408                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4409                                           "too many modify header items");
4410         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
4411                 ns = sh->fdb_domain;
4412         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
4413                 ns = sh->tx_domain;
4414         else
4415                 ns = sh->rx_domain;
4416         /* Lookup a matching resource from cache. */
4417         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4418         hdr_mod_key.cksum = __rte_raw_cksum(resource->actions, actions_len, 0);
4419         resource->entry.key = hdr_mod_key.v64;
4420         entry = mlx5_hlist_lookup_ex(sh->modify_cmds, resource->entry.key,
4421                                      flow_dv_modify_hdr_resource_match,
4422                                      (void *)resource);
4423         if (entry) {
4424                 cache_resource = container_of(entry,
4425                                         struct mlx5_flow_dv_modify_hdr_resource,
4426                                         entry);
4427                 DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
4428                         (void *)cache_resource,
4429                         rte_atomic32_read(&cache_resource->refcnt));
4430                 rte_atomic32_inc(&cache_resource->refcnt);
4431                 dev_flow->handle->dvh.modify_hdr = cache_resource;
4432                 return 0;
4433
4434         }
4435         /* Register new modify-header resource. */
4436         cache_resource = mlx5_malloc(MLX5_MEM_ZERO,
4437                                     sizeof(*cache_resource) + actions_len, 0,
4438                                     SOCKET_ID_ANY);
4439         if (!cache_resource)
4440                 return rte_flow_error_set(error, ENOMEM,
4441                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4442                                           "cannot allocate resource memory");
4443         *cache_resource = *resource;
4444         rte_memcpy(cache_resource->actions, resource->actions, actions_len);
4445         ret = mlx5_flow_os_create_flow_action_modify_header
4446                                         (sh->ctx, ns, cache_resource,
4447                                          actions_len, &cache_resource->action);
4448         if (ret) {
4449                 mlx5_free(cache_resource);
4450                 return rte_flow_error_set(error, ENOMEM,
4451                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4452                                           NULL, "cannot create action");
4453         }
4454         rte_atomic32_init(&cache_resource->refcnt);
4455         rte_atomic32_inc(&cache_resource->refcnt);
4456         if (mlx5_hlist_insert_ex(sh->modify_cmds, &cache_resource->entry,
4457                                  flow_dv_modify_hdr_resource_match,
4458                                  (void *)cache_resource)) {
4459                 claim_zero(mlx5_flow_os_destroy_flow_action
4460                                                 (cache_resource->action));
4461                 mlx5_free(cache_resource);
4462                 return rte_flow_error_set(error, EEXIST,
4463                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4464                                           NULL, "action exist");
4465         }
4466         dev_flow->handle->dvh.modify_hdr = cache_resource;
4467         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
4468                 (void *)cache_resource,
4469                 rte_atomic32_read(&cache_resource->refcnt));
4470         return 0;
4471 }
4472
4473 /**
4474  * Get DV flow counter by index.
4475  *
4476  * @param[in] dev
4477  *   Pointer to the Ethernet device structure.
4478  * @param[in] idx
4479  *   mlx5 flow counter index in the container.
4480  * @param[out] ppool
4481  *   mlx5 flow counter pool in the container,
4482  *
4483  * @return
4484  *   Pointer to the counter, NULL otherwise.
4485  */
4486 static struct mlx5_flow_counter *
4487 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
4488                            uint32_t idx,
4489                            struct mlx5_flow_counter_pool **ppool)
4490 {
4491         struct mlx5_priv *priv = dev->data->dev_private;
4492         struct mlx5_pools_container *cont;
4493         struct mlx5_flow_counter_pool *pool;
4494         uint32_t batch = 0, age = 0;
4495
4496         idx--;
4497         age = MLX_CNT_IS_AGE(idx);
4498         idx = age ? idx - MLX5_CNT_AGE_OFFSET : idx;
4499         if (idx >= MLX5_CNT_BATCH_OFFSET) {
4500                 idx -= MLX5_CNT_BATCH_OFFSET;
4501                 batch = 1;
4502         }
4503         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4504         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n);
4505         pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
4506         MLX5_ASSERT(pool);
4507         if (ppool)
4508                 *ppool = pool;
4509         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
4510 }
4511
4512 /**
4513  * Check the devx counter belongs to the pool.
4514  *
4515  * @param[in] pool
4516  *   Pointer to the counter pool.
4517  * @param[in] id
4518  *   The counter devx ID.
4519  *
4520  * @return
4521  *   True if counter belongs to the pool, false otherwise.
4522  */
4523 static bool
4524 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
4525 {
4526         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
4527                    MLX5_COUNTERS_PER_POOL;
4528
4529         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
4530                 return true;
4531         return false;
4532 }
4533
4534 /**
4535  * Get a pool by devx counter ID.
4536  *
4537  * @param[in] cont
4538  *   Pointer to the counter container.
4539  * @param[in] id
4540  *   The counter devx ID.
4541  *
4542  * @return
4543  *   The counter pool pointer if exists, NULL otherwise,
4544  */
4545 static struct mlx5_flow_counter_pool *
4546 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
4547 {
4548         uint32_t i;
4549
4550         /* Check last used pool. */
4551         if (cont->last_pool_idx != POOL_IDX_INVALID &&
4552             flow_dv_is_counter_in_pool(cont->pools[cont->last_pool_idx], id))
4553                 return cont->pools[cont->last_pool_idx];
4554         /* ID out of range means no suitable pool in the container. */
4555         if (id > cont->max_id || id < cont->min_id)
4556                 return NULL;
4557         /*
4558          * Find the pool from the end of the container, since mostly counter
4559          * ID is sequence increasing, and the last pool should be the needed
4560          * one.
4561          */
4562         i = rte_atomic16_read(&cont->n_valid);
4563         while (i--) {
4564                 struct mlx5_flow_counter_pool *pool = cont->pools[i];
4565
4566                 if (flow_dv_is_counter_in_pool(pool, id))
4567                         return pool;
4568         }
4569         return NULL;
4570 }
4571
4572 /**
4573  * Allocate a new memory for the counter values wrapped by all the needed
4574  * management.
4575  *
4576  * @param[in] dev
4577  *   Pointer to the Ethernet device structure.
4578  * @param[in] raws_n
4579  *   The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
4580  *
4581  * @return
4582  *   The new memory management pointer on success, otherwise NULL and rte_errno
4583  *   is set.
4584  */
4585 static struct mlx5_counter_stats_mem_mng *
4586 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
4587 {
4588         struct mlx5_priv *priv = dev->data->dev_private;
4589         struct mlx5_dev_ctx_shared *sh = priv->sh;
4590         struct mlx5_devx_mkey_attr mkey_attr;
4591         struct mlx5_counter_stats_mem_mng *mem_mng;
4592         volatile struct flow_counter_stats *raw_data;
4593         int size = (sizeof(struct flow_counter_stats) *
4594                         MLX5_COUNTERS_PER_POOL +
4595                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
4596                         sizeof(struct mlx5_counter_stats_mem_mng);
4597         size_t pgsize = rte_mem_page_size();
4598         if (pgsize == (size_t)-1) {
4599                 DRV_LOG(ERR, "Failed to get mem page size");
4600                 rte_errno = ENOMEM;
4601                 return NULL;
4602         }
4603         uint8_t *mem = mlx5_malloc(MLX5_MEM_ZERO, size, pgsize,
4604                                   SOCKET_ID_ANY);
4605         int i;
4606
4607         if (!mem) {
4608                 rte_errno = ENOMEM;
4609                 return NULL;
4610         }
4611         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
4612         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
4613         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
4614                                                  IBV_ACCESS_LOCAL_WRITE);
4615         if (!mem_mng->umem) {
4616                 rte_errno = errno;
4617                 mlx5_free(mem);
4618                 return NULL;
4619         }
4620         mkey_attr.addr = (uintptr_t)mem;
4621         mkey_attr.size = size;
4622         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
4623         mkey_attr.pd = sh->pdn;
4624         mkey_attr.log_entity_size = 0;
4625         mkey_attr.pg_access = 0;
4626         mkey_attr.klm_array = NULL;
4627         mkey_attr.klm_num = 0;
4628         if (priv->config.hca_attr.relaxed_ordering_write &&
4629                 priv->config.hca_attr.relaxed_ordering_read  &&
4630                 !haswell_broadwell_cpu)
4631                 mkey_attr.relaxed_ordering = 1;
4632         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
4633         if (!mem_mng->dm) {
4634                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
4635                 rte_errno = errno;
4636                 mlx5_free(mem);
4637                 return NULL;
4638         }
4639         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
4640         raw_data = (volatile struct flow_counter_stats *)mem;
4641         for (i = 0; i < raws_n; ++i) {
4642                 mem_mng->raws[i].mem_mng = mem_mng;
4643                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
4644         }
4645         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
4646         return mem_mng;
4647 }
4648
4649 /**
4650  * Resize a counter container.
4651  *
4652  * @param[in] dev
4653  *   Pointer to the Ethernet device structure.
4654  * @param[in] batch
4655  *   Whether the pool is for counter that was allocated by batch command.
4656  * @param[in] age
4657  *   Whether the pool is for Aging counter.
4658  *
4659  * @return
4660  *   0 on success, otherwise negative errno value and rte_errno is set.
4661  */
4662 static int
4663 flow_dv_container_resize(struct rte_eth_dev *dev,
4664                                 uint32_t batch, uint32_t age)
4665 {
4666         struct mlx5_priv *priv = dev->data->dev_private;
4667         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4668                                                                age);
4669         struct mlx5_counter_stats_mem_mng *mem_mng = NULL;
4670         void *old_pools = cont->pools;
4671         uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
4672         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
4673         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
4674
4675         if (!pools) {
4676                 rte_errno = ENOMEM;
4677                 return -ENOMEM;
4678         }
4679         if (old_pools)
4680                 memcpy(pools, old_pools, cont->n *
4681                                        sizeof(struct mlx5_flow_counter_pool *));
4682         /*
4683          * Fallback mode query the counter directly, no background query
4684          * resources are needed.
4685          */
4686         if (!priv->counter_fallback) {
4687                 int i;
4688
4689                 mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
4690                           MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
4691                 if (!mem_mng) {
4692                         mlx5_free(pools);
4693                         return -ENOMEM;
4694                 }
4695                 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
4696                         LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
4697                                          mem_mng->raws +
4698                                          MLX5_CNT_CONTAINER_RESIZE +
4699                                          i, next);
4700         }
4701         rte_spinlock_lock(&cont->resize_sl);
4702         cont->n = resize;
4703         cont->mem_mng = mem_mng;
4704         cont->pools = pools;
4705         rte_spinlock_unlock(&cont->resize_sl);
4706         if (old_pools)
4707                 mlx5_free(old_pools);
4708         return 0;
4709 }
4710
4711 /**
4712  * Query a devx flow counter.
4713  *
4714  * @param[in] dev
4715  *   Pointer to the Ethernet device structure.
4716  * @param[in] cnt
4717  *   Index to the flow counter.
4718  * @param[out] pkts
4719  *   The statistics value of packets.
4720  * @param[out] bytes
4721  *   The statistics value of bytes.
4722  *
4723  * @return
4724  *   0 on success, otherwise a negative errno value and rte_errno is set.
4725  */
4726 static inline int
4727 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
4728                      uint64_t *bytes)
4729 {
4730         struct mlx5_priv *priv = dev->data->dev_private;
4731         struct mlx5_flow_counter_pool *pool = NULL;
4732         struct mlx5_flow_counter *cnt;
4733         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4734         int offset;
4735
4736         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4737         MLX5_ASSERT(pool);
4738         if (counter < MLX5_CNT_BATCH_OFFSET) {
4739                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4740                 if (priv->counter_fallback)
4741                         return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
4742                                         0, pkts, bytes, 0, NULL, NULL, 0);
4743         }
4744
4745         rte_spinlock_lock(&pool->sl);
4746         /*
4747          * The single counters allocation may allocate smaller ID than the
4748          * current allocated in parallel to the host reading.
4749          * In this case the new counter values must be reported as 0.
4750          */
4751         if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) {
4752                 *pkts = 0;
4753                 *bytes = 0;
4754         } else {
4755                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
4756                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
4757                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
4758         }
4759         rte_spinlock_unlock(&pool->sl);
4760         return 0;
4761 }
4762
4763 /**
4764  * Create and initialize a new counter pool.
4765  *
4766  * @param[in] dev
4767  *   Pointer to the Ethernet device structure.
4768  * @param[out] dcs
4769  *   The devX counter handle.
4770  * @param[in] batch
4771  *   Whether the pool is for counter that was allocated by batch command.
4772  * @param[in] age
4773  *   Whether the pool is for counter that was allocated for aging.
4774  * @param[in/out] cont_cur
4775  *   Pointer to the container pointer, it will be update in pool resize.
4776  *
4777  * @return
4778  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
4779  */
4780 static struct mlx5_flow_counter_pool *
4781 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
4782                     uint32_t batch, uint32_t age)
4783 {
4784         struct mlx5_priv *priv = dev->data->dev_private;
4785         struct mlx5_flow_counter_pool *pool;
4786         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4787                                                                age);
4788         int16_t n_valid = rte_atomic16_read(&cont->n_valid);
4789         uint32_t size = sizeof(*pool);
4790
4791         if (cont->n == n_valid && flow_dv_container_resize(dev, batch, age))
4792                 return NULL;
4793         size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
4794         size += (batch ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
4795         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE);
4796         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
4797         if (!pool) {
4798                 rte_errno = ENOMEM;
4799                 return NULL;
4800         }
4801         pool->min_dcs = dcs;
4802         if (!priv->counter_fallback)
4803                 pool->raw = cont->mem_mng->raws + n_valid %
4804                                                       MLX5_CNT_CONTAINER_RESIZE;
4805         pool->raw_hw = NULL;
4806         pool->type = 0;
4807         pool->type |= (batch ? 0 :  CNT_POOL_TYPE_EXT);
4808         pool->type |= (!age ? 0 :  CNT_POOL_TYPE_AGE);
4809         pool->query_gen = 0;
4810         rte_spinlock_init(&pool->sl);
4811         TAILQ_INIT(&pool->counters[0]);
4812         TAILQ_INIT(&pool->counters[1]);
4813         TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
4814         pool->index = n_valid;
4815         cont->pools[n_valid] = pool;
4816         if (!batch) {
4817                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
4818
4819                 if (base < cont->min_id)
4820                         cont->min_id = base;
4821                 if (base > cont->max_id)
4822                         cont->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
4823                 cont->last_pool_idx = pool->index;
4824         }
4825         /* Pool initialization must be updated before host thread access. */
4826         rte_io_wmb();
4827         rte_atomic16_add(&cont->n_valid, 1);
4828         return pool;
4829 }
4830
4831 /**
4832  * Restore skipped counters in the pool.
4833  *
4834  * As counter pool query requires the first counter dcs
4835  * ID start with 4 alinged, if the pool counters with
4836  * min_dcs ID are not aligned with 4, the counters will
4837  * be skipped.
4838  * Once other min_dcs ID less than these skipped counter
4839  * dcs ID appears, the skipped counters will be safe to
4840  * use.
4841  * Should be called when min_dcs is updated.
4842  *
4843  * @param[in] pool
4844  *   Current counter pool.
4845  * @param[in] last_min_dcs
4846  *   Last min_dcs.
4847  */
4848 static void
4849 flow_dv_counter_restore(struct mlx5_flow_counter_pool *pool,
4850                         struct mlx5_devx_obj *last_min_dcs)
4851 {
4852         struct mlx5_flow_counter_ext *cnt_ext;
4853         uint32_t offset, new_offset;
4854         uint32_t skip_cnt = 0;
4855         uint32_t i;
4856
4857         if (!pool->skip_cnt)
4858                 return;
4859         /*
4860          * If last min_dcs is not valid. The skipped counter may even after
4861          * last min_dcs, set the offset to the whole pool.
4862          */
4863         if (last_min_dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))
4864                 offset = MLX5_COUNTERS_PER_POOL;
4865         else
4866                 offset = last_min_dcs->id % MLX5_COUNTERS_PER_POOL;
4867         new_offset = pool->min_dcs->id % MLX5_COUNTERS_PER_POOL;
4868         /*
4869          * Check the counters from 1 to the last_min_dcs range. Counters
4870          * before new min_dcs indicates pool still has skipped counters.
4871          * Counters be skipped after new min_dcs will be ready to use.
4872          * Offset 0 counter must be empty or min_dcs, start from 1.
4873          */
4874         for (i = 1; i < offset; i++) {
4875                 cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i);
4876                 if (cnt_ext->skipped) {
4877                         if (i > new_offset) {
4878                                 cnt_ext->skipped = 0;
4879                                 TAILQ_INSERT_TAIL
4880                                         (&pool->counters[pool->query_gen],
4881                                          MLX5_POOL_GET_CNT(pool, i), next);
4882                         } else {
4883                                 skip_cnt++;
4884                         }
4885                 }
4886         }
4887         if (!skip_cnt)
4888                 pool->skip_cnt = 0;
4889 }
4890
4891 /**
4892  * Prepare a new counter and/or a new counter pool.
4893  *
4894  * @param[in] dev
4895  *   Pointer to the Ethernet device structure.
4896  * @param[out] cnt_free
4897  *   Where to put the pointer of a new counter.
4898  * @param[in] batch
4899  *   Whether the pool is for counter that was allocated by batch command.
4900  * @param[in] age
4901  *   Whether the pool is for counter that was allocated for aging.
4902  *
4903  * @return
4904  *   The counter pool pointer and @p cnt_free is set on success,
4905  *   NULL otherwise and rte_errno is set.
4906  */
4907 static struct mlx5_flow_counter_pool *
4908 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
4909                              struct mlx5_flow_counter **cnt_free,
4910                              uint32_t batch, uint32_t age)
4911 {
4912         struct mlx5_priv *priv = dev->data->dev_private;
4913         struct mlx5_pools_container *cont;
4914         struct mlx5_flow_counter_pool *pool;
4915         struct mlx5_counters tmp_tq;
4916         struct mlx5_devx_obj *last_min_dcs;
4917         struct mlx5_devx_obj *dcs = NULL;
4918         struct mlx5_flow_counter *cnt;
4919         uint32_t add2other;
4920         uint32_t i;
4921
4922         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4923         if (!batch) {
4924 retry:
4925                 add2other = 0;
4926                 /* bulk_bitmap must be 0 for single counter allocation. */
4927                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
4928                 if (!dcs)
4929                         return NULL;
4930                 pool = flow_dv_find_pool_by_id(cont, dcs->id);
4931                 /* Check if counter belongs to exist pool ID range. */
4932                 if (!pool) {
4933                         pool = flow_dv_find_pool_by_id
4934                                (MLX5_CNT_CONTAINER
4935                                (priv->sh, batch, (age ^ 0x1)), dcs->id);
4936                         /*
4937                          * Pool eixsts, counter will be added to the other
4938                          * container, need to reallocate it later.
4939                          */
4940                         if (pool) {
4941                                 add2other = 1;
4942                         } else {
4943                                 pool = flow_dv_pool_create(dev, dcs, batch,
4944                                                            age);
4945                                 if (!pool) {
4946                                         mlx5_devx_cmd_destroy(dcs);
4947                                         return NULL;
4948                                 }
4949                         }
4950                 }
4951                 if ((dcs->id < pool->min_dcs->id ||
4952                     pool->min_dcs->id &
4953                     (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1)) &&
4954                     !(dcs->id & (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))) {
4955                         /*
4956                          * Update the pool min_dcs only if current dcs is
4957                          * valid and exist min_dcs is not valid or greater
4958                          * than new dcs.
4959                          */
4960                         last_min_dcs = pool->min_dcs;
4961                         rte_atomic64_set(&pool->a64_dcs,
4962                                          (int64_t)(uintptr_t)dcs);
4963                         /*
4964                          * Restore any skipped counters if the new min_dcs
4965                          * ID is smaller or min_dcs is not valid.
4966                          */
4967                         if (dcs->id < last_min_dcs->id ||
4968                             last_min_dcs->id &
4969                             (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1))
4970                                 flow_dv_counter_restore(pool, last_min_dcs);
4971                 }
4972                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
4973                 cnt = MLX5_POOL_GET_CNT(pool, i);
4974                 cnt->pool = pool;
4975                 MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs;
4976                 /*
4977                  * If min_dcs is not valid, it means the new allocated dcs
4978                  * also fail to become the valid min_dcs, just skip it.
4979                  * Or if min_dcs is valid, and new dcs ID is smaller than
4980                  * min_dcs, but not become the min_dcs, also skip it.
4981                  */
4982                 if (pool->min_dcs->id &
4983                     (MLX5_CNT_BATCH_QUERY_ID_ALIGNMENT - 1) ||
4984                     dcs->id < pool->min_dcs->id) {
4985                         MLX5_GET_POOL_CNT_EXT(pool, i)->skipped = 1;
4986                         pool->skip_cnt = 1;
4987                         goto retry;
4988                 }
4989                 if (add2other) {
4990                         TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen],
4991                                           cnt, next);
4992                         goto retry;
4993                 }
4994                 *cnt_free = cnt;
4995                 return pool;
4996         }
4997         /* bulk_bitmap is in 128 counters units. */
4998         if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
4999                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
5000         if (!dcs) {
5001                 rte_errno = ENODATA;
5002                 return NULL;
5003         }
5004         pool = flow_dv_pool_create(dev, dcs, batch, age);
5005         if (!pool) {
5006                 mlx5_devx_cmd_destroy(dcs);
5007                 return NULL;
5008         }
5009         TAILQ_INIT(&tmp_tq);
5010         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
5011                 cnt = MLX5_POOL_GET_CNT(pool, i);
5012                 cnt->pool = pool;
5013                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
5014         }
5015         rte_spinlock_lock(&cont->csl);
5016         TAILQ_CONCAT(&cont->counters, &tmp_tq, next);
5017         rte_spinlock_unlock(&cont->csl);
5018         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
5019         (*cnt_free)->pool = pool;
5020         return pool;
5021 }
5022
5023 /**
5024  * Search for existed shared counter.
5025  *
5026  * @param[in] dev
5027  *   Pointer to the Ethernet device structure.
5028  * @param[in] id
5029  *   The shared counter ID to search.
5030  * @param[out] ppool
5031  *   mlx5 flow counter pool in the container,
5032  *
5033  * @return
5034  *   NULL if not existed, otherwise pointer to the shared extend counter.
5035  */
5036 static struct mlx5_flow_counter_ext *
5037 flow_dv_counter_shared_search(struct rte_eth_dev *dev, uint32_t id,
5038                               struct mlx5_flow_counter_pool **ppool)
5039 {
5040         struct mlx5_priv *priv = dev->data->dev_private;
5041         union mlx5_l3t_data data;
5042         uint32_t cnt_idx;
5043
5044         if (mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) || !data.dword)
5045                 return NULL;
5046         cnt_idx = data.dword;
5047         /*
5048          * Shared counters don't have age info. The counter extend is after
5049          * the counter datat structure.
5050          */
5051         return (struct mlx5_flow_counter_ext *)
5052                ((flow_dv_counter_get_by_idx(dev, cnt_idx, ppool)) + 1);
5053 }
5054
5055 /**
5056  * Allocate a flow counter.
5057  *
5058  * @param[in] dev
5059  *   Pointer to the Ethernet device structure.
5060  * @param[in] shared
5061  *   Indicate if this counter is shared with other flows.
5062  * @param[in] id
5063  *   Counter identifier.
5064  * @param[in] group
5065  *   Counter flow group.
5066  * @param[in] age
5067  *   Whether the counter was allocated for aging.
5068  *
5069  * @return
5070  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
5071  */
5072 static uint32_t
5073 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
5074                       uint16_t group, uint32_t age)
5075 {
5076         struct mlx5_priv *priv = dev->data->dev_private;
5077         struct mlx5_flow_counter_pool *pool = NULL;
5078         struct mlx5_flow_counter *cnt_free = NULL;
5079         struct mlx5_flow_counter_ext *cnt_ext = NULL;
5080         /*
5081          * Currently group 0 flow counter cannot be assigned to a flow if it is
5082          * not the first one in the batch counter allocation, so it is better
5083          * to allocate counters one by one for these flows in a separate
5084          * container.
5085          * A counter can be shared between different groups so need to take
5086          * shared counters from the single container.
5087          */
5088         uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0;
5089         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
5090                                                                age);
5091         uint32_t cnt_idx;
5092
5093         if (!priv->config.devx) {
5094                 rte_errno = ENOTSUP;
5095                 return 0;
5096         }
5097         if (shared) {
5098                 cnt_ext = flow_dv_counter_shared_search(dev, id, &pool);
5099                 if (cnt_ext) {
5100                         if (cnt_ext->ref_cnt + 1 == 0) {
5101                                 rte_errno = E2BIG;
5102                                 return 0;
5103                         }
5104                         cnt_ext->ref_cnt++;
5105                         cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL +
5106                                   (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL)
5107                                   + 1;
5108                         return cnt_idx;
5109                 }
5110         }
5111         /* Get free counters from container. */
5112         rte_spinlock_lock(&cont->csl);
5113         cnt_free = TAILQ_FIRST(&cont->counters);
5114         if (cnt_free)
5115                 TAILQ_REMOVE(&cont->counters, cnt_free, next);
5116         rte_spinlock_unlock(&cont->csl);
5117         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free,
5118                                                        batch, age))
5119                 goto err;
5120         pool = cnt_free->pool;
5121         if (!batch)
5122                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free);
5123         /* Create a DV counter action only in the first time usage. */
5124         if (!cnt_free->action) {
5125                 uint16_t offset;
5126                 struct mlx5_devx_obj *dcs;
5127                 int ret;
5128
5129                 if (batch) {
5130                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
5131                         dcs = pool->min_dcs;
5132                 } else {
5133                         offset = 0;
5134                         dcs = cnt_ext->dcs;
5135                 }
5136                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
5137                                                             &cnt_free->action);
5138                 if (ret) {
5139                         rte_errno = errno;
5140                         goto err;
5141                 }
5142         }
5143         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
5144                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
5145         cnt_idx += batch * MLX5_CNT_BATCH_OFFSET;
5146         cnt_idx += age * MLX5_CNT_AGE_OFFSET;
5147         /* Update the counter reset values. */
5148         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
5149                                  &cnt_free->bytes))
5150                 goto err;
5151         if (cnt_ext) {
5152                 cnt_ext->shared = shared;
5153                 cnt_ext->ref_cnt = 1;
5154                 cnt_ext->id = id;
5155                 if (shared) {
5156                         union mlx5_l3t_data data;
5157
5158                         data.dword = cnt_idx;
5159                         if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
5160                                 return 0;
5161                 }
5162         }
5163         if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on)
5164                 /* Start the asynchronous batch query by the host thread. */
5165                 mlx5_set_query_alarm(priv->sh);
5166         return cnt_idx;
5167 err:
5168         if (cnt_free) {
5169                 cnt_free->pool = pool;
5170                 rte_spinlock_lock(&cont->csl);
5171                 TAILQ_INSERT_TAIL(&cont->counters, cnt_free, next);
5172                 rte_spinlock_unlock(&cont->csl);
5173         }
5174         return 0;
5175 }
5176
5177 /**
5178  * Get age param from counter index.
5179  *
5180  * @param[in] dev
5181  *   Pointer to the Ethernet device structure.
5182  * @param[in] counter
5183  *   Index to the counter handler.
5184  *
5185  * @return
5186  *   The aging parameter specified for the counter index.
5187  */
5188 static struct mlx5_age_param*
5189 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
5190                                 uint32_t counter)
5191 {
5192         struct mlx5_flow_counter *cnt;
5193         struct mlx5_flow_counter_pool *pool = NULL;
5194
5195         flow_dv_counter_get_by_idx(dev, counter, &pool);
5196         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
5197         cnt = MLX5_POOL_GET_CNT(pool, counter);
5198         return MLX5_CNT_TO_AGE(cnt);
5199 }
5200
5201 /**
5202  * Remove a flow counter from aged counter list.
5203  *
5204  * @param[in] dev
5205  *   Pointer to the Ethernet device structure.
5206  * @param[in] counter
5207  *   Index to the counter handler.
5208  * @param[in] cnt
5209  *   Pointer to the counter handler.
5210  */
5211 static void
5212 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
5213                                 uint32_t counter, struct mlx5_flow_counter *cnt)
5214 {
5215         struct mlx5_age_info *age_info;
5216         struct mlx5_age_param *age_param;
5217         struct mlx5_priv *priv = dev->data->dev_private;
5218
5219         age_info = GET_PORT_AGE_INFO(priv);
5220         age_param = flow_dv_counter_idx_get_age(dev, counter);
5221         if (rte_atomic16_cmpset((volatile uint16_t *)
5222                         &age_param->state,
5223                         AGE_CANDIDATE, AGE_FREE)
5224                         != AGE_CANDIDATE) {
5225                 /**
5226                  * We need the lock even it is age timeout,
5227                  * since counter may still in process.
5228                  */
5229                 rte_spinlock_lock(&age_info->aged_sl);
5230                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
5231                 rte_spinlock_unlock(&age_info->aged_sl);
5232         }
5233         rte_atomic16_set(&age_param->state, AGE_FREE);
5234 }
5235 /**
5236  * Release a flow counter.
5237  *
5238  * @param[in] dev
5239  *   Pointer to the Ethernet device structure.
5240  * @param[in] counter
5241  *   Index to the counter handler.
5242  */
5243 static void
5244 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
5245 {
5246         struct mlx5_priv *priv = dev->data->dev_private;
5247         struct mlx5_flow_counter_pool *pool = NULL;
5248         struct mlx5_flow_counter *cnt;
5249         struct mlx5_flow_counter_ext *cnt_ext = NULL;
5250
5251         if (!counter)
5252                 return;
5253         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
5254         MLX5_ASSERT(pool);
5255         if (counter < MLX5_CNT_BATCH_OFFSET) {
5256                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
5257                 if (cnt_ext) {
5258                         if (--cnt_ext->ref_cnt)
5259                                 return;
5260                         if (cnt_ext->shared)
5261                                 mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
5262                                                      cnt_ext->id);
5263                 }
5264         }
5265         if (IS_AGE_POOL(pool))
5266                 flow_dv_counter_remove_from_age(dev, counter, cnt);
5267         cnt->pool = pool;
5268         /*
5269          * Put the counter back to list to be updated in none fallback mode.
5270          * Currently, we are using two list alternately, while one is in query,
5271          * add the freed counter to the other list based on the pool query_gen
5272          * value. After query finishes, add counter the list to the global
5273          * container counter list. The list changes while query starts. In
5274          * this case, lock will not be needed as query callback and release
5275          * function both operate with the different list.
5276          *
5277          */
5278         if (!priv->counter_fallback)
5279                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
5280         else
5281                 TAILQ_INSERT_TAIL(&((MLX5_CNT_CONTAINER
5282                                   (priv->sh, 0, 0))->counters),
5283                                   cnt, next);
5284 }
5285
5286 /**
5287  * Verify the @p attributes will be correctly understood by the NIC and store
5288  * them in the @p flow if everything is correct.
5289  *
5290  * @param[in] dev
5291  *   Pointer to dev struct.
5292  * @param[in] attributes
5293  *   Pointer to flow attributes
5294  * @param[in] external
5295  *   This flow rule is created by request external to PMD.
5296  * @param[out] error
5297  *   Pointer to error structure.
5298  *
5299  * @return
5300  *   - 0 on success and non root table.
5301  *   - 1 on success and root table.
5302  *   - a negative errno value otherwise and rte_errno is set.
5303  */
5304 static int
5305 flow_dv_validate_attributes(struct rte_eth_dev *dev,
5306                             const struct rte_flow_attr *attributes,
5307                             bool external __rte_unused,
5308                             struct rte_flow_error *error)
5309 {
5310         struct mlx5_priv *priv = dev->data->dev_private;
5311         uint32_t priority_max = priv->config.flow_prio - 1;
5312         int ret = 0;
5313
5314 #ifndef HAVE_MLX5DV_DR
5315         if (attributes->group)
5316                 return rte_flow_error_set(error, ENOTSUP,
5317                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
5318                                           NULL,
5319                                           "groups are not supported");
5320 #else
5321         uint32_t table = 0;
5322
5323         ret = mlx5_flow_group_to_table(attributes, external,
5324                                        attributes->group, !!priv->fdb_def_rule,
5325                                        &table, error);
5326         if (ret)
5327                 return ret;
5328         if (!table)
5329                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
5330 #endif
5331         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
5332             attributes->priority >= priority_max)
5333                 return rte_flow_error_set(error, ENOTSUP,
5334                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
5335                                           NULL,
5336                                           "priority out of range");
5337         if (attributes->transfer) {
5338                 if (!priv->config.dv_esw_en)
5339                         return rte_flow_error_set
5340                                 (error, ENOTSUP,
5341                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5342                                  "E-Switch dr is not supported");
5343                 if (!(priv->representor || priv->master))
5344                         return rte_flow_error_set
5345                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
5346                                  NULL, "E-Switch configuration can only be"
5347                                  " done by a master or a representor device");
5348                 if (attributes->egress)
5349                         return rte_flow_error_set
5350                                 (error, ENOTSUP,
5351                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
5352                                  "egress is not supported");
5353         }
5354         if (!(attributes->egress ^ attributes->ingress))
5355                 return rte_flow_error_set(error, ENOTSUP,
5356                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
5357                                           "must specify exactly one of "
5358                                           "ingress or egress");
5359         return ret;
5360 }
5361
5362 /**
5363  * Internal validation function. For validating both actions and items.
5364  *
5365  * @param[in] dev
5366  *   Pointer to the rte_eth_dev structure.
5367  * @param[in] attr
5368  *   Pointer to the flow attributes.
5369  * @param[in] items
5370  *   Pointer to the list of items.
5371  * @param[in] actions
5372  *   Pointer to the list of actions.
5373  * @param[in] external
5374  *   This flow rule is created by request external to PMD.
5375  * @param[in] hairpin
5376  *   Number of hairpin TX actions, 0 means classic flow.
5377  * @param[out] error
5378  *   Pointer to the error structure.
5379  *
5380  * @return
5381  *   0 on success, a negative errno value otherwise and rte_errno is set.
5382  */
5383 static int
5384 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
5385                  const struct rte_flow_item items[],
5386                  const struct rte_flow_action actions[],
5387                  bool external, int hairpin, struct rte_flow_error *error)
5388 {
5389         int ret;
5390         uint64_t action_flags = 0;
5391         uint64_t item_flags = 0;
5392         uint64_t last_item = 0;
5393         uint8_t next_protocol = 0xff;
5394         uint16_t ether_type = 0;
5395         int actions_n = 0;
5396         uint8_t item_ipv6_proto = 0;
5397         const struct rte_flow_item *gre_item = NULL;
5398         const struct rte_flow_action_raw_decap *decap;
5399         const struct rte_flow_action_raw_encap *encap;
5400         const struct rte_flow_action_rss *rss;
5401         const struct rte_flow_item_tcp nic_tcp_mask = {
5402                 .hdr = {
5403                         .tcp_flags = 0xFF,
5404                         .src_port = RTE_BE16(UINT16_MAX),
5405                         .dst_port = RTE_BE16(UINT16_MAX),
5406                 }
5407         };
5408         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
5409                 .hdr = {
5410                         .src_addr =
5411                         "\xff\xff\xff\xff\xff\xff\xff\xff"
5412                         "\xff\xff\xff\xff\xff\xff\xff\xff",
5413                         .dst_addr =
5414                         "\xff\xff\xff\xff\xff\xff\xff\xff"
5415                         "\xff\xff\xff\xff\xff\xff\xff\xff",
5416                         .vtc_flow = RTE_BE32(0xffffffff),
5417                         .proto = 0xff,
5418                         .hop_limits = 0xff,
5419                 },
5420         };
5421         const struct rte_flow_item_ecpri nic_ecpri_mask = {
5422                 .hdr = {
5423                         .common = {
5424                                 .u32 =
5425                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
5426                                         .type = 0xFF,
5427                                         }).u32),
5428                         },
5429                         .dummy[0] = 0xffffffff,
5430                 },
5431         };
5432         struct mlx5_priv *priv = dev->data->dev_private;
5433         struct mlx5_dev_config *dev_conf = &priv->config;
5434         uint16_t queue_index = 0xFFFF;
5435         const struct rte_flow_item_vlan *vlan_m = NULL;
5436         int16_t rw_act_num = 0;
5437         uint64_t is_root;
5438
5439         if (items == NULL)
5440                 return -1;
5441         ret = flow_dv_validate_attributes(dev, attr, external, error);
5442         if (ret < 0)
5443                 return ret;
5444         is_root = (uint64_t)ret;
5445         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
5446                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
5447                 int type = items->type;
5448
5449                 if (!mlx5_flow_os_item_supported(type))
5450                         return rte_flow_error_set(error, ENOTSUP,
5451                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5452                                                   NULL, "item not supported");
5453                 switch (type) {
5454                 case RTE_FLOW_ITEM_TYPE_VOID:
5455                         break;
5456                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
5457                         ret = flow_dv_validate_item_port_id
5458                                         (dev, items, attr, item_flags, error);
5459                         if (ret < 0)
5460                                 return ret;
5461                         last_item = MLX5_FLOW_ITEM_PORT_ID;
5462                         break;
5463                 case RTE_FLOW_ITEM_TYPE_ETH:
5464                         ret = mlx5_flow_validate_item_eth(items, item_flags,
5465                                                           error);
5466                         if (ret < 0)
5467                                 return ret;
5468                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
5469                                              MLX5_FLOW_LAYER_OUTER_L2;
5470                         if (items->mask != NULL && items->spec != NULL) {
5471                                 ether_type =
5472                                         ((const struct rte_flow_item_eth *)
5473                                          items->spec)->type;
5474                                 ether_type &=
5475                                         ((const struct rte_flow_item_eth *)
5476                                          items->mask)->type;
5477                                 ether_type = rte_be_to_cpu_16(ether_type);
5478                         } else {
5479                                 ether_type = 0;
5480                         }
5481                         break;
5482                 case RTE_FLOW_ITEM_TYPE_VLAN:
5483                         ret = flow_dv_validate_item_vlan(items, item_flags,
5484                                                          dev, error);
5485                         if (ret < 0)
5486                                 return ret;
5487                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
5488                                              MLX5_FLOW_LAYER_OUTER_VLAN;
5489                         if (items->mask != NULL && items->spec != NULL) {
5490                                 ether_type =
5491                                         ((const struct rte_flow_item_vlan *)
5492                                          items->spec)->inner_type;
5493                                 ether_type &=
5494                                         ((const struct rte_flow_item_vlan *)
5495                                          items->mask)->inner_type;
5496                                 ether_type = rte_be_to_cpu_16(ether_type);
5497                         } else {
5498                                 ether_type = 0;
5499                         }
5500                         /* Store outer VLAN mask for of_push_vlan action. */
5501                         if (!tunnel)
5502                                 vlan_m = items->mask;
5503                         break;
5504                 case RTE_FLOW_ITEM_TYPE_IPV4:
5505                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5506                                                   &item_flags, &tunnel);
5507                         ret = flow_dv_validate_item_ipv4(items, item_flags,
5508                                                          last_item, ether_type,
5509                                                          error);
5510                         if (ret < 0)
5511                                 return ret;
5512                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5513                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5514                         if (items->mask != NULL &&
5515                             ((const struct rte_flow_item_ipv4 *)
5516                              items->mask)->hdr.next_proto_id) {
5517                                 next_protocol =
5518                                         ((const struct rte_flow_item_ipv4 *)
5519                                          (items->spec))->hdr.next_proto_id;
5520                                 next_protocol &=
5521                                         ((const struct rte_flow_item_ipv4 *)
5522                                          (items->mask))->hdr.next_proto_id;
5523                         } else {
5524                                 /* Reset for inner layer. */
5525                                 next_protocol = 0xff;
5526                         }
5527                         break;
5528                 case RTE_FLOW_ITEM_TYPE_IPV6:
5529                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5530                                                   &item_flags, &tunnel);
5531                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
5532                                                            last_item,
5533                                                            ether_type,
5534                                                            &nic_ipv6_mask,
5535                                                            error);
5536                         if (ret < 0)
5537                                 return ret;
5538                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5539                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5540                         if (items->mask != NULL &&
5541                             ((const struct rte_flow_item_ipv6 *)
5542                              items->mask)->hdr.proto) {
5543                                 item_ipv6_proto =
5544                                         ((const struct rte_flow_item_ipv6 *)
5545                                          items->spec)->hdr.proto;
5546                                 next_protocol =
5547                                         ((const struct rte_flow_item_ipv6 *)
5548                                          items->spec)->hdr.proto;
5549                                 next_protocol &=
5550                                         ((const struct rte_flow_item_ipv6 *)
5551                                          items->mask)->hdr.proto;
5552                         } else {
5553                                 /* Reset for inner layer. */
5554                                 next_protocol = 0xff;
5555                         }
5556                         break;
5557                 case RTE_FLOW_ITEM_TYPE_TCP:
5558                         ret = mlx5_flow_validate_item_tcp
5559                                                 (items, item_flags,
5560                                                  next_protocol,
5561                                                  &nic_tcp_mask,
5562                                                  error);
5563                         if (ret < 0)
5564                                 return ret;
5565                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5566                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5567                         break;
5568                 case RTE_FLOW_ITEM_TYPE_UDP:
5569                         ret = mlx5_flow_validate_item_udp(items, item_flags,
5570                                                           next_protocol,
5571                                                           error);
5572                         if (ret < 0)
5573                                 return ret;
5574                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5575                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5576                         break;
5577                 case RTE_FLOW_ITEM_TYPE_GRE:
5578                         ret = mlx5_flow_validate_item_gre(items, item_flags,
5579                                                           next_protocol, error);
5580                         if (ret < 0)
5581                                 return ret;
5582                         gre_item = items;
5583                         last_item = MLX5_FLOW_LAYER_GRE;
5584                         break;
5585                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5586                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
5587                                                             next_protocol,
5588                                                             error);
5589                         if (ret < 0)
5590                                 return ret;
5591                         last_item = MLX5_FLOW_LAYER_NVGRE;
5592                         break;
5593                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5594                         ret = mlx5_flow_validate_item_gre_key
5595                                 (items, item_flags, gre_item, error);
5596                         if (ret < 0)
5597                                 return ret;
5598                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5599                         break;
5600                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5601                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
5602                                                             error);
5603                         if (ret < 0)
5604                                 return ret;
5605                         last_item = MLX5_FLOW_LAYER_VXLAN;
5606                         break;
5607                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5608                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
5609                                                                 item_flags, dev,
5610                                                                 error);
5611                         if (ret < 0)
5612                                 return ret;
5613                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5614                         break;
5615                 case RTE_FLOW_ITEM_TYPE_GENEVE:
5616                         ret = mlx5_flow_validate_item_geneve(items,
5617                                                              item_flags, dev,
5618                                                              error);
5619                         if (ret < 0)
5620                                 return ret;
5621                         last_item = MLX5_FLOW_LAYER_GENEVE;
5622                         break;
5623                 case RTE_FLOW_ITEM_TYPE_MPLS:
5624                         ret = mlx5_flow_validate_item_mpls(dev, items,
5625                                                            item_flags,
5626                                                            last_item, error);
5627                         if (ret < 0)
5628                                 return ret;
5629                         last_item = MLX5_FLOW_LAYER_MPLS;
5630                         break;
5631
5632                 case RTE_FLOW_ITEM_TYPE_MARK:
5633                         ret = flow_dv_validate_item_mark(dev, items, attr,
5634                                                          error);
5635                         if (ret < 0)
5636                                 return ret;
5637                         last_item = MLX5_FLOW_ITEM_MARK;
5638                         break;
5639                 case RTE_FLOW_ITEM_TYPE_META:
5640                         ret = flow_dv_validate_item_meta(dev, items, attr,
5641                                                          error);
5642                         if (ret < 0)
5643                                 return ret;
5644                         last_item = MLX5_FLOW_ITEM_METADATA;
5645                         break;
5646                 case RTE_FLOW_ITEM_TYPE_ICMP:
5647                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
5648                                                            next_protocol,
5649                                                            error);
5650                         if (ret < 0)
5651                                 return ret;
5652                         last_item = MLX5_FLOW_LAYER_ICMP;
5653                         break;
5654                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5655                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
5656                                                             next_protocol,
5657                                                             error);
5658                         if (ret < 0)
5659                                 return ret;
5660                         item_ipv6_proto = IPPROTO_ICMPV6;
5661                         last_item = MLX5_FLOW_LAYER_ICMP6;
5662                         break;
5663                 case RTE_FLOW_ITEM_TYPE_TAG:
5664                         ret = flow_dv_validate_item_tag(dev, items,
5665                                                         attr, error);
5666                         if (ret < 0)
5667                                 return ret;
5668                         last_item = MLX5_FLOW_ITEM_TAG;
5669                         break;
5670                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
5671                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
5672                         break;
5673                 case RTE_FLOW_ITEM_TYPE_GTP:
5674                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
5675                                                         error);
5676                         if (ret < 0)
5677                                 return ret;
5678                         last_item = MLX5_FLOW_LAYER_GTP;
5679                         break;
5680                 case RTE_FLOW_ITEM_TYPE_ECPRI:
5681                         /* Capacity will be checked in the translate stage. */
5682                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
5683                                                             last_item,
5684                                                             ether_type,
5685                                                             &nic_ecpri_mask,
5686                                                             error);
5687                         if (ret < 0)
5688                                 return ret;
5689                         last_item = MLX5_FLOW_LAYER_ECPRI;
5690                         break;
5691                 default:
5692                         return rte_flow_error_set(error, ENOTSUP,
5693                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5694                                                   NULL, "item not supported");
5695                 }
5696                 item_flags |= last_item;
5697         }
5698         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5699                 int type = actions->type;
5700
5701                 if (!mlx5_flow_os_action_supported(type))
5702                         return rte_flow_error_set(error, ENOTSUP,
5703                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5704                                                   actions,
5705                                                   "action not supported");
5706                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5707                         return rte_flow_error_set(error, ENOTSUP,
5708                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5709                                                   actions, "too many actions");
5710                 switch (type) {
5711                 case RTE_FLOW_ACTION_TYPE_VOID:
5712                         break;
5713                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5714                         ret = flow_dv_validate_action_port_id(dev,
5715                                                               action_flags,
5716                                                               actions,
5717                                                               attr,
5718                                                               error);
5719                         if (ret)
5720                                 return ret;
5721                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5722                         ++actions_n;
5723                         break;
5724                 case RTE_FLOW_ACTION_TYPE_FLAG:
5725                         ret = flow_dv_validate_action_flag(dev, action_flags,
5726                                                            attr, error);
5727                         if (ret < 0)
5728                                 return ret;
5729                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5730                                 /* Count all modify-header actions as one. */
5731                                 if (!(action_flags &
5732                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5733                                         ++actions_n;
5734                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
5735                                                 MLX5_FLOW_ACTION_MARK_EXT;
5736                         } else {
5737                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
5738                                 ++actions_n;
5739                         }
5740                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5741                         break;
5742                 case RTE_FLOW_ACTION_TYPE_MARK:
5743                         ret = flow_dv_validate_action_mark(dev, actions,
5744                                                            action_flags,
5745                                                            attr, error);
5746                         if (ret < 0)
5747                                 return ret;
5748                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5749                                 /* Count all modify-header actions as one. */
5750                                 if (!(action_flags &
5751                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5752                                         ++actions_n;
5753                                 action_flags |= MLX5_FLOW_ACTION_MARK |
5754                                                 MLX5_FLOW_ACTION_MARK_EXT;
5755                         } else {
5756                                 action_flags |= MLX5_FLOW_ACTION_MARK;
5757                                 ++actions_n;
5758                         }
5759                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5760                         break;
5761                 case RTE_FLOW_ACTION_TYPE_SET_META:
5762                         ret = flow_dv_validate_action_set_meta(dev, actions,
5763                                                                action_flags,
5764                                                                attr, error);
5765                         if (ret < 0)
5766                                 return ret;
5767                         /* Count all modify-header actions as one action. */
5768                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5769                                 ++actions_n;
5770                         action_flags |= MLX5_FLOW_ACTION_SET_META;
5771                         rw_act_num += MLX5_ACT_NUM_SET_META;
5772                         break;
5773                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5774                         ret = flow_dv_validate_action_set_tag(dev, actions,
5775                                                               action_flags,
5776                                                               attr, error);
5777                         if (ret < 0)
5778                                 return ret;
5779                         /* Count all modify-header actions as one action. */
5780                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5781                                 ++actions_n;
5782                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
5783                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5784                         break;
5785                 case RTE_FLOW_ACTION_TYPE_DROP:
5786                         ret = mlx5_flow_validate_action_drop(action_flags,
5787                                                              attr, error);
5788                         if (ret < 0)
5789                                 return ret;
5790                         action_flags |= MLX5_FLOW_ACTION_DROP;
5791                         ++actions_n;
5792                         break;
5793                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5794                         ret = mlx5_flow_validate_action_queue(actions,
5795                                                               action_flags, dev,
5796                                                               attr, error);
5797                         if (ret < 0)
5798                                 return ret;
5799                         queue_index = ((const struct rte_flow_action_queue *)
5800                                                         (actions->conf))->index;
5801                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
5802                         ++actions_n;
5803                         break;
5804                 case RTE_FLOW_ACTION_TYPE_RSS:
5805                         rss = actions->conf;
5806                         ret = mlx5_flow_validate_action_rss(actions,
5807                                                             action_flags, dev,
5808                                                             attr, item_flags,
5809                                                             error);
5810                         if (ret < 0)
5811                                 return ret;
5812                         if (rss != NULL && rss->queue_num)
5813                                 queue_index = rss->queue[0];
5814                         action_flags |= MLX5_FLOW_ACTION_RSS;
5815                         ++actions_n;
5816                         break;
5817                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
5818                         ret =
5819                         mlx5_flow_validate_action_default_miss(action_flags,
5820                                         attr, error);
5821                         if (ret < 0)
5822                                 return ret;
5823                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
5824                         ++actions_n;
5825                         break;
5826                 case RTE_FLOW_ACTION_TYPE_COUNT:
5827                         ret = flow_dv_validate_action_count(dev, error);
5828                         if (ret < 0)
5829                                 return ret;
5830                         action_flags |= MLX5_FLOW_ACTION_COUNT;
5831                         ++actions_n;
5832                         break;
5833                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5834                         if (flow_dv_validate_action_pop_vlan(dev,
5835                                                              action_flags,
5836                                                              actions,
5837                                                              item_flags, attr,
5838                                                              error))
5839                                 return -rte_errno;
5840                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5841                         ++actions_n;
5842                         break;
5843                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5844                         ret = flow_dv_validate_action_push_vlan(dev,
5845                                                                 action_flags,
5846                                                                 vlan_m,
5847                                                                 actions, attr,
5848                                                                 error);
5849                         if (ret < 0)
5850                                 return ret;
5851                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5852                         ++actions_n;
5853                         break;
5854                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5855                         ret = flow_dv_validate_action_set_vlan_pcp
5856                                                 (action_flags, actions, error);
5857                         if (ret < 0)
5858                                 return ret;
5859                         /* Count PCP with push_vlan command. */
5860                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
5861                         break;
5862                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5863                         ret = flow_dv_validate_action_set_vlan_vid
5864                                                 (item_flags, action_flags,
5865                                                  actions, error);
5866                         if (ret < 0)
5867                                 return ret;
5868                         /* Count VID with push_vlan command. */
5869                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
5870                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
5871                         break;
5872                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5873                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5874                         ret = flow_dv_validate_action_l2_encap(dev,
5875                                                                action_flags,
5876                                                                actions, attr,
5877                                                                error);
5878                         if (ret < 0)
5879                                 return ret;
5880                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
5881                         ++actions_n;
5882                         break;
5883                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5884                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5885                         ret = flow_dv_validate_action_decap(dev, action_flags,
5886                                                             attr, error);
5887                         if (ret < 0)
5888                                 return ret;
5889                         action_flags |= MLX5_FLOW_ACTION_DECAP;
5890                         ++actions_n;
5891                         break;
5892                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5893                         ret = flow_dv_validate_action_raw_encap_decap
5894                                 (dev, NULL, actions->conf, attr, &action_flags,
5895                                  &actions_n, error);
5896                         if (ret < 0)
5897                                 return ret;
5898                         break;
5899                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5900                         decap = actions->conf;
5901                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5902                                 ;
5903                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5904                                 encap = NULL;
5905                                 actions--;
5906                         } else {
5907                                 encap = actions->conf;
5908                         }
5909                         ret = flow_dv_validate_action_raw_encap_decap
5910                                            (dev,
5911                                             decap ? decap : &empty_decap, encap,
5912                                             attr, &action_flags, &actions_n,
5913                                             error);
5914                         if (ret < 0)
5915                                 return ret;
5916                         break;
5917                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5918                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5919                         ret = flow_dv_validate_action_modify_mac(action_flags,
5920                                                                  actions,
5921                                                                  item_flags,
5922                                                                  error);
5923                         if (ret < 0)
5924                                 return ret;
5925                         /* Count all modify-header actions as one action. */
5926                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5927                                 ++actions_n;
5928                         action_flags |= actions->type ==
5929                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5930                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
5931                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
5932                         /*
5933                          * Even if the source and destination MAC addresses have
5934                          * overlap in the header with 4B alignment, the convert
5935                          * function will handle them separately and 4 SW actions
5936                          * will be created. And 2 actions will be added each
5937                          * time no matter how many bytes of address will be set.
5938                          */
5939                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
5940                         break;
5941                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5942                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5943                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
5944                                                                   actions,
5945                                                                   item_flags,
5946                                                                   error);
5947                         if (ret < 0)
5948                                 return ret;
5949                         /* Count all modify-header actions as one action. */
5950                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5951                                 ++actions_n;
5952                         action_flags |= actions->type ==
5953                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5954                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
5955                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
5956                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
5957                         break;
5958                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5959                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5960                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
5961                                                                   actions,
5962                                                                   item_flags,
5963                                                                   error);
5964                         if (ret < 0)
5965                                 return ret;
5966                         if (item_ipv6_proto == IPPROTO_ICMPV6)
5967                                 return rte_flow_error_set(error, ENOTSUP,
5968                                         RTE_FLOW_ERROR_TYPE_ACTION,
5969                                         actions,
5970                                         "Can't change header "
5971                                         "with ICMPv6 proto");
5972                         /* Count all modify-header actions as one action. */
5973                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5974                                 ++actions_n;
5975                         action_flags |= actions->type ==
5976                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5977                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
5978                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
5979                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
5980                         break;
5981                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5982                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5983                         ret = flow_dv_validate_action_modify_tp(action_flags,
5984                                                                 actions,
5985                                                                 item_flags,
5986                                                                 error);
5987                         if (ret < 0)
5988                                 return ret;
5989                         /* Count all modify-header actions as one action. */
5990                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5991                                 ++actions_n;
5992                         action_flags |= actions->type ==
5993                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5994                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
5995                                                 MLX5_FLOW_ACTION_SET_TP_DST;
5996                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
5997                         break;
5998                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5999                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
6000                         ret = flow_dv_validate_action_modify_ttl(action_flags,
6001                                                                  actions,
6002                                                                  item_flags,
6003                                                                  error);
6004                         if (ret < 0)
6005                                 return ret;
6006                         /* Count all modify-header actions as one action. */
6007                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6008                                 ++actions_n;
6009                         action_flags |= actions->type ==
6010                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
6011                                                 MLX5_FLOW_ACTION_SET_TTL :
6012                                                 MLX5_FLOW_ACTION_DEC_TTL;
6013                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
6014                         break;
6015                 case RTE_FLOW_ACTION_TYPE_JUMP:
6016                         ret = flow_dv_validate_action_jump(actions,
6017                                                            action_flags,
6018                                                            attr, external,
6019                                                            error);
6020                         if (ret)
6021                                 return ret;
6022                         ++actions_n;
6023                         action_flags |= MLX5_FLOW_ACTION_JUMP;
6024                         break;
6025                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
6026                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
6027                         ret = flow_dv_validate_action_modify_tcp_seq
6028                                                                 (action_flags,
6029                                                                  actions,
6030                                                                  item_flags,
6031                                                                  error);
6032                         if (ret < 0)
6033                                 return ret;
6034                         /* Count all modify-header actions as one action. */
6035                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6036                                 ++actions_n;
6037                         action_flags |= actions->type ==
6038                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
6039                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
6040                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
6041                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
6042                         break;
6043                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
6044                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
6045                         ret = flow_dv_validate_action_modify_tcp_ack
6046                                                                 (action_flags,
6047                                                                  actions,
6048                                                                  item_flags,
6049                                                                  error);
6050                         if (ret < 0)
6051                                 return ret;
6052                         /* Count all modify-header actions as one action. */
6053                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6054                                 ++actions_n;
6055                         action_flags |= actions->type ==
6056                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
6057                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
6058                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
6059                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
6060                         break;
6061                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
6062                         break;
6063                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
6064                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
6065                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
6066                         break;
6067                 case RTE_FLOW_ACTION_TYPE_METER:
6068                         ret = mlx5_flow_validate_action_meter(dev,
6069                                                               action_flags,
6070                                                               actions, attr,
6071                                                               error);
6072                         if (ret < 0)
6073                                 return ret;
6074                         action_flags |= MLX5_FLOW_ACTION_METER;
6075                         ++actions_n;
6076                         /* Meter action will add one more TAG action. */
6077                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
6078                         break;
6079                 case RTE_FLOW_ACTION_TYPE_AGE:
6080                         ret = flow_dv_validate_action_age(action_flags,
6081                                                           actions, dev,
6082                                                           error);
6083                         if (ret < 0)
6084                                 return ret;
6085                         action_flags |= MLX5_FLOW_ACTION_AGE;
6086                         ++actions_n;
6087                         break;
6088                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
6089                         ret = flow_dv_validate_action_modify_ipv4_dscp
6090                                                          (action_flags,
6091                                                           actions,
6092                                                           item_flags,
6093                                                           error);
6094                         if (ret < 0)
6095                                 return ret;
6096                         /* Count all modify-header actions as one action. */
6097                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6098                                 ++actions_n;
6099                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
6100                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
6101                         break;
6102                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
6103                         ret = flow_dv_validate_action_modify_ipv6_dscp
6104                                                                 (action_flags,
6105                                                                  actions,
6106                                                                  item_flags,
6107                                                                  error);
6108                         if (ret < 0)
6109                                 return ret;
6110                         /* Count all modify-header actions as one action. */
6111                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
6112                                 ++actions_n;
6113                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
6114                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
6115                         break;
6116                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
6117                         ret = flow_dv_validate_action_sample(action_flags,
6118                                                              actions, dev,
6119                                                              attr, error);
6120                         if (ret < 0)
6121                                 return ret;
6122                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
6123                         ++actions_n;
6124                         break;
6125                 default:
6126                         return rte_flow_error_set(error, ENOTSUP,
6127                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6128                                                   actions,
6129                                                   "action not supported");
6130                 }
6131         }
6132         /*
6133          * Validate the drop action mutual exclusion with other actions.
6134          * Drop action is mutually-exclusive with any other action, except for
6135          * Count action.
6136          */
6137         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
6138             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
6139                 return rte_flow_error_set(error, EINVAL,
6140                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
6141                                           "Drop action is mutually-exclusive "
6142                                           "with any other action, except for "
6143                                           "Count action");
6144         /* Eswitch has few restrictions on using items and actions */
6145         if (attr->transfer) {
6146                 if (!mlx5_flow_ext_mreg_supported(dev) &&
6147                     action_flags & MLX5_FLOW_ACTION_FLAG)
6148                         return rte_flow_error_set(error, ENOTSUP,
6149                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6150                                                   NULL,
6151                                                   "unsupported action FLAG");
6152                 if (!mlx5_flow_ext_mreg_supported(dev) &&
6153                     action_flags & MLX5_FLOW_ACTION_MARK)
6154                         return rte_flow_error_set(error, ENOTSUP,
6155                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6156                                                   NULL,
6157                                                   "unsupported action MARK");
6158                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
6159                         return rte_flow_error_set(error, ENOTSUP,
6160                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6161                                                   NULL,
6162                                                   "unsupported action QUEUE");
6163                 if (action_flags & MLX5_FLOW_ACTION_RSS)
6164                         return rte_flow_error_set(error, ENOTSUP,
6165                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6166                                                   NULL,
6167                                                   "unsupported action RSS");
6168                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
6169                         return rte_flow_error_set(error, EINVAL,
6170                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6171                                                   actions,
6172                                                   "no fate action is found");
6173         } else {
6174                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
6175                         return rte_flow_error_set(error, EINVAL,
6176                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6177                                                   actions,
6178                                                   "no fate action is found");
6179         }
6180         /* Continue validation for Xcap and VLAN actions.*/
6181         if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
6182                              MLX5_FLOW_VLAN_ACTIONS)) &&
6183             (queue_index == 0xFFFF ||
6184              mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
6185                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
6186                     MLX5_FLOW_XCAP_ACTIONS)
6187                         return rte_flow_error_set(error, ENOTSUP,
6188                                                   RTE_FLOW_ERROR_TYPE_ACTION,
6189                                                   NULL, "encap and decap "
6190                                                   "combination aren't supported");
6191                 if (!attr->transfer && attr->ingress) {
6192                         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
6193                                 return rte_flow_error_set
6194                                                 (error, ENOTSUP,
6195                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6196                                                  NULL, "encap is not supported"
6197                                                  " for ingress traffic");
6198                         else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
6199                                 return rte_flow_error_set
6200                                                 (error, ENOTSUP,
6201                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6202                                                  NULL, "push VLAN action not "
6203                                                  "supported for ingress");
6204                         else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
6205                                         MLX5_FLOW_VLAN_ACTIONS)
6206                                 return rte_flow_error_set
6207                                                 (error, ENOTSUP,
6208                                                  RTE_FLOW_ERROR_TYPE_ACTION,
6209                                                  NULL, "no support for "
6210                                                  "multiple VLAN actions");
6211                 }
6212         }
6213         /* Hairpin flow will add one more TAG action. */
6214         if (hairpin > 0)
6215                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6216         /* extra metadata enabled: one more TAG action will be add. */
6217         if (dev_conf->dv_flow_en &&
6218             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
6219             mlx5_flow_ext_mreg_supported(dev))
6220                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
6221         if ((uint32_t)rw_act_num >
6222                         flow_dv_modify_hdr_action_max(dev, is_root)) {
6223                 return rte_flow_error_set(error, ENOTSUP,
6224                                           RTE_FLOW_ERROR_TYPE_ACTION,
6225                                           NULL, "too many header modify"
6226                                           " actions to support");
6227         }
6228         return 0;
6229 }
6230
6231 /**
6232  * Internal preparation function. Allocates the DV flow size,
6233  * this size is constant.
6234  *
6235  * @param[in] dev
6236  *   Pointer to the rte_eth_dev structure.
6237  * @param[in] attr
6238  *   Pointer to the flow attributes.
6239  * @param[in] items
6240  *   Pointer to the list of items.
6241  * @param[in] actions
6242  *   Pointer to the list of actions.
6243  * @param[out] error
6244  *   Pointer to the error structure.
6245  *
6246  * @return
6247  *   Pointer to mlx5_flow object on success,
6248  *   otherwise NULL and rte_errno is set.
6249  */
6250 static struct mlx5_flow *
6251 flow_dv_prepare(struct rte_eth_dev *dev,
6252                 const struct rte_flow_attr *attr __rte_unused,
6253                 const struct rte_flow_item items[] __rte_unused,
6254                 const struct rte_flow_action actions[] __rte_unused,
6255                 struct rte_flow_error *error)
6256 {
6257         uint32_t handle_idx = 0;
6258         struct mlx5_flow *dev_flow;
6259         struct mlx5_flow_handle *dev_handle;
6260         struct mlx5_priv *priv = dev->data->dev_private;
6261
6262         /* In case of corrupting the memory. */
6263         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
6264                 rte_flow_error_set(error, ENOSPC,
6265                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6266                                    "not free temporary device flow");
6267                 return NULL;
6268         }
6269         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
6270                                    &handle_idx);
6271         if (!dev_handle) {
6272                 rte_flow_error_set(error, ENOMEM,
6273                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
6274                                    "not enough memory to create flow handle");
6275                 return NULL;
6276         }
6277         /* No multi-thread supporting. */
6278         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
6279         dev_flow->handle = dev_handle;
6280         dev_flow->handle_idx = handle_idx;
6281         /*
6282          * In some old rdma-core releases, before continuing, a check of the
6283          * length of matching parameter will be done at first. It needs to use
6284          * the length without misc4 param. If the flow has misc4 support, then
6285          * the length needs to be adjusted accordingly. Each param member is
6286          * aligned with a 64B boundary naturally.
6287          */
6288         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) -
6289                                   MLX5_ST_SZ_BYTES(fte_match_set_misc4);
6290         /*
6291          * The matching value needs to be cleared to 0 before using. In the
6292          * past, it will be automatically cleared when using rte_*alloc
6293          * API. The time consumption will be almost the same as before.
6294          */
6295         memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
6296         dev_flow->ingress = attr->ingress;
6297         dev_flow->dv.transfer = attr->transfer;
6298         return dev_flow;
6299 }
6300
6301 #ifdef RTE_LIBRTE_MLX5_DEBUG
6302 /**
6303  * Sanity check for match mask and value. Similar to check_valid_spec() in
6304  * kernel driver. If unmasked bit is present in value, it returns failure.
6305  *
6306  * @param match_mask
6307  *   pointer to match mask buffer.
6308  * @param match_value
6309  *   pointer to match value buffer.
6310  *
6311  * @return
6312  *   0 if valid, -EINVAL otherwise.
6313  */
6314 static int
6315 flow_dv_check_valid_spec(void *match_mask, void *match_value)
6316 {
6317         uint8_t *m = match_mask;
6318         uint8_t *v = match_value;
6319         unsigned int i;
6320
6321         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
6322                 if (v[i] & ~m[i]) {
6323                         DRV_LOG(ERR,
6324                                 "match_value differs from match_criteria"
6325                                 " %p[%u] != %p[%u]",
6326                                 match_value, i, match_mask, i);
6327                         return -EINVAL;
6328                 }
6329         }
6330         return 0;
6331 }
6332 #endif
6333
6334 /**
6335  * Add match of ip_version.
6336  *
6337  * @param[in] group
6338  *   Flow group.
6339  * @param[in] headers_v
6340  *   Values header pointer.
6341  * @param[in] headers_m
6342  *   Masks header pointer.
6343  * @param[in] ip_version
6344  *   The IP version to set.
6345  */
6346 static inline void
6347 flow_dv_set_match_ip_version(uint32_t group,
6348                              void *headers_v,
6349                              void *headers_m,
6350                              uint8_t ip_version)
6351 {
6352         if (group == 0)
6353                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
6354         else
6355                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
6356                          ip_version);
6357         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
6358         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
6359         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
6360 }
6361
6362 /**
6363  * Add Ethernet item to matcher and to the value.
6364  *
6365  * @param[in, out] matcher
6366  *   Flow matcher.
6367  * @param[in, out] key
6368  *   Flow matcher value.
6369  * @param[in] item
6370  *   Flow pattern to translate.
6371  * @param[in] inner
6372  *   Item is inner pattern.
6373  */
6374 static void
6375 flow_dv_translate_item_eth(void *matcher, void *key,
6376                            const struct rte_flow_item *item, int inner,
6377                            uint32_t group)
6378 {
6379         const struct rte_flow_item_eth *eth_m = item->mask;
6380         const struct rte_flow_item_eth *eth_v = item->spec;
6381         const struct rte_flow_item_eth nic_mask = {
6382                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6383                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
6384                 .type = RTE_BE16(0xffff),
6385         };
6386         void *headers_m;
6387         void *headers_v;
6388         char *l24_v;
6389         unsigned int i;
6390
6391         if (!eth_v)
6392                 return;
6393         if (!eth_m)
6394                 eth_m = &nic_mask;
6395         if (inner) {
6396                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6397                                          inner_headers);
6398                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6399         } else {
6400                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6401                                          outer_headers);
6402                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6403         }
6404         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
6405                &eth_m->dst, sizeof(eth_m->dst));
6406         /* The value must be in the range of the mask. */
6407         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
6408         for (i = 0; i < sizeof(eth_m->dst); ++i)
6409                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
6410         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
6411                &eth_m->src, sizeof(eth_m->src));
6412         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
6413         /* The value must be in the range of the mask. */
6414         for (i = 0; i < sizeof(eth_m->dst); ++i)
6415                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
6416         if (eth_v->type) {
6417                 /* When ethertype is present set mask for tagged VLAN. */
6418                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6419                 /* Set value for tagged VLAN if ethertype is 802.1Q. */
6420                 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
6421                     eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
6422                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag,
6423                                  1);
6424                         /* Return here to avoid setting match on ethertype. */
6425                         return;
6426                 }
6427         }
6428         /*
6429          * HW supports match on one Ethertype, the Ethertype following the last
6430          * VLAN tag of the packet (see PRM).
6431          * Set match on ethertype only if ETH header is not followed by VLAN.
6432          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6433          * ethertype, and use ip_version field instead.
6434          * eCPRI over Ether layer will use type value 0xAEFE.
6435          */
6436         if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6437             eth_m->type == 0xFFFF) {
6438                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6439         } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6440                    eth_m->type == 0xFFFF) {
6441                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6442         } else {
6443                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6444                          rte_be_to_cpu_16(eth_m->type));
6445                 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6446                                      ethertype);
6447                 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
6448         }
6449 }
6450
6451 /**
6452  * Add VLAN item to matcher and to the value.
6453  *
6454  * @param[in, out] dev_flow
6455  *   Flow descriptor.
6456  * @param[in, out] matcher
6457  *   Flow matcher.
6458  * @param[in, out] key
6459  *   Flow matcher value.
6460  * @param[in] item
6461  *   Flow pattern to translate.
6462  * @param[in] inner
6463  *   Item is inner pattern.
6464  */
6465 static void
6466 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
6467                             void *matcher, void *key,
6468                             const struct rte_flow_item *item,
6469                             int inner, uint32_t group)
6470 {
6471         const struct rte_flow_item_vlan *vlan_m = item->mask;
6472         const struct rte_flow_item_vlan *vlan_v = item->spec;
6473         void *headers_m;
6474         void *headers_v;
6475         uint16_t tci_m;
6476         uint16_t tci_v;
6477
6478         if (inner) {
6479                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6480                                          inner_headers);
6481                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6482         } else {
6483                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6484                                          outer_headers);
6485                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6486                 /*
6487                  * This is workaround, masks are not supported,
6488                  * and pre-validated.
6489                  */
6490                 if (vlan_v)
6491                         dev_flow->handle->vf_vlan.tag =
6492                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
6493         }
6494         /*
6495          * When VLAN item exists in flow, mark packet as tagged,
6496          * even if TCI is not specified.
6497          */
6498         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6499         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
6500         if (!vlan_v)
6501                 return;
6502         if (!vlan_m)
6503                 vlan_m = &rte_flow_item_vlan_mask;
6504         tci_m = rte_be_to_cpu_16(vlan_m->tci);
6505         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
6506         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
6507         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
6508         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
6509         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
6510         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
6511         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
6512         /*
6513          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6514          * ethertype, and use ip_version field instead.
6515          */
6516         if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6517             vlan_m->inner_type == 0xFFFF) {
6518                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6519         } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6520                    vlan_m->inner_type == 0xFFFF) {
6521                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6522         } else {
6523                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6524                          rte_be_to_cpu_16(vlan_m->inner_type));
6525                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
6526                          rte_be_to_cpu_16(vlan_m->inner_type &
6527                                           vlan_v->inner_type));
6528         }
6529 }
6530
6531 /**
6532  * Add IPV4 item to matcher and to the value.
6533  *
6534  * @param[in, out] matcher
6535  *   Flow matcher.
6536  * @param[in, out] key
6537  *   Flow matcher value.
6538  * @param[in] item
6539  *   Flow pattern to translate.
6540  * @param[in] item_flags
6541  *   Bit-fields that holds the items detected until now.
6542  * @param[in] inner
6543  *   Item is inner pattern.
6544  * @param[in] group
6545  *   The group to insert the rule.
6546  */
6547 static void
6548 flow_dv_translate_item_ipv4(void *matcher, void *key,
6549                             const struct rte_flow_item *item,
6550                             const uint64_t item_flags,
6551                             int inner, uint32_t group)
6552 {
6553         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
6554         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
6555         const struct rte_flow_item_ipv4 nic_mask = {
6556                 .hdr = {
6557                         .src_addr = RTE_BE32(0xffffffff),
6558                         .dst_addr = RTE_BE32(0xffffffff),
6559                         .type_of_service = 0xff,
6560                         .next_proto_id = 0xff,
6561                         .time_to_live = 0xff,
6562                 },
6563         };
6564         void *headers_m;
6565         void *headers_v;
6566         char *l24_m;
6567         char *l24_v;
6568         uint8_t tos;
6569
6570         if (inner) {
6571                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6572                                          inner_headers);
6573                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6574         } else {
6575                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6576                                          outer_headers);
6577                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6578         }
6579         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6580         /*
6581          * On outer header (which must contains L2), or inner header with L2,
6582          * set cvlan_tag mask bit to mark this packet as untagged.
6583          * This should be done even if item->spec is empty.
6584          */
6585         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6586                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6587         if (!ipv4_v)
6588                 return;
6589         if (!ipv4_m)
6590                 ipv4_m = &nic_mask;
6591         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6592                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6593         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6594                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6595         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
6596         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
6597         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6598                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6599         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6600                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6601         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
6602         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
6603         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
6604         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
6605                  ipv4_m->hdr.type_of_service);
6606         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
6607         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
6608                  ipv4_m->hdr.type_of_service >> 2);
6609         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
6610         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6611                  ipv4_m->hdr.next_proto_id);
6612         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6613                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
6614         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6615                  ipv4_m->hdr.time_to_live);
6616         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6617                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
6618         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag,
6619                  !!(ipv4_m->hdr.fragment_offset));
6620         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
6621                  !!(ipv4_v->hdr.fragment_offset & ipv4_m->hdr.fragment_offset));
6622 }
6623
6624 /**
6625  * Add IPV6 item to matcher and to the value.
6626  *
6627  * @param[in, out] matcher
6628  *   Flow matcher.
6629  * @param[in, out] key
6630  *   Flow matcher value.
6631  * @param[in] item
6632  *   Flow pattern to translate.
6633  * @param[in] item_flags
6634  *   Bit-fields that holds the items detected until now.
6635  * @param[in] inner
6636  *   Item is inner pattern.
6637  * @param[in] group
6638  *   The group to insert the rule.
6639  */
6640 static void
6641 flow_dv_translate_item_ipv6(void *matcher, void *key,
6642                             const struct rte_flow_item *item,
6643                             const uint64_t item_flags,
6644                             int inner, uint32_t group)
6645 {
6646         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
6647         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
6648         const struct rte_flow_item_ipv6 nic_mask = {
6649                 .hdr = {
6650                         .src_addr =
6651                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6652                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6653                         .dst_addr =
6654                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6655                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6656                         .vtc_flow = RTE_BE32(0xffffffff),
6657                         .proto = 0xff,
6658                         .hop_limits = 0xff,
6659                 },
6660         };
6661         void *headers_m;
6662         void *headers_v;
6663         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6664         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6665         char *l24_m;
6666         char *l24_v;
6667         uint32_t vtc_m;
6668         uint32_t vtc_v;
6669         int i;
6670         int size;
6671
6672         if (inner) {
6673                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6674                                          inner_headers);
6675                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6676         } else {
6677                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6678                                          outer_headers);
6679                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6680         }
6681         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6682         /*
6683          * On outer header (which must contains L2), or inner header with L2,
6684          * set cvlan_tag mask bit to mark this packet as untagged.
6685          * This should be done even if item->spec is empty.
6686          */
6687         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6688                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6689         if (!ipv6_v)
6690                 return;
6691         if (!ipv6_m)
6692                 ipv6_m = &nic_mask;
6693         size = sizeof(ipv6_m->hdr.dst_addr);
6694         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6695                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6696         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6697                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6698         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
6699         for (i = 0; i < size; ++i)
6700                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
6701         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6702                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6703         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6704                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6705         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
6706         for (i = 0; i < size; ++i)
6707                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
6708         /* TOS. */
6709         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
6710         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
6711         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
6712         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
6713         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
6714         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
6715         /* Label. */
6716         if (inner) {
6717                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
6718                          vtc_m);
6719                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
6720                          vtc_v);
6721         } else {
6722                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
6723                          vtc_m);
6724                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
6725                          vtc_v);
6726         }
6727         /* Protocol. */
6728         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6729                  ipv6_m->hdr.proto);
6730         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6731                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
6732         /* Hop limit. */
6733         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6734                  ipv6_m->hdr.hop_limits);
6735         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6736                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
6737 }
6738
6739 /**
6740  * Add TCP item to matcher and to the value.
6741  *
6742  * @param[in, out] matcher
6743  *   Flow matcher.
6744  * @param[in, out] key
6745  *   Flow matcher value.
6746  * @param[in] item
6747  *   Flow pattern to translate.
6748  * @param[in] inner
6749  *   Item is inner pattern.
6750  */
6751 static void
6752 flow_dv_translate_item_tcp(void *matcher, void *key,
6753                            const struct rte_flow_item *item,
6754                            int inner)
6755 {
6756         const struct rte_flow_item_tcp *tcp_m = item->mask;
6757         const struct rte_flow_item_tcp *tcp_v = item->spec;
6758         void *headers_m;
6759         void *headers_v;
6760
6761         if (inner) {
6762                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6763                                          inner_headers);
6764                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6765         } else {
6766                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6767                                          outer_headers);
6768                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6769         }
6770         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6771         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
6772         if (!tcp_v)
6773                 return;
6774         if (!tcp_m)
6775                 tcp_m = &rte_flow_item_tcp_mask;
6776         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
6777                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
6778         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
6779                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
6780         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
6781                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
6782         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
6783                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
6784         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
6785                  tcp_m->hdr.tcp_flags);
6786         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
6787                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
6788 }
6789
6790 /**
6791  * Add UDP item to matcher and to the value.
6792  *
6793  * @param[in, out] matcher
6794  *   Flow matcher.
6795  * @param[in, out] key
6796  *   Flow matcher value.
6797  * @param[in] item
6798  *   Flow pattern to translate.
6799  * @param[in] inner
6800  *   Item is inner pattern.
6801  */
6802 static void
6803 flow_dv_translate_item_udp(void *matcher, void *key,
6804                            const struct rte_flow_item *item,
6805                            int inner)
6806 {
6807         const struct rte_flow_item_udp *udp_m = item->mask;
6808         const struct rte_flow_item_udp *udp_v = item->spec;
6809         void *headers_m;
6810         void *headers_v;
6811
6812         if (inner) {
6813                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6814                                          inner_headers);
6815                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6816         } else {
6817                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6818                                          outer_headers);
6819                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6820         }
6821         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6822         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
6823         if (!udp_v)
6824                 return;
6825         if (!udp_m)
6826                 udp_m = &rte_flow_item_udp_mask;
6827         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
6828                  rte_be_to_cpu_16(udp_m->hdr.src_port));
6829         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
6830                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
6831         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
6832                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
6833         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6834                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
6835 }
6836
6837 /**
6838  * Add GRE optional Key item to matcher and to the value.
6839  *
6840  * @param[in, out] matcher
6841  *   Flow matcher.
6842  * @param[in, out] key
6843  *   Flow matcher value.
6844  * @param[in] item
6845  *   Flow pattern to translate.
6846  * @param[in] inner
6847  *   Item is inner pattern.
6848  */
6849 static void
6850 flow_dv_translate_item_gre_key(void *matcher, void *key,
6851                                    const struct rte_flow_item *item)
6852 {
6853         const rte_be32_t *key_m = item->mask;
6854         const rte_be32_t *key_v = item->spec;
6855         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6856         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6857         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
6858
6859         /* GRE K bit must be on and should already be validated */
6860         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
6861         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
6862         if (!key_v)
6863                 return;
6864         if (!key_m)
6865                 key_m = &gre_key_default_mask;
6866         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
6867                  rte_be_to_cpu_32(*key_m) >> 8);
6868         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
6869                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
6870         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
6871                  rte_be_to_cpu_32(*key_m) & 0xFF);
6872         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
6873                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
6874 }
6875
6876 /**
6877  * Add GRE item to matcher and to the value.
6878  *
6879  * @param[in, out] matcher
6880  *   Flow matcher.
6881  * @param[in, out] key
6882  *   Flow matcher value.
6883  * @param[in] item
6884  *   Flow pattern to translate.
6885  * @param[in] inner
6886  *   Item is inner pattern.
6887  */
6888 static void
6889 flow_dv_translate_item_gre(void *matcher, void *key,
6890                            const struct rte_flow_item *item,
6891                            int inner)
6892 {
6893         const struct rte_flow_item_gre *gre_m = item->mask;
6894         const struct rte_flow_item_gre *gre_v = item->spec;
6895         void *headers_m;
6896         void *headers_v;
6897         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6898         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6899         struct {
6900                 union {
6901                         __extension__
6902                         struct {
6903                                 uint16_t version:3;
6904                                 uint16_t rsvd0:9;
6905                                 uint16_t s_present:1;
6906                                 uint16_t k_present:1;
6907                                 uint16_t rsvd_bit1:1;
6908                                 uint16_t c_present:1;
6909                         };
6910                         uint16_t value;
6911                 };
6912         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
6913
6914         if (inner) {
6915                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6916                                          inner_headers);
6917                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6918         } else {
6919                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6920                                          outer_headers);
6921                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6922         }
6923         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6924         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
6925         if (!gre_v)
6926                 return;
6927         if (!gre_m)
6928                 gre_m = &rte_flow_item_gre_mask;
6929         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
6930                  rte_be_to_cpu_16(gre_m->protocol));
6931         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6932                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
6933         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
6934         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
6935         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
6936                  gre_crks_rsvd0_ver_m.c_present);
6937         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
6938                  gre_crks_rsvd0_ver_v.c_present &
6939                  gre_crks_rsvd0_ver_m.c_present);
6940         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
6941                  gre_crks_rsvd0_ver_m.k_present);
6942         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
6943                  gre_crks_rsvd0_ver_v.k_present &
6944                  gre_crks_rsvd0_ver_m.k_present);
6945         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
6946                  gre_crks_rsvd0_ver_m.s_present);
6947         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
6948                  gre_crks_rsvd0_ver_v.s_present &
6949                  gre_crks_rsvd0_ver_m.s_present);
6950 }
6951
6952 /**
6953  * Add NVGRE item to matcher and to the value.
6954  *
6955  * @param[in, out] matcher
6956  *   Flow matcher.
6957  * @param[in, out] key
6958  *   Flow matcher value.
6959  * @param[in] item
6960  *   Flow pattern to translate.
6961  * @param[in] inner
6962  *   Item is inner pattern.
6963  */
6964 static void
6965 flow_dv_translate_item_nvgre(void *matcher, void *key,
6966                              const struct rte_flow_item *item,
6967                              int inner)
6968 {
6969         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
6970         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
6971         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6972         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6973         const char *tni_flow_id_m;
6974         const char *tni_flow_id_v;
6975         char *gre_key_m;
6976         char *gre_key_v;
6977         int size;
6978         int i;
6979
6980         /* For NVGRE, GRE header fields must be set with defined values. */
6981         const struct rte_flow_item_gre gre_spec = {
6982                 .c_rsvd0_ver = RTE_BE16(0x2000),
6983                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
6984         };
6985         const struct rte_flow_item_gre gre_mask = {
6986                 .c_rsvd0_ver = RTE_BE16(0xB000),
6987                 .protocol = RTE_BE16(UINT16_MAX),
6988         };
6989         const struct rte_flow_item gre_item = {
6990                 .spec = &gre_spec,
6991                 .mask = &gre_mask,
6992                 .last = NULL,
6993         };
6994         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
6995         if (!nvgre_v)
6996                 return;
6997         if (!nvgre_m)
6998                 nvgre_m = &rte_flow_item_nvgre_mask;
6999         tni_flow_id_m = (const char *)nvgre_m->tni;
7000         tni_flow_id_v = (const char *)nvgre_v->tni;
7001         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
7002         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
7003         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
7004         memcpy(gre_key_m, tni_flow_id_m, size);
7005         for (i = 0; i < size; ++i)
7006                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
7007 }
7008
7009 /**
7010  * Add VXLAN item to matcher and to the value.
7011  *
7012  * @param[in, out] matcher
7013  *   Flow matcher.
7014  * @param[in, out] key
7015  *   Flow matcher value.
7016  * @param[in] item
7017  *   Flow pattern to translate.
7018  * @param[in] inner
7019  *   Item is inner pattern.
7020  */
7021 static void
7022 flow_dv_translate_item_vxlan(void *matcher, void *key,
7023                              const struct rte_flow_item *item,
7024                              int inner)
7025 {
7026         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
7027         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
7028         void *headers_m;
7029         void *headers_v;
7030         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7031         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7032         char *vni_m;
7033         char *vni_v;
7034         uint16_t dport;
7035         int size;
7036         int i;
7037
7038         if (inner) {
7039                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7040                                          inner_headers);
7041                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7042         } else {
7043                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7044                                          outer_headers);
7045                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7046         }
7047         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
7048                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
7049         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7050                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7051                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7052         }
7053         if (!vxlan_v)
7054                 return;
7055         if (!vxlan_m)
7056                 vxlan_m = &rte_flow_item_vxlan_mask;
7057         size = sizeof(vxlan_m->vni);
7058         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
7059         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
7060         memcpy(vni_m, vxlan_m->vni, size);
7061         for (i = 0; i < size; ++i)
7062                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
7063 }
7064
7065 /**
7066  * Add VXLAN-GPE item to matcher and to the value.
7067  *
7068  * @param[in, out] matcher
7069  *   Flow matcher.
7070  * @param[in, out] key
7071  *   Flow matcher value.
7072  * @param[in] item
7073  *   Flow pattern to translate.
7074  * @param[in] inner
7075  *   Item is inner pattern.
7076  */
7077
7078 static void
7079 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
7080                                  const struct rte_flow_item *item, int inner)
7081 {
7082         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
7083         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
7084         void *headers_m;
7085         void *headers_v;
7086         void *misc_m =
7087                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
7088         void *misc_v =
7089                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7090         char *vni_m;
7091         char *vni_v;
7092         uint16_t dport;
7093         int size;
7094         int i;
7095         uint8_t flags_m = 0xff;
7096         uint8_t flags_v = 0xc;
7097
7098         if (inner) {
7099                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7100                                          inner_headers);
7101                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7102         } else {
7103                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7104                                          outer_headers);
7105                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7106         }
7107         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
7108                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
7109         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7110                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7111                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7112         }
7113         if (!vxlan_v)
7114                 return;
7115         if (!vxlan_m)
7116                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
7117         size = sizeof(vxlan_m->vni);
7118         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
7119         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
7120         memcpy(vni_m, vxlan_m->vni, size);
7121         for (i = 0; i < size; ++i)
7122                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
7123         if (vxlan_m->flags) {
7124                 flags_m = vxlan_m->flags;
7125                 flags_v = vxlan_v->flags;
7126         }
7127         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
7128         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
7129         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
7130                  vxlan_m->protocol);
7131         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
7132                  vxlan_v->protocol);
7133 }
7134
7135 /**
7136  * Add Geneve item to matcher and to the value.
7137  *
7138  * @param[in, out] matcher
7139  *   Flow matcher.
7140  * @param[in, out] key
7141  *   Flow matcher value.
7142  * @param[in] item
7143  *   Flow pattern to translate.
7144  * @param[in] inner
7145  *   Item is inner pattern.
7146  */
7147
7148 static void
7149 flow_dv_translate_item_geneve(void *matcher, void *key,
7150                               const struct rte_flow_item *item, int inner)
7151 {
7152         const struct rte_flow_item_geneve *geneve_m = item->mask;
7153         const struct rte_flow_item_geneve *geneve_v = item->spec;
7154         void *headers_m;
7155         void *headers_v;
7156         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7157         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7158         uint16_t dport;
7159         uint16_t gbhdr_m;
7160         uint16_t gbhdr_v;
7161         char *vni_m;
7162         char *vni_v;
7163         size_t size, i;
7164
7165         if (inner) {
7166                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7167                                          inner_headers);
7168                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7169         } else {
7170                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7171                                          outer_headers);
7172                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7173         }
7174         dport = MLX5_UDP_PORT_GENEVE;
7175         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7176                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7177                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7178         }
7179         if (!geneve_v)
7180                 return;
7181         if (!geneve_m)
7182                 geneve_m = &rte_flow_item_geneve_mask;
7183         size = sizeof(geneve_m->vni);
7184         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
7185         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
7186         memcpy(vni_m, geneve_m->vni, size);
7187         for (i = 0; i < size; ++i)
7188                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
7189         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
7190                  rte_be_to_cpu_16(geneve_m->protocol));
7191         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
7192                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
7193         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
7194         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
7195         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
7196                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7197         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
7198                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
7199         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
7200                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7201         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
7202                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
7203                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
7204 }
7205
7206 /**
7207  * Add MPLS item to matcher and to the value.
7208  *
7209  * @param[in, out] matcher
7210  *   Flow matcher.
7211  * @param[in, out] key
7212  *   Flow matcher value.
7213  * @param[in] item
7214  *   Flow pattern to translate.
7215  * @param[in] prev_layer
7216  *   The protocol layer indicated in previous item.
7217  * @param[in] inner
7218  *   Item is inner pattern.
7219  */
7220 static void
7221 flow_dv_translate_item_mpls(void *matcher, void *key,
7222                             const struct rte_flow_item *item,
7223                             uint64_t prev_layer,
7224                             int inner)
7225 {
7226         const uint32_t *in_mpls_m = item->mask;
7227         const uint32_t *in_mpls_v = item->spec;
7228         uint32_t *out_mpls_m = 0;
7229         uint32_t *out_mpls_v = 0;
7230         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7231         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7232         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
7233                                      misc_parameters_2);
7234         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7235         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
7236         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7237
7238         switch (prev_layer) {
7239         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7240                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
7241                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
7242                          MLX5_UDP_PORT_MPLS);
7243                 break;
7244         case MLX5_FLOW_LAYER_GRE:
7245                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
7246                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
7247                          RTE_ETHER_TYPE_MPLS);
7248                 break;
7249         default:
7250                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
7251                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
7252                          IPPROTO_MPLS);
7253                 break;
7254         }
7255         if (!in_mpls_v)
7256                 return;
7257         if (!in_mpls_m)
7258                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
7259         switch (prev_layer) {
7260         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
7261                 out_mpls_m =
7262                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7263                                                  outer_first_mpls_over_udp);
7264                 out_mpls_v =
7265                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7266                                                  outer_first_mpls_over_udp);
7267                 break;
7268         case MLX5_FLOW_LAYER_GRE:
7269                 out_mpls_m =
7270                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
7271                                                  outer_first_mpls_over_gre);
7272                 out_mpls_v =
7273                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
7274                                                  outer_first_mpls_over_gre);
7275                 break;
7276         default:
7277                 /* Inner MPLS not over GRE is not supported. */
7278                 if (!inner) {
7279                         out_mpls_m =
7280                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7281                                                          misc2_m,
7282                                                          outer_first_mpls);
7283                         out_mpls_v =
7284                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
7285                                                          misc2_v,
7286                                                          outer_first_mpls);
7287                 }
7288                 break;
7289         }
7290         if (out_mpls_m && out_mpls_v) {
7291                 *out_mpls_m = *in_mpls_m;
7292                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
7293         }
7294 }
7295
7296 /**
7297  * Add metadata register item to matcher
7298  *
7299  * @param[in, out] matcher
7300  *   Flow matcher.
7301  * @param[in, out] key
7302  *   Flow matcher value.
7303  * @param[in] reg_type
7304  *   Type of device metadata register
7305  * @param[in] value
7306  *   Register value
7307  * @param[in] mask
7308  *   Register mask
7309  */
7310 static void
7311 flow_dv_match_meta_reg(void *matcher, void *key,
7312                        enum modify_reg reg_type,
7313                        uint32_t data, uint32_t mask)
7314 {
7315         void *misc2_m =
7316                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
7317         void *misc2_v =
7318                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
7319         uint32_t temp;
7320
7321         data &= mask;
7322         switch (reg_type) {
7323         case REG_A:
7324                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
7325                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
7326                 break;
7327         case REG_B:
7328                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
7329                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
7330                 break;
7331         case REG_C_0:
7332                 /*
7333                  * The metadata register C0 field might be divided into
7334                  * source vport index and META item value, we should set
7335                  * this field according to specified mask, not as whole one.
7336                  */
7337                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
7338                 temp |= mask;
7339                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
7340                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
7341                 temp &= ~mask;
7342                 temp |= data;
7343                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
7344                 break;
7345         case REG_C_1:
7346                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
7347                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
7348                 break;
7349         case REG_C_2:
7350                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
7351                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
7352                 break;
7353         case REG_C_3:
7354                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
7355                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
7356                 break;
7357         case REG_C_4:
7358                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
7359                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
7360                 break;
7361         case REG_C_5:
7362                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
7363                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
7364                 break;
7365         case REG_C_6:
7366                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
7367                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
7368                 break;
7369         case REG_C_7:
7370                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
7371                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
7372                 break;
7373         default:
7374                 MLX5_ASSERT(false);
7375                 break;
7376         }
7377 }
7378
7379 /**
7380  * Add MARK item to matcher
7381  *
7382  * @param[in] dev
7383  *   The device to configure through.
7384  * @param[in, out] matcher
7385  *   Flow matcher.
7386  * @param[in, out] key
7387  *   Flow matcher value.
7388  * @param[in] item
7389  *   Flow pattern to translate.
7390  */
7391 static void
7392 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
7393                             void *matcher, void *key,
7394                             const struct rte_flow_item *item)
7395 {
7396         struct mlx5_priv *priv = dev->data->dev_private;
7397         const struct rte_flow_item_mark *mark;
7398         uint32_t value;
7399         uint32_t mask;
7400
7401         mark = item->mask ? (const void *)item->mask :
7402                             &rte_flow_item_mark_mask;
7403         mask = mark->id & priv->sh->dv_mark_mask;
7404         mark = (const void *)item->spec;
7405         MLX5_ASSERT(mark);
7406         value = mark->id & priv->sh->dv_mark_mask & mask;
7407         if (mask) {
7408                 enum modify_reg reg;
7409
7410                 /* Get the metadata register index for the mark. */
7411                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
7412                 MLX5_ASSERT(reg > 0);
7413                 if (reg == REG_C_0) {
7414                         struct mlx5_priv *priv = dev->data->dev_private;
7415                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7416                         uint32_t shl_c0 = rte_bsf32(msk_c0);
7417
7418                         mask &= msk_c0;
7419                         mask <<= shl_c0;
7420                         value <<= shl_c0;
7421                 }
7422                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7423         }
7424 }
7425
7426 /**
7427  * Add META item to matcher
7428  *
7429  * @param[in] dev
7430  *   The devich to configure through.
7431  * @param[in, out] matcher
7432  *   Flow matcher.
7433  * @param[in, out] key
7434  *   Flow matcher value.
7435  * @param[in] attr
7436  *   Attributes of flow that includes this item.
7437  * @param[in] item
7438  *   Flow pattern to translate.
7439  */
7440 static void
7441 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
7442                             void *matcher, void *key,
7443                             const struct rte_flow_attr *attr,
7444                             const struct rte_flow_item *item)
7445 {
7446         const struct rte_flow_item_meta *meta_m;
7447         const struct rte_flow_item_meta *meta_v;
7448
7449         meta_m = (const void *)item->mask;
7450         if (!meta_m)
7451                 meta_m = &rte_flow_item_meta_mask;
7452         meta_v = (const void *)item->spec;
7453         if (meta_v) {
7454                 int reg;
7455                 uint32_t value = meta_v->data;
7456                 uint32_t mask = meta_m->data;
7457
7458                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
7459                 if (reg < 0)
7460                         return;
7461                 /*
7462                  * In datapath code there is no endianness
7463                  * coversions for perfromance reasons, all
7464                  * pattern conversions are done in rte_flow.
7465                  */
7466                 value = rte_cpu_to_be_32(value);
7467                 mask = rte_cpu_to_be_32(mask);
7468                 if (reg == REG_C_0) {
7469                         struct mlx5_priv *priv = dev->data->dev_private;
7470                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7471                         uint32_t shl_c0 = rte_bsf32(msk_c0);
7472 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
7473                         uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
7474
7475                         value >>= shr_c0;
7476                         mask >>= shr_c0;
7477 #endif
7478                         value <<= shl_c0;
7479                         mask <<= shl_c0;
7480                         MLX5_ASSERT(msk_c0);
7481                         MLX5_ASSERT(!(~msk_c0 & mask));
7482                 }
7483                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
7484         }
7485 }
7486
7487 /**
7488  * Add vport metadata Reg C0 item to matcher
7489  *
7490  * @param[in, out] matcher
7491  *   Flow matcher.
7492  * @param[in, out] key
7493  *   Flow matcher value.
7494  * @param[in] reg
7495  *   Flow pattern to translate.
7496  */
7497 static void
7498 flow_dv_translate_item_meta_vport(void *matcher, void *key,
7499                                   uint32_t value, uint32_t mask)
7500 {
7501         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
7502 }
7503
7504 /**
7505  * Add tag item to matcher
7506  *
7507  * @param[in] dev
7508  *   The devich to configure through.
7509  * @param[in, out] matcher
7510  *   Flow matcher.
7511  * @param[in, out] key
7512  *   Flow matcher value.
7513  * @param[in] item
7514  *   Flow pattern to translate.
7515  */
7516 static void
7517 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
7518                                 void *matcher, void *key,
7519                                 const struct rte_flow_item *item)
7520 {
7521         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
7522         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
7523         uint32_t mask, value;
7524
7525         MLX5_ASSERT(tag_v);
7526         value = tag_v->data;
7527         mask = tag_m ? tag_m->data : UINT32_MAX;
7528         if (tag_v->id == REG_C_0) {
7529                 struct mlx5_priv *priv = dev->data->dev_private;
7530                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7531                 uint32_t shl_c0 = rte_bsf32(msk_c0);
7532
7533                 mask &= msk_c0;
7534                 mask <<= shl_c0;
7535                 value <<= shl_c0;
7536         }
7537         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
7538 }
7539
7540 /**
7541  * Add TAG item to matcher
7542  *
7543  * @param[in] dev
7544  *   The devich to configure through.
7545  * @param[in, out] matcher
7546  *   Flow matcher.
7547  * @param[in, out] key
7548  *   Flow matcher value.
7549  * @param[in] item
7550  *   Flow pattern to translate.
7551  */
7552 static void
7553 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
7554                            void *matcher, void *key,
7555                            const struct rte_flow_item *item)
7556 {
7557         const struct rte_flow_item_tag *tag_v = item->spec;
7558         const struct rte_flow_item_tag *tag_m = item->mask;
7559         enum modify_reg reg;
7560
7561         MLX5_ASSERT(tag_v);
7562         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
7563         /* Get the metadata register index for the tag. */
7564         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
7565         MLX5_ASSERT(reg > 0);
7566         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
7567 }
7568
7569 /**
7570  * Add source vport match to the specified matcher.
7571  *
7572  * @param[in, out] matcher
7573  *   Flow matcher.
7574  * @param[in, out] key
7575  *   Flow matcher value.
7576  * @param[in] port
7577  *   Source vport value to match
7578  * @param[in] mask
7579  *   Mask
7580  */
7581 static void
7582 flow_dv_translate_item_source_vport(void *matcher, void *key,
7583                                     int16_t port, uint16_t mask)
7584 {
7585         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7586         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7587
7588         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
7589         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
7590 }
7591
7592 /**
7593  * Translate port-id item to eswitch match on  port-id.
7594  *
7595  * @param[in] dev
7596  *   The devich to configure through.
7597  * @param[in, out] matcher
7598  *   Flow matcher.
7599  * @param[in, out] key
7600  *   Flow matcher value.
7601  * @param[in] item
7602  *   Flow pattern to translate.
7603  *
7604  * @return
7605  *   0 on success, a negative errno value otherwise.
7606  */
7607 static int
7608 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
7609                                void *key, const struct rte_flow_item *item)
7610 {
7611         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
7612         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
7613         struct mlx5_priv *priv;
7614         uint16_t mask, id;
7615
7616         mask = pid_m ? pid_m->id : 0xffff;
7617         id = pid_v ? pid_v->id : dev->data->port_id;
7618         priv = mlx5_port_to_eswitch_info(id, item == NULL);
7619         if (!priv)
7620                 return -rte_errno;
7621         /* Translate to vport field or to metadata, depending on mode. */
7622         if (priv->vport_meta_mask)
7623                 flow_dv_translate_item_meta_vport(matcher, key,
7624                                                   priv->vport_meta_tag,
7625                                                   priv->vport_meta_mask);
7626         else
7627                 flow_dv_translate_item_source_vport(matcher, key,
7628                                                     priv->vport_id, mask);
7629         return 0;
7630 }
7631
7632 /**
7633  * Add ICMP6 item to matcher and to the value.
7634  *
7635  * @param[in, out] matcher
7636  *   Flow matcher.
7637  * @param[in, out] key
7638  *   Flow matcher value.
7639  * @param[in] item
7640  *   Flow pattern to translate.
7641  * @param[in] inner
7642  *   Item is inner pattern.
7643  */
7644 static void
7645 flow_dv_translate_item_icmp6(void *matcher, void *key,
7646                               const struct rte_flow_item *item,
7647                               int inner)
7648 {
7649         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
7650         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
7651         void *headers_m;
7652         void *headers_v;
7653         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7654                                      misc_parameters_3);
7655         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7656         if (inner) {
7657                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7658                                          inner_headers);
7659                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7660         } else {
7661                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7662                                          outer_headers);
7663                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7664         }
7665         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7666         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
7667         if (!icmp6_v)
7668                 return;
7669         if (!icmp6_m)
7670                 icmp6_m = &rte_flow_item_icmp6_mask;
7671         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
7672         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
7673                  icmp6_v->type & icmp6_m->type);
7674         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
7675         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
7676                  icmp6_v->code & icmp6_m->code);
7677 }
7678
7679 /**
7680  * Add ICMP item to matcher and to the value.
7681  *
7682  * @param[in, out] matcher
7683  *   Flow matcher.
7684  * @param[in, out] key
7685  *   Flow matcher value.
7686  * @param[in] item
7687  *   Flow pattern to translate.
7688  * @param[in] inner
7689  *   Item is inner pattern.
7690  */
7691 static void
7692 flow_dv_translate_item_icmp(void *matcher, void *key,
7693                             const struct rte_flow_item *item,
7694                             int inner)
7695 {
7696         const struct rte_flow_item_icmp *icmp_m = item->mask;
7697         const struct rte_flow_item_icmp *icmp_v = item->spec;
7698         uint32_t icmp_header_data_m = 0;
7699         uint32_t icmp_header_data_v = 0;
7700         void *headers_m;
7701         void *headers_v;
7702         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7703                                      misc_parameters_3);
7704         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7705         if (inner) {
7706                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7707                                          inner_headers);
7708                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7709         } else {
7710                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7711                                          outer_headers);
7712                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7713         }
7714         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7715         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
7716         if (!icmp_v)
7717                 return;
7718         if (!icmp_m)
7719                 icmp_m = &rte_flow_item_icmp_mask;
7720         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
7721                  icmp_m->hdr.icmp_type);
7722         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
7723                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
7724         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
7725                  icmp_m->hdr.icmp_code);
7726         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
7727                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
7728         icmp_header_data_m = rte_be_to_cpu_16(icmp_m->hdr.icmp_seq_nb);
7729         icmp_header_data_m |= rte_be_to_cpu_16(icmp_m->hdr.icmp_ident) << 16;
7730         if (icmp_header_data_m) {
7731                 icmp_header_data_v = rte_be_to_cpu_16(icmp_v->hdr.icmp_seq_nb);
7732                 icmp_header_data_v |=
7733                          rte_be_to_cpu_16(icmp_v->hdr.icmp_ident) << 16;
7734                 MLX5_SET(fte_match_set_misc3, misc3_m, icmp_header_data,
7735                          icmp_header_data_m);
7736                 MLX5_SET(fte_match_set_misc3, misc3_v, icmp_header_data,
7737                          icmp_header_data_v & icmp_header_data_m);
7738         }
7739 }
7740
7741 /**
7742  * Add GTP item to matcher and to the value.
7743  *
7744  * @param[in, out] matcher
7745  *   Flow matcher.
7746  * @param[in, out] key
7747  *   Flow matcher value.
7748  * @param[in] item
7749  *   Flow pattern to translate.
7750  * @param[in] inner
7751  *   Item is inner pattern.
7752  */
7753 static void
7754 flow_dv_translate_item_gtp(void *matcher, void *key,
7755                            const struct rte_flow_item *item, int inner)
7756 {
7757         const struct rte_flow_item_gtp *gtp_m = item->mask;
7758         const struct rte_flow_item_gtp *gtp_v = item->spec;
7759         void *headers_m;
7760         void *headers_v;
7761         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7762                                      misc_parameters_3);
7763         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7764         uint16_t dport = RTE_GTPU_UDP_PORT;
7765
7766         if (inner) {
7767                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7768                                          inner_headers);
7769                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7770         } else {
7771                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7772                                          outer_headers);
7773                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7774         }
7775         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7776                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7777                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7778         }
7779         if (!gtp_v)
7780                 return;
7781         if (!gtp_m)
7782                 gtp_m = &rte_flow_item_gtp_mask;
7783         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
7784                  gtp_m->v_pt_rsv_flags);
7785         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
7786                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
7787         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
7788         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
7789                  gtp_v->msg_type & gtp_m->msg_type);
7790         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
7791                  rte_be_to_cpu_32(gtp_m->teid));
7792         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
7793                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
7794 }
7795
7796 /**
7797  * Add eCPRI item to matcher and to the value.
7798  *
7799  * @param[in] dev
7800  *   The devich to configure through.
7801  * @param[in, out] matcher
7802  *   Flow matcher.
7803  * @param[in, out] key
7804  *   Flow matcher value.
7805  * @param[in] item
7806  *   Flow pattern to translate.
7807  * @param[in] samples
7808  *   Sample IDs to be used in the matching.
7809  */
7810 static void
7811 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
7812                              void *key, const struct rte_flow_item *item)
7813 {
7814         struct mlx5_priv *priv = dev->data->dev_private;
7815         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
7816         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
7817         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
7818                                      misc_parameters_4);
7819         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
7820         uint32_t *samples;
7821         void *dw_m;
7822         void *dw_v;
7823
7824         if (!ecpri_v)
7825                 return;
7826         if (!ecpri_m)
7827                 ecpri_m = &rte_flow_item_ecpri_mask;
7828         /*
7829          * Maximal four DW samples are supported in a single matching now.
7830          * Two are used now for a eCPRI matching:
7831          * 1. Type: one byte, mask should be 0x00ff0000 in network order
7832          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
7833          *    if any.
7834          */
7835         if (!ecpri_m->hdr.common.u32)
7836                 return;
7837         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
7838         /* Need to take the whole DW as the mask to fill the entry. */
7839         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
7840                             prog_sample_field_value_0);
7841         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
7842                             prog_sample_field_value_0);
7843         /* Already big endian (network order) in the header. */
7844         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
7845         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32;
7846         /* Sample#0, used for matching type, offset 0. */
7847         MLX5_SET(fte_match_set_misc4, misc4_m,
7848                  prog_sample_field_id_0, samples[0]);
7849         /* It makes no sense to set the sample ID in the mask field. */
7850         MLX5_SET(fte_match_set_misc4, misc4_v,
7851                  prog_sample_field_id_0, samples[0]);
7852         /*
7853          * Checking if message body part needs to be matched.
7854          * Some wildcard rules only matching type field should be supported.
7855          */
7856         if (ecpri_m->hdr.dummy[0]) {
7857                 switch (ecpri_v->hdr.common.type) {
7858                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
7859                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
7860                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
7861                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
7862                                             prog_sample_field_value_1);
7863                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
7864                                             prog_sample_field_value_1);
7865                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
7866                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0];
7867                         /* Sample#1, to match message body, offset 4. */
7868                         MLX5_SET(fte_match_set_misc4, misc4_m,
7869                                  prog_sample_field_id_1, samples[1]);
7870                         MLX5_SET(fte_match_set_misc4, misc4_v,
7871                                  prog_sample_field_id_1, samples[1]);
7872                         break;
7873                 default:
7874                         /* Others, do not match any sample ID. */
7875                         break;
7876                 }
7877         }
7878 }
7879
7880 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
7881
7882 #define HEADER_IS_ZERO(match_criteria, headers)                              \
7883         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
7884                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
7885
7886 /**
7887  * Calculate flow matcher enable bitmap.
7888  *
7889  * @param match_criteria
7890  *   Pointer to flow matcher criteria.
7891  *
7892  * @return
7893  *   Bitmap of enabled fields.
7894  */
7895 static uint8_t
7896 flow_dv_matcher_enable(uint32_t *match_criteria)
7897 {
7898         uint8_t match_criteria_enable;
7899
7900         match_criteria_enable =
7901                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
7902                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
7903         match_criteria_enable |=
7904                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
7905                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
7906         match_criteria_enable |=
7907                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
7908                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
7909         match_criteria_enable |=
7910                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
7911                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
7912         match_criteria_enable |=
7913                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
7914                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
7915         match_criteria_enable |=
7916                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
7917                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
7918         return match_criteria_enable;
7919 }
7920
7921
7922 /**
7923  * Get a flow table.
7924  *
7925  * @param[in, out] dev
7926  *   Pointer to rte_eth_dev structure.
7927  * @param[in] table_id
7928  *   Table id to use.
7929  * @param[in] egress
7930  *   Direction of the table.
7931  * @param[in] transfer
7932  *   E-Switch or NIC flow.
7933  * @param[out] error
7934  *   pointer to error structure.
7935  *
7936  * @return
7937  *   Returns tables resource based on the index, NULL in case of failed.
7938  */
7939 static struct mlx5_flow_tbl_resource *
7940 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
7941                          uint32_t table_id, uint8_t egress,
7942                          uint8_t transfer,
7943                          struct rte_flow_error *error)
7944 {
7945         struct mlx5_priv *priv = dev->data->dev_private;
7946         struct mlx5_dev_ctx_shared *sh = priv->sh;
7947         struct mlx5_flow_tbl_resource *tbl;
7948         union mlx5_flow_tbl_key table_key = {
7949                 {
7950                         .table_id = table_id,
7951                         .reserved = 0,
7952                         .domain = !!transfer,
7953                         .direction = !!egress,
7954                 }
7955         };
7956         struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
7957                                                          table_key.v64);
7958         struct mlx5_flow_tbl_data_entry *tbl_data;
7959         uint32_t idx = 0;
7960         int ret;
7961         void *domain;
7962
7963         if (pos) {
7964                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
7965                                         entry);
7966                 tbl = &tbl_data->tbl;
7967                 rte_atomic32_inc(&tbl->refcnt);
7968                 return tbl;
7969         }
7970         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
7971         if (!tbl_data) {
7972                 rte_flow_error_set(error, ENOMEM,
7973                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7974                                    NULL,
7975                                    "cannot allocate flow table data entry");
7976                 return NULL;
7977         }
7978         tbl_data->idx = idx;
7979         tbl = &tbl_data->tbl;
7980         pos = &tbl_data->entry;
7981         if (transfer)
7982                 domain = sh->fdb_domain;
7983         else if (egress)
7984                 domain = sh->tx_domain;
7985         else
7986                 domain = sh->rx_domain;
7987         ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj);
7988         if (ret) {
7989                 rte_flow_error_set(error, ENOMEM,
7990                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7991                                    NULL, "cannot create flow table object");
7992                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7993                 return NULL;
7994         }
7995         /*
7996          * No multi-threads now, but still better to initialize the reference
7997          * count before insert it into the hash list.
7998          */
7999         rte_atomic32_init(&tbl->refcnt);
8000         /* Jump action reference count is initialized here. */
8001         rte_atomic32_init(&tbl_data->jump.refcnt);
8002         pos->key = table_key.v64;
8003         ret = mlx5_hlist_insert(sh->flow_tbls, pos);
8004         if (ret < 0) {
8005                 rte_flow_error_set(error, -ret,
8006                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8007                                    "cannot insert flow table data entry");
8008                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
8009                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
8010         }
8011         rte_atomic32_inc(&tbl->refcnt);
8012         return tbl;
8013 }
8014
8015 /**
8016  * Release a flow table.
8017  *
8018  * @param[in] dev
8019  *   Pointer to rte_eth_dev structure.
8020  * @param[in] tbl
8021  *   Table resource to be released.
8022  *
8023  * @return
8024  *   Returns 0 if table was released, else return 1;
8025  */
8026 static int
8027 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
8028                              struct mlx5_flow_tbl_resource *tbl)
8029 {
8030         struct mlx5_priv *priv = dev->data->dev_private;
8031         struct mlx5_dev_ctx_shared *sh = priv->sh;
8032         struct mlx5_flow_tbl_data_entry *tbl_data =
8033                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
8034
8035         if (!tbl)
8036                 return 0;
8037         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
8038                 struct mlx5_hlist_entry *pos = &tbl_data->entry;
8039
8040                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
8041                 tbl->obj = NULL;
8042                 /* remove the entry from the hash list and free memory. */
8043                 mlx5_hlist_remove(sh->flow_tbls, pos);
8044                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
8045                                 tbl_data->idx);
8046                 return 0;
8047         }
8048         return 1;
8049 }
8050
8051 /**
8052  * Register the flow matcher.
8053  *
8054  * @param[in, out] dev
8055  *   Pointer to rte_eth_dev structure.
8056  * @param[in, out] matcher
8057  *   Pointer to flow matcher.
8058  * @param[in, out] key
8059  *   Pointer to flow table key.
8060  * @parm[in, out] dev_flow
8061  *   Pointer to the dev_flow.
8062  * @param[out] error
8063  *   pointer to error structure.
8064  *
8065  * @return
8066  *   0 on success otherwise -errno and errno is set.
8067  */
8068 static int
8069 flow_dv_matcher_register(struct rte_eth_dev *dev,
8070                          struct mlx5_flow_dv_matcher *matcher,
8071                          union mlx5_flow_tbl_key *key,
8072                          struct mlx5_flow *dev_flow,
8073                          struct rte_flow_error *error)
8074 {
8075         struct mlx5_priv *priv = dev->data->dev_private;
8076         struct mlx5_dev_ctx_shared *sh = priv->sh;
8077         struct mlx5_flow_dv_matcher *cache_matcher;
8078         struct mlx5dv_flow_matcher_attr dv_attr = {
8079                 .type = IBV_FLOW_ATTR_NORMAL,
8080                 .match_mask = (void *)&matcher->mask,
8081         };
8082         struct mlx5_flow_tbl_resource *tbl;
8083         struct mlx5_flow_tbl_data_entry *tbl_data;
8084         int ret;
8085
8086         tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
8087                                        key->domain, error);
8088         if (!tbl)
8089                 return -rte_errno;      /* No need to refill the error info */
8090         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
8091         /* Lookup from cache. */
8092         LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
8093                 if (matcher->crc == cache_matcher->crc &&
8094                     matcher->priority == cache_matcher->priority &&
8095                     !memcmp((const void *)matcher->mask.buf,
8096                             (const void *)cache_matcher->mask.buf,
8097                             cache_matcher->mask.size)) {
8098                         DRV_LOG(DEBUG,
8099                                 "%s group %u priority %hd use %s "
8100                                 "matcher %p: refcnt %d++",
8101                                 key->domain ? "FDB" : "NIC", key->table_id,
8102                                 cache_matcher->priority,
8103                                 key->direction ? "tx" : "rx",
8104                                 (void *)cache_matcher,
8105                                 rte_atomic32_read(&cache_matcher->refcnt));
8106                         rte_atomic32_inc(&cache_matcher->refcnt);
8107                         dev_flow->handle->dvh.matcher = cache_matcher;
8108                         /* old matcher should not make the table ref++. */
8109                         flow_dv_tbl_resource_release(dev, tbl);
8110                         return 0;
8111                 }
8112         }
8113         /* Register new matcher. */
8114         cache_matcher = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache_matcher), 0,
8115                                     SOCKET_ID_ANY);
8116         if (!cache_matcher) {
8117                 flow_dv_tbl_resource_release(dev, tbl);
8118                 return rte_flow_error_set(error, ENOMEM,
8119                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8120                                           "cannot allocate matcher memory");
8121         }
8122         *cache_matcher = *matcher;
8123         dv_attr.match_criteria_enable =
8124                 flow_dv_matcher_enable(cache_matcher->mask.buf);
8125         dv_attr.priority = matcher->priority;
8126         if (key->direction)
8127                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
8128         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
8129                                                &cache_matcher->matcher_object);
8130         if (ret) {
8131                 mlx5_free(cache_matcher);
8132 #ifdef HAVE_MLX5DV_DR
8133                 flow_dv_tbl_resource_release(dev, tbl);
8134 #endif
8135                 return rte_flow_error_set(error, ENOMEM,
8136                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8137                                           NULL, "cannot create matcher");
8138         }
8139         /* Save the table information */
8140         cache_matcher->tbl = tbl;
8141         rte_atomic32_init(&cache_matcher->refcnt);
8142         /* only matcher ref++, table ref++ already done above in get API. */
8143         rte_atomic32_inc(&cache_matcher->refcnt);
8144         LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
8145         dev_flow->handle->dvh.matcher = cache_matcher;
8146         DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
8147                 key->domain ? "FDB" : "NIC", key->table_id,
8148                 cache_matcher->priority,
8149                 key->direction ? "tx" : "rx", (void *)cache_matcher,
8150                 rte_atomic32_read(&cache_matcher->refcnt));
8151         return 0;
8152 }
8153
8154 /**
8155  * Find existing tag resource or create and register a new one.
8156  *
8157  * @param dev[in, out]
8158  *   Pointer to rte_eth_dev structure.
8159  * @param[in, out] tag_be24
8160  *   Tag value in big endian then R-shift 8.
8161  * @parm[in, out] dev_flow
8162  *   Pointer to the dev_flow.
8163  * @param[out] error
8164  *   pointer to error structure.
8165  *
8166  * @return
8167  *   0 on success otherwise -errno and errno is set.
8168  */
8169 static int
8170 flow_dv_tag_resource_register
8171                         (struct rte_eth_dev *dev,
8172                          uint32_t tag_be24,
8173                          struct mlx5_flow *dev_flow,
8174                          struct rte_flow_error *error)
8175 {
8176         struct mlx5_priv *priv = dev->data->dev_private;
8177         struct mlx5_dev_ctx_shared *sh = priv->sh;
8178         struct mlx5_flow_dv_tag_resource *cache_resource;
8179         struct mlx5_hlist_entry *entry;
8180         int ret;
8181
8182         /* Lookup a matching resource from cache. */
8183         entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
8184         if (entry) {
8185                 cache_resource = container_of
8186                         (entry, struct mlx5_flow_dv_tag_resource, entry);
8187                 rte_atomic32_inc(&cache_resource->refcnt);
8188                 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
8189                 dev_flow->dv.tag_resource = cache_resource;
8190                 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
8191                         (void *)cache_resource,
8192                         rte_atomic32_read(&cache_resource->refcnt));
8193                 return 0;
8194         }
8195         /* Register new resource. */
8196         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG],
8197                                        &dev_flow->handle->dvh.rix_tag);
8198         if (!cache_resource)
8199                 return rte_flow_error_set(error, ENOMEM,
8200                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8201                                           "cannot allocate resource memory");
8202         cache_resource->entry.key = (uint64_t)tag_be24;
8203         ret = mlx5_flow_os_create_flow_action_tag(tag_be24,
8204                                                   &cache_resource->action);
8205         if (ret) {
8206                 mlx5_free(cache_resource);
8207                 return rte_flow_error_set(error, ENOMEM,
8208                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8209                                           NULL, "cannot create action");
8210         }
8211         rte_atomic32_init(&cache_resource->refcnt);
8212         rte_atomic32_inc(&cache_resource->refcnt);
8213         if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
8214                 mlx5_flow_os_destroy_flow_action(cache_resource->action);
8215                 mlx5_free(cache_resource);
8216                 return rte_flow_error_set(error, EEXIST,
8217                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8218                                           NULL, "cannot insert tag");
8219         }
8220         dev_flow->dv.tag_resource = cache_resource;
8221         DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
8222                 (void *)cache_resource,
8223                 rte_atomic32_read(&cache_resource->refcnt));
8224         return 0;
8225 }
8226
8227 /**
8228  * Release the tag.
8229  *
8230  * @param dev
8231  *   Pointer to Ethernet device.
8232  * @param tag_idx
8233  *   Tag index.
8234  *
8235  * @return
8236  *   1 while a reference on it exists, 0 when freed.
8237  */
8238 static int
8239 flow_dv_tag_release(struct rte_eth_dev *dev,
8240                     uint32_t tag_idx)
8241 {
8242         struct mlx5_priv *priv = dev->data->dev_private;
8243         struct mlx5_dev_ctx_shared *sh = priv->sh;
8244         struct mlx5_flow_dv_tag_resource *tag;
8245
8246         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8247         if (!tag)
8248                 return 0;
8249         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
8250                 dev->data->port_id, (void *)tag,
8251                 rte_atomic32_read(&tag->refcnt));
8252         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
8253                 claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
8254                 mlx5_hlist_remove(sh->tag_table, &tag->entry);
8255                 DRV_LOG(DEBUG, "port %u tag %p: removed",
8256                         dev->data->port_id, (void *)tag);
8257                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
8258                 return 0;
8259         }
8260         return 1;
8261 }
8262
8263 /**
8264  * Translate port ID action to vport.
8265  *
8266  * @param[in] dev
8267  *   Pointer to rte_eth_dev structure.
8268  * @param[in] action
8269  *   Pointer to the port ID action.
8270  * @param[out] dst_port_id
8271  *   The target port ID.
8272  * @param[out] error
8273  *   Pointer to the error structure.
8274  *
8275  * @return
8276  *   0 on success, a negative errno value otherwise and rte_errno is set.
8277  */
8278 static int
8279 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
8280                                  const struct rte_flow_action *action,
8281                                  uint32_t *dst_port_id,
8282                                  struct rte_flow_error *error)
8283 {
8284         uint32_t port;
8285         struct mlx5_priv *priv;
8286         const struct rte_flow_action_port_id *conf =
8287                         (const struct rte_flow_action_port_id *)action->conf;
8288
8289         port = conf->original ? dev->data->port_id : conf->id;
8290         priv = mlx5_port_to_eswitch_info(port, false);
8291         if (!priv)
8292                 return rte_flow_error_set(error, -rte_errno,
8293                                           RTE_FLOW_ERROR_TYPE_ACTION,
8294                                           NULL,
8295                                           "No eswitch info was found for port");
8296 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
8297         /*
8298          * This parameter is transferred to
8299          * mlx5dv_dr_action_create_dest_ib_port().
8300          */
8301         *dst_port_id = priv->dev_port;
8302 #else
8303         /*
8304          * Legacy mode, no LAG configurations is supported.
8305          * This parameter is transferred to
8306          * mlx5dv_dr_action_create_dest_vport().
8307          */
8308         *dst_port_id = priv->vport_id;
8309 #endif
8310         return 0;
8311 }
8312
8313 /**
8314  * Create a counter with aging configuration.
8315  *
8316  * @param[in] dev
8317  *   Pointer to rte_eth_dev structure.
8318  * @param[out] count
8319  *   Pointer to the counter action configuration.
8320  * @param[in] age
8321  *   Pointer to the aging action configuration.
8322  *
8323  * @return
8324  *   Index to flow counter on success, 0 otherwise.
8325  */
8326 static uint32_t
8327 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
8328                                 struct mlx5_flow *dev_flow,
8329                                 const struct rte_flow_action_count *count,
8330                                 const struct rte_flow_action_age *age)
8331 {
8332         uint32_t counter;
8333         struct mlx5_age_param *age_param;
8334
8335         counter = flow_dv_counter_alloc(dev,
8336                                 count ? count->shared : 0,
8337                                 count ? count->id : 0,
8338                                 dev_flow->dv.group, !!age);
8339         if (!counter || age == NULL)
8340                 return counter;
8341         age_param  = flow_dv_counter_idx_get_age(dev, counter);
8342         /*
8343          * The counter age accuracy may have a bit delay. Have 3/4
8344          * second bias on the timeount in order to let it age in time.
8345          */
8346         age_param->context = age->context ? age->context :
8347                 (void *)(uintptr_t)(dev_flow->flow_idx);
8348         /*
8349          * The counter age accuracy may have a bit delay. Have 3/4
8350          * second bias on the timeount in order to let it age in time.
8351          */
8352         age_param->timeout = age->timeout * 10 - MLX5_AGING_TIME_DELAY;
8353         /* Set expire time in unit of 0.1 sec. */
8354         age_param->port_id = dev->data->port_id;
8355         age_param->expire = age_param->timeout +
8356                         rte_rdtsc() / (rte_get_tsc_hz() / 10);
8357         rte_atomic16_set(&age_param->state, AGE_CANDIDATE);
8358         return counter;
8359 }
8360 /**
8361  * Add Tx queue matcher
8362  *
8363  * @param[in] dev
8364  *   Pointer to the dev struct.
8365  * @param[in, out] matcher
8366  *   Flow matcher.
8367  * @param[in, out] key
8368  *   Flow matcher value.
8369  * @param[in] item
8370  *   Flow pattern to translate.
8371  * @param[in] inner
8372  *   Item is inner pattern.
8373  */
8374 static void
8375 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
8376                                 void *matcher, void *key,
8377                                 const struct rte_flow_item *item)
8378 {
8379         const struct mlx5_rte_flow_item_tx_queue *queue_m;
8380         const struct mlx5_rte_flow_item_tx_queue *queue_v;
8381         void *misc_m =
8382                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
8383         void *misc_v =
8384                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
8385         struct mlx5_txq_ctrl *txq;
8386         uint32_t queue;
8387
8388
8389         queue_m = (const void *)item->mask;
8390         if (!queue_m)
8391                 return;
8392         queue_v = (const void *)item->spec;
8393         if (!queue_v)
8394                 return;
8395         txq = mlx5_txq_get(dev, queue_v->queue);
8396         if (!txq)
8397                 return;
8398         queue = txq->obj->sq->id;
8399         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
8400         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
8401                  queue & queue_m->queue);
8402         mlx5_txq_release(dev, queue_v->queue);
8403 }
8404
8405 /**
8406  * Set the hash fields according to the @p flow information.
8407  *
8408  * @param[in] dev_flow
8409  *   Pointer to the mlx5_flow.
8410  * @param[in] rss_desc
8411  *   Pointer to the mlx5_flow_rss_desc.
8412  */
8413 static void
8414 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
8415                        struct mlx5_flow_rss_desc *rss_desc)
8416 {
8417         uint64_t items = dev_flow->handle->layers;
8418         int rss_inner = 0;
8419         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
8420
8421         dev_flow->hash_fields = 0;
8422 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
8423         if (rss_desc->level >= 2) {
8424                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
8425                 rss_inner = 1;
8426         }
8427 #endif
8428         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
8429             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
8430                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
8431                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
8432                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
8433                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
8434                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
8435                         else
8436                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
8437                 }
8438         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
8439                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
8440                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
8441                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
8442                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
8443                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
8444                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
8445                         else
8446                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
8447                 }
8448         }
8449         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
8450             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
8451                 if (rss_types & ETH_RSS_UDP) {
8452                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
8453                                 dev_flow->hash_fields |=
8454                                                 IBV_RX_HASH_SRC_PORT_UDP;
8455                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
8456                                 dev_flow->hash_fields |=
8457                                                 IBV_RX_HASH_DST_PORT_UDP;
8458                         else
8459                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
8460                 }
8461         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
8462                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
8463                 if (rss_types & ETH_RSS_TCP) {
8464                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
8465                                 dev_flow->hash_fields |=
8466                                                 IBV_RX_HASH_SRC_PORT_TCP;
8467                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
8468                                 dev_flow->hash_fields |=
8469                                                 IBV_RX_HASH_DST_PORT_TCP;
8470                         else
8471                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
8472                 }
8473         }
8474 }
8475
8476 /**
8477  * Create an Rx Hash queue.
8478  *
8479  * @param dev
8480  *   Pointer to Ethernet device.
8481  * @param[in] dev_flow
8482  *   Pointer to the mlx5_flow.
8483  * @param[in] rss_desc
8484  *   Pointer to the mlx5_flow_rss_desc.
8485  * @param[out] hrxq_idx
8486  *   Hash Rx queue index.
8487  *
8488  * @return
8489  *   The Verbs/DevX object initialised, NULL otherwise and rte_errno is set.
8490  */
8491 static struct mlx5_hrxq *
8492 flow_dv_handle_rx_queue(struct rte_eth_dev *dev,
8493                         struct mlx5_flow *dev_flow,
8494                         struct mlx5_flow_rss_desc *rss_desc,
8495                         uint32_t *hrxq_idx)
8496 {
8497         struct mlx5_priv *priv = dev->data->dev_private;
8498         struct mlx5_flow_handle *dh = dev_flow->handle;
8499         struct mlx5_hrxq *hrxq;
8500
8501         MLX5_ASSERT(rss_desc->queue_num);
8502         *hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
8503                                   MLX5_RSS_HASH_KEY_LEN,
8504                                   dev_flow->hash_fields,
8505                                   rss_desc->queue,
8506                                   rss_desc->queue_num);
8507         if (!*hrxq_idx) {
8508                 *hrxq_idx = mlx5_hrxq_new
8509                                 (dev, rss_desc->key,
8510                                  MLX5_RSS_HASH_KEY_LEN,
8511                                  dev_flow->hash_fields,
8512                                  rss_desc->queue,
8513                                  rss_desc->queue_num,
8514                                  !!(dh->layers &
8515                                  MLX5_FLOW_LAYER_TUNNEL));
8516                 if (!*hrxq_idx)
8517                         return NULL;
8518         }
8519         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
8520                               *hrxq_idx);
8521         return hrxq;
8522 }
8523
8524 /**
8525  * Find existing sample resource or create and register a new one.
8526  *
8527  * @param[in, out] dev
8528  *   Pointer to rte_eth_dev structure.
8529  * @param[in] attr
8530  *   Attributes of flow that includes this item.
8531  * @param[in] resource
8532  *   Pointer to sample resource.
8533  * @parm[in, out] dev_flow
8534  *   Pointer to the dev_flow.
8535  * @param[in, out] sample_dv_actions
8536  *   Pointer to sample actions list.
8537  * @param[out] error
8538  *   pointer to error structure.
8539  *
8540  * @return
8541  *   0 on success otherwise -errno and errno is set.
8542  */
8543 static int
8544 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
8545                          const struct rte_flow_attr *attr,
8546                          struct mlx5_flow_dv_sample_resource *resource,
8547                          struct mlx5_flow *dev_flow,
8548                          void **sample_dv_actions,
8549                          struct rte_flow_error *error)
8550 {
8551         struct mlx5_flow_dv_sample_resource *cache_resource;
8552         struct mlx5dv_dr_flow_sampler_attr sampler_attr;
8553         struct mlx5_priv *priv = dev->data->dev_private;
8554         struct mlx5_dev_ctx_shared *sh = priv->sh;
8555         struct mlx5_flow_tbl_resource *tbl;
8556         uint32_t idx = 0;
8557         const uint32_t next_ft_step = 1;
8558         uint32_t next_ft_id = resource->ft_id + next_ft_step;
8559
8560         /* Lookup a matching resource from cache. */
8561         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_SAMPLE], sh->sample_action_list,
8562                       idx, cache_resource, next) {
8563                 if (resource->ratio == cache_resource->ratio &&
8564                     resource->ft_type == cache_resource->ft_type &&
8565                     resource->ft_id == cache_resource->ft_id &&
8566                     resource->set_action == cache_resource->set_action &&
8567                     !memcmp((void *)&resource->sample_act,
8568                             (void *)&cache_resource->sample_act,
8569                             sizeof(struct mlx5_flow_sub_actions_list))) {
8570                         DRV_LOG(DEBUG, "sample resource %p: refcnt %d++",
8571                                 (void *)cache_resource,
8572                                 __atomic_load_n(&cache_resource->refcnt,
8573                                                 __ATOMIC_RELAXED));
8574                         __atomic_fetch_add(&cache_resource->refcnt, 1,
8575                                            __ATOMIC_RELAXED);
8576                         dev_flow->handle->dvh.rix_sample = idx;
8577                         dev_flow->dv.sample_res = cache_resource;
8578                         return 0;
8579                 }
8580         }
8581         /* Register new sample resource. */
8582         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE],
8583                                        &dev_flow->handle->dvh.rix_sample);
8584         if (!cache_resource)
8585                 return rte_flow_error_set(error, ENOMEM,
8586                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8587                                           NULL,
8588                                           "cannot allocate resource memory");
8589         *cache_resource = *resource;
8590         /* Create normal path table level */
8591         tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
8592                                         attr->egress, attr->transfer, error);
8593         if (!tbl) {
8594                 rte_flow_error_set(error, ENOMEM,
8595                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8596                                           NULL,
8597                                           "fail to create normal path table "
8598                                           "for sample");
8599                 goto error;
8600         }
8601         cache_resource->normal_path_tbl = tbl;
8602         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
8603                 cache_resource->default_miss =
8604                                 mlx5_glue->dr_create_flow_action_default_miss();
8605                 if (!cache_resource->default_miss) {
8606                         rte_flow_error_set(error, ENOMEM,
8607                                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8608                                                 NULL,
8609                                                 "cannot create default miss "
8610                                                 "action");
8611                         goto error;
8612                 }
8613                 sample_dv_actions[resource->sample_act.actions_num++] =
8614                                                 cache_resource->default_miss;
8615         }
8616         /* Create a DR sample action */
8617         sampler_attr.sample_ratio = cache_resource->ratio;
8618         sampler_attr.default_next_table = tbl->obj;
8619         sampler_attr.num_sample_actions = resource->sample_act.actions_num;
8620         sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
8621                                                         &sample_dv_actions[0];
8622         sampler_attr.action = cache_resource->set_action;
8623         cache_resource->verbs_action =
8624                 mlx5_glue->dr_create_flow_action_sampler(&sampler_attr);
8625         if (!cache_resource->verbs_action) {
8626                 rte_flow_error_set(error, ENOMEM,
8627                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8628                                         NULL, "cannot create sample action");
8629                 goto error;
8630         }
8631         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8632         ILIST_INSERT(sh->ipool[MLX5_IPOOL_SAMPLE], &sh->sample_action_list,
8633                      dev_flow->handle->dvh.rix_sample, cache_resource,
8634                      next);
8635         dev_flow->dv.sample_res = cache_resource;
8636         DRV_LOG(DEBUG, "new sample resource %p: refcnt %d++",
8637                 (void *)cache_resource,
8638                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8639         return 0;
8640 error:
8641         if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
8642                 if (cache_resource->default_miss)
8643                         claim_zero(mlx5_glue->destroy_flow_action
8644                                 (cache_resource->default_miss));
8645         } else {
8646                 if (cache_resource->sample_idx.rix_hrxq &&
8647                     !mlx5_hrxq_release(dev,
8648                                 cache_resource->sample_idx.rix_hrxq))
8649                         cache_resource->sample_idx.rix_hrxq = 0;
8650                 if (cache_resource->sample_idx.rix_tag &&
8651                     !flow_dv_tag_release(dev,
8652                                 cache_resource->sample_idx.rix_tag))
8653                         cache_resource->sample_idx.rix_tag = 0;
8654                 if (cache_resource->sample_idx.cnt) {
8655                         flow_dv_counter_release(dev,
8656                                 cache_resource->sample_idx.cnt);
8657                         cache_resource->sample_idx.cnt = 0;
8658                 }
8659         }
8660         if (cache_resource->normal_path_tbl)
8661                 flow_dv_tbl_resource_release(dev,
8662                                 cache_resource->normal_path_tbl);
8663         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE],
8664                                 dev_flow->handle->dvh.rix_sample);
8665         dev_flow->handle->dvh.rix_sample = 0;
8666         return -rte_errno;
8667 }
8668
8669 /**
8670  * Find existing destination array resource or create and register a new one.
8671  *
8672  * @param[in, out] dev
8673  *   Pointer to rte_eth_dev structure.
8674  * @param[in] attr
8675  *   Attributes of flow that includes this item.
8676  * @param[in] resource
8677  *   Pointer to destination array resource.
8678  * @parm[in, out] dev_flow
8679  *   Pointer to the dev_flow.
8680  * @param[out] error
8681  *   pointer to error structure.
8682  *
8683  * @return
8684  *   0 on success otherwise -errno and errno is set.
8685  */
8686 static int
8687 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
8688                          const struct rte_flow_attr *attr,
8689                          struct mlx5_flow_dv_dest_array_resource *resource,
8690                          struct mlx5_flow *dev_flow,
8691                          struct rte_flow_error *error)
8692 {
8693         struct mlx5_flow_dv_dest_array_resource *cache_resource;
8694         struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
8695         struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
8696         struct mlx5_priv *priv = dev->data->dev_private;
8697         struct mlx5_dev_ctx_shared *sh = priv->sh;
8698         struct mlx5_flow_sub_actions_list *sample_act;
8699         struct mlx5dv_dr_domain *domain;
8700         uint32_t idx = 0;
8701
8702         /* Lookup a matching resource from cache. */
8703         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8704                       sh->dest_array_list,
8705                       idx, cache_resource, next) {
8706                 if (resource->num_of_dest == cache_resource->num_of_dest &&
8707                     resource->ft_type == cache_resource->ft_type &&
8708                     !memcmp((void *)cache_resource->sample_act,
8709                             (void *)resource->sample_act,
8710                            (resource->num_of_dest *
8711                            sizeof(struct mlx5_flow_sub_actions_list)))) {
8712                         DRV_LOG(DEBUG, "dest array resource %p: refcnt %d++",
8713                                 (void *)cache_resource,
8714                                 __atomic_load_n(&cache_resource->refcnt,
8715                                                 __ATOMIC_RELAXED));
8716                         __atomic_fetch_add(&cache_resource->refcnt, 1,
8717                                            __ATOMIC_RELAXED);
8718                         dev_flow->handle->dvh.rix_dest_array = idx;
8719                         dev_flow->dv.dest_array_res = cache_resource;
8720                         return 0;
8721                 }
8722         }
8723         /* Register new destination array resource. */
8724         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8725                                        &dev_flow->handle->dvh.rix_dest_array);
8726         if (!cache_resource)
8727                 return rte_flow_error_set(error, ENOMEM,
8728                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8729                                           NULL,
8730                                           "cannot allocate resource memory");
8731         *cache_resource = *resource;
8732         if (attr->transfer)
8733                 domain = sh->fdb_domain;
8734         else if (attr->ingress)
8735                 domain = sh->rx_domain;
8736         else
8737                 domain = sh->tx_domain;
8738         for (idx = 0; idx < resource->num_of_dest; idx++) {
8739                 dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
8740                                  mlx5_malloc(MLX5_MEM_ZERO,
8741                                  sizeof(struct mlx5dv_dr_action_dest_attr),
8742                                  0, SOCKET_ID_ANY);
8743                 if (!dest_attr[idx]) {
8744                         rte_flow_error_set(error, ENOMEM,
8745                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8746                                            NULL,
8747                                            "cannot allocate resource memory");
8748                         goto error;
8749                 }
8750                 dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
8751                 sample_act = &resource->sample_act[idx];
8752                 if (sample_act->action_flags == MLX5_FLOW_ACTION_QUEUE) {
8753                         dest_attr[idx]->dest = sample_act->dr_queue_action;
8754                 } else if (sample_act->action_flags ==
8755                           (MLX5_FLOW_ACTION_PORT_ID | MLX5_FLOW_ACTION_ENCAP)) {
8756                         dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST_REFORMAT;
8757                         dest_attr[idx]->dest_reformat = &dest_reformat[idx];
8758                         dest_attr[idx]->dest_reformat->reformat =
8759                                         sample_act->dr_encap_action;
8760                         dest_attr[idx]->dest_reformat->dest =
8761                                         sample_act->dr_port_id_action;
8762                 } else if (sample_act->action_flags ==
8763                            MLX5_FLOW_ACTION_PORT_ID) {
8764                         dest_attr[idx]->dest = sample_act->dr_port_id_action;
8765                 }
8766         }
8767         /* create a dest array actioin */
8768         cache_resource->action = mlx5_glue->dr_create_flow_action_dest_array
8769                                                 (domain,
8770                                                  cache_resource->num_of_dest,
8771                                                  dest_attr);
8772         if (!cache_resource->action) {
8773                 rte_flow_error_set(error, ENOMEM,
8774                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8775                                    NULL,
8776                                    "cannot create destination array action");
8777                 goto error;
8778         }
8779         __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
8780         ILIST_INSERT(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8781                      &sh->dest_array_list,
8782                      dev_flow->handle->dvh.rix_dest_array, cache_resource,
8783                      next);
8784         dev_flow->dv.dest_array_res = cache_resource;
8785         DRV_LOG(DEBUG, "new destination array resource %p: refcnt %d++",
8786                 (void *)cache_resource,
8787                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
8788         for (idx = 0; idx < resource->num_of_dest; idx++)
8789                 mlx5_free(dest_attr[idx]);
8790         return 0;
8791 error:
8792         for (idx = 0; idx < resource->num_of_dest; idx++) {
8793                 struct mlx5_flow_sub_actions_idx *act_res =
8794                                         &cache_resource->sample_idx[idx];
8795                 if (act_res->rix_hrxq &&
8796                     !mlx5_hrxq_release(dev,
8797                                 act_res->rix_hrxq))
8798                         act_res->rix_hrxq = 0;
8799                 if (act_res->rix_encap_decap &&
8800                         !flow_dv_encap_decap_resource_release(dev,
8801                                 act_res->rix_encap_decap))
8802                         act_res->rix_encap_decap = 0;
8803                 if (act_res->rix_port_id_action &&
8804                         !flow_dv_port_id_action_resource_release(dev,
8805                                 act_res->rix_port_id_action))
8806                         act_res->rix_port_id_action = 0;
8807                 if (dest_attr[idx])
8808                         mlx5_free(dest_attr[idx]);
8809         }
8810
8811         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
8812                                 dev_flow->handle->dvh.rix_dest_array);
8813         dev_flow->handle->dvh.rix_dest_array = 0;
8814         return -rte_errno;
8815 }
8816
8817 /**
8818  * Convert Sample action to DV specification.
8819  *
8820  * @param[in] dev
8821  *   Pointer to rte_eth_dev structure.
8822  * @param[in] action
8823  *   Pointer to action structure.
8824  * @param[in, out] dev_flow
8825  *   Pointer to the mlx5_flow.
8826  * @param[in] attr
8827  *   Pointer to the flow attributes.
8828  * @param[in, out] num_of_dest
8829  *   Pointer to the num of destination.
8830  * @param[in, out] sample_actions
8831  *   Pointer to sample actions list.
8832  * @param[in, out] res
8833  *   Pointer to sample resource.
8834  * @param[out] error
8835  *   Pointer to the error structure.
8836  *
8837  * @return
8838  *   0 on success, a negative errno value otherwise and rte_errno is set.
8839  */
8840 static int
8841 flow_dv_translate_action_sample(struct rte_eth_dev *dev,
8842                                 const struct rte_flow_action *action,
8843                                 struct mlx5_flow *dev_flow,
8844                                 const struct rte_flow_attr *attr,
8845                                 uint32_t *num_of_dest,
8846                                 void **sample_actions,
8847                                 struct mlx5_flow_dv_sample_resource *res,
8848                                 struct rte_flow_error *error)
8849 {
8850         struct mlx5_priv *priv = dev->data->dev_private;
8851         const struct rte_flow_action_sample *sample_action;
8852         const struct rte_flow_action *sub_actions;
8853         const struct rte_flow_action_queue *queue;
8854         struct mlx5_flow_sub_actions_list *sample_act;
8855         struct mlx5_flow_sub_actions_idx *sample_idx;
8856         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
8857                                               priv->rss_desc)
8858                                               [!!priv->flow_nested_idx];
8859         uint64_t action_flags = 0;
8860
8861         sample_act = &res->sample_act;
8862         sample_idx = &res->sample_idx;
8863         sample_action = (const struct rte_flow_action_sample *)action->conf;
8864         res->ratio = sample_action->ratio;
8865         sub_actions = sample_action->actions;
8866         for (; sub_actions->type != RTE_FLOW_ACTION_TYPE_END; sub_actions++) {
8867                 int type = sub_actions->type;
8868                 uint32_t pre_rix = 0;
8869                 void *pre_r;
8870                 switch (type) {
8871                 case RTE_FLOW_ACTION_TYPE_QUEUE:
8872                 {
8873                         struct mlx5_hrxq *hrxq;
8874                         uint32_t hrxq_idx;
8875
8876                         queue = sub_actions->conf;
8877                         rss_desc->queue_num = 1;
8878                         rss_desc->queue[0] = queue->index;
8879                         hrxq = flow_dv_handle_rx_queue(dev, dev_flow,
8880                                         rss_desc, &hrxq_idx);
8881                         if (!hrxq)
8882                                 return rte_flow_error_set
8883                                         (error, rte_errno,
8884                                          RTE_FLOW_ERROR_TYPE_ACTION,
8885                                          NULL,
8886                                          "cannot create fate queue");
8887                         sample_act->dr_queue_action = hrxq->action;
8888                         sample_idx->rix_hrxq = hrxq_idx;
8889                         sample_actions[sample_act->actions_num++] =
8890                                                 hrxq->action;
8891                         (*num_of_dest)++;
8892                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
8893                         if (action_flags & MLX5_FLOW_ACTION_MARK)
8894                                 dev_flow->handle->rix_hrxq = hrxq_idx;
8895                         dev_flow->handle->fate_action =
8896                                         MLX5_FLOW_FATE_QUEUE;
8897                         break;
8898                 }
8899                 case RTE_FLOW_ACTION_TYPE_MARK:
8900                 {
8901                         uint32_t tag_be = mlx5_flow_mark_set
8902                                 (((const struct rte_flow_action_mark *)
8903                                 (sub_actions->conf))->id);
8904
8905                         dev_flow->handle->mark = 1;
8906                         pre_rix = dev_flow->handle->dvh.rix_tag;
8907                         /* Save the mark resource before sample */
8908                         pre_r = dev_flow->dv.tag_resource;
8909                         if (flow_dv_tag_resource_register(dev, tag_be,
8910                                                   dev_flow, error))
8911                                 return -rte_errno;
8912                         MLX5_ASSERT(dev_flow->dv.tag_resource);
8913                         sample_act->dr_tag_action =
8914                                 dev_flow->dv.tag_resource->action;
8915                         sample_idx->rix_tag =
8916                                 dev_flow->handle->dvh.rix_tag;
8917                         sample_actions[sample_act->actions_num++] =
8918                                                 sample_act->dr_tag_action;
8919                         /* Recover the mark resource after sample */
8920                         dev_flow->dv.tag_resource = pre_r;
8921                         dev_flow->handle->dvh.rix_tag = pre_rix;
8922                         action_flags |= MLX5_FLOW_ACTION_MARK;
8923                         break;
8924                 }
8925                 case RTE_FLOW_ACTION_TYPE_COUNT:
8926                 {
8927                         uint32_t counter;
8928
8929                         counter = flow_dv_translate_create_counter(dev,
8930                                         dev_flow, sub_actions->conf, 0);
8931                         if (!counter)
8932                                 return rte_flow_error_set
8933                                                 (error, rte_errno,
8934                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8935                                                  NULL,
8936                                                  "cannot create counter"
8937                                                  " object.");
8938                         sample_idx->cnt = counter;
8939                         sample_act->dr_cnt_action =
8940                                   (flow_dv_counter_get_by_idx(dev,
8941                                   counter, NULL))->action;
8942                         sample_actions[sample_act->actions_num++] =
8943                                                 sample_act->dr_cnt_action;
8944                         action_flags |= MLX5_FLOW_ACTION_COUNT;
8945                         break;
8946                 }
8947                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
8948                 {
8949                         struct mlx5_flow_dv_port_id_action_resource
8950                                         port_id_resource;
8951                         uint32_t port_id = 0;
8952
8953                         memset(&port_id_resource, 0, sizeof(port_id_resource));
8954                         /* Save the port id resource before sample */
8955                         pre_rix = dev_flow->handle->rix_port_id_action;
8956                         pre_r = dev_flow->dv.port_id_action;
8957                         if (flow_dv_translate_action_port_id(dev, sub_actions,
8958                                                              &port_id, error))
8959                                 return -rte_errno;
8960                         port_id_resource.port_id = port_id;
8961                         if (flow_dv_port_id_action_resource_register
8962                             (dev, &port_id_resource, dev_flow, error))
8963                                 return -rte_errno;
8964                         sample_act->dr_port_id_action =
8965                                 dev_flow->dv.port_id_action->action;
8966                         sample_idx->rix_port_id_action =
8967                                 dev_flow->handle->rix_port_id_action;
8968                         sample_actions[sample_act->actions_num++] =
8969                                                 sample_act->dr_port_id_action;
8970                         /* Recover the port id resource after sample */
8971                         dev_flow->dv.port_id_action = pre_r;
8972                         dev_flow->handle->rix_port_id_action = pre_rix;
8973                         (*num_of_dest)++;
8974                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
8975                         break;
8976                 }
8977                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
8978                         /* Save the encap resource before sample */
8979                         pre_rix = dev_flow->handle->dvh.rix_encap_decap;
8980                         pre_r = dev_flow->dv.encap_decap;
8981                         if (flow_dv_create_action_l2_encap(dev, sub_actions,
8982                                                            dev_flow,
8983                                                            attr->transfer,
8984                                                            error))
8985                                 return -rte_errno;
8986                         sample_act->dr_encap_action =
8987                                 dev_flow->dv.encap_decap->action;
8988                         sample_idx->rix_encap_decap =
8989                                 dev_flow->handle->dvh.rix_encap_decap;
8990                         sample_actions[sample_act->actions_num++] =
8991                                                 sample_act->dr_encap_action;
8992                         /* Recover the encap resource after sample */
8993                         dev_flow->dv.encap_decap = pre_r;
8994                         dev_flow->handle->dvh.rix_encap_decap = pre_rix;
8995                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
8996                         break;
8997                 default:
8998                         return rte_flow_error_set(error, EINVAL,
8999                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9000                                 NULL,
9001                                 "Not support for sampler action");
9002                 }
9003         }
9004         sample_act->action_flags = action_flags;
9005         res->ft_id = dev_flow->dv.group;
9006         if (attr->transfer) {
9007                 union {
9008                         uint32_t action_in[MLX5_ST_SZ_DW(set_action_in)];
9009                         uint64_t set_action;
9010                 } action_ctx = { .set_action = 0 };
9011
9012                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
9013                 MLX5_SET(set_action_in, action_ctx.action_in, action_type,
9014                          MLX5_MODIFICATION_TYPE_SET);
9015                 MLX5_SET(set_action_in, action_ctx.action_in, field,
9016                          MLX5_MODI_META_REG_C_0);
9017                 MLX5_SET(set_action_in, action_ctx.action_in, data,
9018                          priv->vport_meta_tag);
9019                 res->set_action = action_ctx.set_action;
9020         } else if (attr->ingress) {
9021                 res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
9022         }
9023         return 0;
9024 }
9025
9026 /**
9027  * Convert Sample action to DV specification.
9028  *
9029  * @param[in] dev
9030  *   Pointer to rte_eth_dev structure.
9031  * @param[in, out] dev_flow
9032  *   Pointer to the mlx5_flow.
9033  * @param[in] attr
9034  *   Pointer to the flow attributes.
9035  * @param[in] num_of_dest
9036  *   The num of destination.
9037  * @param[in, out] res
9038  *   Pointer to sample resource.
9039  * @param[in, out] mdest_res
9040  *   Pointer to destination array resource.
9041  * @param[in] sample_actions
9042  *   Pointer to sample path actions list.
9043  * @param[in] action_flags
9044  *   Holds the actions detected until now.
9045  * @param[out] error
9046  *   Pointer to the error structure.
9047  *
9048  * @return
9049  *   0 on success, a negative errno value otherwise and rte_errno is set.
9050  */
9051 static int
9052 flow_dv_create_action_sample(struct rte_eth_dev *dev,
9053                              struct mlx5_flow *dev_flow,
9054                              const struct rte_flow_attr *attr,
9055                              uint32_t num_of_dest,
9056                              struct mlx5_flow_dv_sample_resource *res,
9057                              struct mlx5_flow_dv_dest_array_resource *mdest_res,
9058                              void **sample_actions,
9059                              uint64_t action_flags,
9060                              struct rte_flow_error *error)
9061 {
9062         struct mlx5_priv *priv = dev->data->dev_private;
9063         /* update normal path action resource into last index of array */
9064         uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
9065         struct mlx5_flow_sub_actions_list *sample_act =
9066                                         &mdest_res->sample_act[dest_index];
9067         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
9068                                               priv->rss_desc)
9069                                               [!!priv->flow_nested_idx];
9070         uint32_t normal_idx = 0;
9071         struct mlx5_hrxq *hrxq;
9072         uint32_t hrxq_idx;
9073
9074         if (num_of_dest > 1) {
9075                 if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
9076                         /* Handle QP action for mirroring */
9077                         hrxq = flow_dv_handle_rx_queue(dev, dev_flow,
9078                                                        rss_desc, &hrxq_idx);
9079                         if (!hrxq)
9080                                 return rte_flow_error_set
9081                                      (error, rte_errno,
9082                                       RTE_FLOW_ERROR_TYPE_ACTION,
9083                                       NULL,
9084                                       "cannot create rx queue");
9085                         normal_idx++;
9086                         mdest_res->sample_idx[dest_index].rix_hrxq = hrxq_idx;
9087                         sample_act->dr_queue_action = hrxq->action;
9088                         if (action_flags & MLX5_FLOW_ACTION_MARK)
9089                                 dev_flow->handle->rix_hrxq = hrxq_idx;
9090                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9091                 }
9092                 if (sample_act->action_flags & MLX5_FLOW_ACTION_ENCAP) {
9093                         normal_idx++;
9094                         mdest_res->sample_idx[dest_index].rix_encap_decap =
9095                                 dev_flow->handle->dvh.rix_encap_decap;
9096                         sample_act->dr_encap_action =
9097                                 dev_flow->dv.encap_decap->action;
9098                 }
9099                 if (sample_act->action_flags & MLX5_FLOW_ACTION_PORT_ID) {
9100                         normal_idx++;
9101                         mdest_res->sample_idx[dest_index].rix_port_id_action =
9102                                 dev_flow->handle->rix_port_id_action;
9103                         sample_act->dr_port_id_action =
9104                                 dev_flow->dv.port_id_action->action;
9105                 }
9106                 sample_act->actions_num = normal_idx;
9107                 /* update sample action resource into first index of array */
9108                 mdest_res->ft_type = res->ft_type;
9109                 memcpy(&mdest_res->sample_idx[0], &res->sample_idx,
9110                                 sizeof(struct mlx5_flow_sub_actions_idx));
9111                 memcpy(&mdest_res->sample_act[0], &res->sample_act,
9112                                 sizeof(struct mlx5_flow_sub_actions_list));
9113                 mdest_res->num_of_dest = num_of_dest;
9114                 if (flow_dv_dest_array_resource_register(dev, attr, mdest_res,
9115                                                          dev_flow, error))
9116                         return rte_flow_error_set(error, EINVAL,
9117                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9118                                                   NULL, "can't create sample "
9119                                                   "action");
9120         } else {
9121                 if (flow_dv_sample_resource_register(dev, attr, res, dev_flow,
9122                                                      sample_actions, error))
9123                         return rte_flow_error_set(error, EINVAL,
9124                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9125                                                   NULL,
9126                                                   "can't create sample action");
9127         }
9128         return 0;
9129 }
9130
9131 /**
9132  * Fill the flow with DV spec, lock free
9133  * (mutex should be acquired by caller).
9134  *
9135  * @param[in] dev
9136  *   Pointer to rte_eth_dev structure.
9137  * @param[in, out] dev_flow
9138  *   Pointer to the sub flow.
9139  * @param[in] attr
9140  *   Pointer to the flow attributes.
9141  * @param[in] items
9142  *   Pointer to the list of items.
9143  * @param[in] actions
9144  *   Pointer to the list of actions.
9145  * @param[out] error
9146  *   Pointer to the error structure.
9147  *
9148  * @return
9149  *   0 on success, a negative errno value otherwise and rte_errno is set.
9150  */
9151 static int
9152 __flow_dv_translate(struct rte_eth_dev *dev,
9153                     struct mlx5_flow *dev_flow,
9154                     const struct rte_flow_attr *attr,
9155                     const struct rte_flow_item items[],
9156                     const struct rte_flow_action actions[],
9157                     struct rte_flow_error *error)
9158 {
9159         struct mlx5_priv *priv = dev->data->dev_private;
9160         struct mlx5_dev_config *dev_conf = &priv->config;
9161         struct rte_flow *flow = dev_flow->flow;
9162         struct mlx5_flow_handle *handle = dev_flow->handle;
9163         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
9164                                               priv->rss_desc)
9165                                               [!!priv->flow_nested_idx];
9166         uint64_t item_flags = 0;
9167         uint64_t last_item = 0;
9168         uint64_t action_flags = 0;
9169         uint64_t priority = attr->priority;
9170         struct mlx5_flow_dv_matcher matcher = {
9171                 .mask = {
9172                         .size = sizeof(matcher.mask.buf) -
9173                                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
9174                 },
9175         };
9176         int actions_n = 0;
9177         bool actions_end = false;
9178         union {
9179                 struct mlx5_flow_dv_modify_hdr_resource res;
9180                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
9181                             sizeof(struct mlx5_modification_cmd) *
9182                             (MLX5_MAX_MODIFY_NUM + 1)];
9183         } mhdr_dummy;
9184         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
9185         const struct rte_flow_action_count *count = NULL;
9186         const struct rte_flow_action_age *age = NULL;
9187         union flow_dv_attr flow_attr = { .attr = 0 };
9188         uint32_t tag_be;
9189         union mlx5_flow_tbl_key tbl_key;
9190         uint32_t modify_action_position = UINT32_MAX;
9191         void *match_mask = matcher.mask.buf;
9192         void *match_value = dev_flow->dv.value.buf;
9193         uint8_t next_protocol = 0xff;
9194         struct rte_vlan_hdr vlan = { 0 };
9195         struct mlx5_flow_dv_dest_array_resource mdest_res;
9196         struct mlx5_flow_dv_sample_resource sample_res;
9197         void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
9198         struct mlx5_flow_sub_actions_list *sample_act;
9199         uint32_t sample_act_pos = UINT32_MAX;
9200         uint32_t num_of_dest = 0;
9201         int tmp_actions_n = 0;
9202         uint32_t table;
9203         int ret = 0;
9204
9205         memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
9206         memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
9207         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
9208                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
9209         /* update normal path action resource into last index of array */
9210         sample_act = &mdest_res.sample_act[MLX5_MAX_DEST_NUM - 1];
9211         ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
9212                                        !!priv->fdb_def_rule, &table, error);
9213         if (ret)
9214                 return ret;
9215         dev_flow->dv.group = table;
9216         if (attr->transfer)
9217                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
9218         if (priority == MLX5_FLOW_PRIO_RSVD)
9219                 priority = dev_conf->flow_prio - 1;
9220         /* number of actions must be set to 0 in case of dirty stack. */
9221         mhdr_res->actions_num = 0;
9222         for (; !actions_end ; actions++) {
9223                 const struct rte_flow_action_queue *queue;
9224                 const struct rte_flow_action_rss *rss;
9225                 const struct rte_flow_action *action = actions;
9226                 const uint8_t *rss_key;
9227                 const struct rte_flow_action_meter *mtr;
9228                 struct mlx5_flow_tbl_resource *tbl;
9229                 uint32_t port_id = 0;
9230                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
9231                 int action_type = actions->type;
9232                 const struct rte_flow_action *found_action = NULL;
9233                 struct mlx5_flow_meter *fm = NULL;
9234                 uint32_t jump_group = 0;
9235
9236                 if (!mlx5_flow_os_action_supported(action_type))
9237                         return rte_flow_error_set(error, ENOTSUP,
9238                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9239                                                   actions,
9240                                                   "action not supported");
9241                 switch (action_type) {
9242                 case RTE_FLOW_ACTION_TYPE_VOID:
9243                         break;
9244                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
9245                         if (flow_dv_translate_action_port_id(dev, action,
9246                                                              &port_id, error))
9247                                 return -rte_errno;
9248                         port_id_resource.port_id = port_id;
9249                         MLX5_ASSERT(!handle->rix_port_id_action);
9250                         if (flow_dv_port_id_action_resource_register
9251                             (dev, &port_id_resource, dev_flow, error))
9252                                 return -rte_errno;
9253                         dev_flow->dv.actions[actions_n++] =
9254                                         dev_flow->dv.port_id_action->action;
9255                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9256                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
9257                         sample_act->action_flags |= MLX5_FLOW_ACTION_PORT_ID;
9258                         num_of_dest++;
9259                         break;
9260                 case RTE_FLOW_ACTION_TYPE_FLAG:
9261                         action_flags |= MLX5_FLOW_ACTION_FLAG;
9262                         dev_flow->handle->mark = 1;
9263                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
9264                                 struct rte_flow_action_mark mark = {
9265                                         .id = MLX5_FLOW_MARK_DEFAULT,
9266                                 };
9267
9268                                 if (flow_dv_convert_action_mark(dev, &mark,
9269                                                                 mhdr_res,
9270                                                                 error))
9271                                         return -rte_errno;
9272                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
9273                                 break;
9274                         }
9275                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
9276                         /*
9277                          * Only one FLAG or MARK is supported per device flow
9278                          * right now. So the pointer to the tag resource must be
9279                          * zero before the register process.
9280                          */
9281                         MLX5_ASSERT(!handle->dvh.rix_tag);
9282                         if (flow_dv_tag_resource_register(dev, tag_be,
9283                                                           dev_flow, error))
9284                                 return -rte_errno;
9285                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9286                         dev_flow->dv.actions[actions_n++] =
9287                                         dev_flow->dv.tag_resource->action;
9288                         break;
9289                 case RTE_FLOW_ACTION_TYPE_MARK:
9290                         action_flags |= MLX5_FLOW_ACTION_MARK;
9291                         dev_flow->handle->mark = 1;
9292                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
9293                                 const struct rte_flow_action_mark *mark =
9294                                         (const struct rte_flow_action_mark *)
9295                                                 actions->conf;
9296
9297                                 if (flow_dv_convert_action_mark(dev, mark,
9298                                                                 mhdr_res,
9299                                                                 error))
9300                                         return -rte_errno;
9301                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
9302                                 break;
9303                         }
9304                         /* Fall-through */
9305                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
9306                         /* Legacy (non-extensive) MARK action. */
9307                         tag_be = mlx5_flow_mark_set
9308                               (((const struct rte_flow_action_mark *)
9309                                (actions->conf))->id);
9310                         MLX5_ASSERT(!handle->dvh.rix_tag);
9311                         if (flow_dv_tag_resource_register(dev, tag_be,
9312                                                           dev_flow, error))
9313                                 return -rte_errno;
9314                         MLX5_ASSERT(dev_flow->dv.tag_resource);
9315                         dev_flow->dv.actions[actions_n++] =
9316                                         dev_flow->dv.tag_resource->action;
9317                         break;
9318                 case RTE_FLOW_ACTION_TYPE_SET_META:
9319                         if (flow_dv_convert_action_set_meta
9320                                 (dev, mhdr_res, attr,
9321                                  (const struct rte_flow_action_set_meta *)
9322                                   actions->conf, error))
9323                                 return -rte_errno;
9324                         action_flags |= MLX5_FLOW_ACTION_SET_META;
9325                         break;
9326                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
9327                         if (flow_dv_convert_action_set_tag
9328                                 (dev, mhdr_res,
9329                                  (const struct rte_flow_action_set_tag *)
9330                                   actions->conf, error))
9331                                 return -rte_errno;
9332                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9333                         break;
9334                 case RTE_FLOW_ACTION_TYPE_DROP:
9335                         action_flags |= MLX5_FLOW_ACTION_DROP;
9336                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
9337                         break;
9338                 case RTE_FLOW_ACTION_TYPE_QUEUE:
9339                         queue = actions->conf;
9340                         rss_desc->queue_num = 1;
9341                         rss_desc->queue[0] = queue->index;
9342                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
9343                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9344                         sample_act->action_flags |= MLX5_FLOW_ACTION_QUEUE;
9345                         num_of_dest++;
9346                         break;
9347                 case RTE_FLOW_ACTION_TYPE_RSS:
9348                         rss = actions->conf;
9349                         memcpy(rss_desc->queue, rss->queue,
9350                                rss->queue_num * sizeof(uint16_t));
9351                         rss_desc->queue_num = rss->queue_num;
9352                         /* NULL RSS key indicates default RSS key. */
9353                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
9354                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
9355                         /*
9356                          * rss->level and rss.types should be set in advance
9357                          * when expanding items for RSS.
9358                          */
9359                         action_flags |= MLX5_FLOW_ACTION_RSS;
9360                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
9361                         break;
9362                 case RTE_FLOW_ACTION_TYPE_AGE:
9363                 case RTE_FLOW_ACTION_TYPE_COUNT:
9364                         if (!dev_conf->devx) {
9365                                 return rte_flow_error_set
9366                                               (error, ENOTSUP,
9367                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9368                                                NULL,
9369                                                "count action not supported");
9370                         }
9371                         /* Save information first, will apply later. */
9372                         if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT)
9373                                 count = action->conf;
9374                         else
9375                                 age = action->conf;
9376                         action_flags |= MLX5_FLOW_ACTION_COUNT;
9377                         break;
9378                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
9379                         dev_flow->dv.actions[actions_n++] =
9380                                                 priv->sh->pop_vlan_action;
9381                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
9382                         break;
9383                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
9384                         if (!(action_flags &
9385                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
9386                                 flow_dev_get_vlan_info_from_items(items, &vlan);
9387                         vlan.eth_proto = rte_be_to_cpu_16
9388                              ((((const struct rte_flow_action_of_push_vlan *)
9389                                                    actions->conf)->ethertype));
9390                         found_action = mlx5_flow_find_action
9391                                         (actions + 1,
9392                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
9393                         if (found_action)
9394                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
9395                         found_action = mlx5_flow_find_action
9396                                         (actions + 1,
9397                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
9398                         if (found_action)
9399                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
9400                         if (flow_dv_create_action_push_vlan
9401                                             (dev, attr, &vlan, dev_flow, error))
9402                                 return -rte_errno;
9403                         dev_flow->dv.actions[actions_n++] =
9404                                         dev_flow->dv.push_vlan_res->action;
9405                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
9406                         break;
9407                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
9408                         /* of_vlan_push action handled this action */
9409                         MLX5_ASSERT(action_flags &
9410                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
9411                         break;
9412                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
9413                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
9414                                 break;
9415                         flow_dev_get_vlan_info_from_items(items, &vlan);
9416                         mlx5_update_vlan_vid_pcp(actions, &vlan);
9417                         /* If no VLAN push - this is a modify header action */
9418                         if (flow_dv_convert_action_modify_vlan_vid
9419                                                 (mhdr_res, actions, error))
9420                                 return -rte_errno;
9421                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
9422                         break;
9423                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
9424                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
9425                         if (flow_dv_create_action_l2_encap(dev, actions,
9426                                                            dev_flow,
9427                                                            attr->transfer,
9428                                                            error))
9429                                 return -rte_errno;
9430                         dev_flow->dv.actions[actions_n++] =
9431                                         dev_flow->dv.encap_decap->action;
9432                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9433                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
9434                                 sample_act->action_flags |=
9435                                                         MLX5_FLOW_ACTION_ENCAP;
9436                         break;
9437                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
9438                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
9439                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
9440                                                            attr->transfer,
9441                                                            error))
9442                                 return -rte_errno;
9443                         dev_flow->dv.actions[actions_n++] =
9444                                         dev_flow->dv.encap_decap->action;
9445                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9446                         break;
9447                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
9448                         /* Handle encap with preceding decap. */
9449                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
9450                                 if (flow_dv_create_action_raw_encap
9451                                         (dev, actions, dev_flow, attr, error))
9452                                         return -rte_errno;
9453                                 dev_flow->dv.actions[actions_n++] =
9454                                         dev_flow->dv.encap_decap->action;
9455                         } else {
9456                                 /* Handle encap without preceding decap. */
9457                                 if (flow_dv_create_action_l2_encap
9458                                     (dev, actions, dev_flow, attr->transfer,
9459                                      error))
9460                                         return -rte_errno;
9461                                 dev_flow->dv.actions[actions_n++] =
9462                                         dev_flow->dv.encap_decap->action;
9463                         }
9464                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
9465                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
9466                                 sample_act->action_flags |=
9467                                                         MLX5_FLOW_ACTION_ENCAP;
9468                         break;
9469                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
9470                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
9471                                 ;
9472                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
9473                                 if (flow_dv_create_action_l2_decap
9474                                     (dev, dev_flow, attr->transfer, error))
9475                                         return -rte_errno;
9476                                 dev_flow->dv.actions[actions_n++] =
9477                                         dev_flow->dv.encap_decap->action;
9478                         }
9479                         /* If decap is followed by encap, handle it at encap. */
9480                         action_flags |= MLX5_FLOW_ACTION_DECAP;
9481                         break;
9482                 case RTE_FLOW_ACTION_TYPE_JUMP:
9483                         jump_group = ((const struct rte_flow_action_jump *)
9484                                                         action->conf)->group;
9485                         if (dev_flow->external && jump_group <
9486                                         MLX5_MAX_TABLES_EXTERNAL)
9487                                 jump_group *= MLX5_FLOW_TABLE_FACTOR;
9488                         ret = mlx5_flow_group_to_table(attr, dev_flow->external,
9489                                                        jump_group,
9490                                                        !!priv->fdb_def_rule,
9491                                                        &table, error);
9492                         if (ret)
9493                                 return ret;
9494                         tbl = flow_dv_tbl_resource_get(dev, table,
9495                                                        attr->egress,
9496                                                        attr->transfer, error);
9497                         if (!tbl)
9498                                 return rte_flow_error_set
9499                                                 (error, errno,
9500                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9501                                                  NULL,
9502                                                  "cannot create jump action.");
9503                         if (flow_dv_jump_tbl_resource_register
9504                             (dev, tbl, dev_flow, error)) {
9505                                 flow_dv_tbl_resource_release(dev, tbl);
9506                                 return rte_flow_error_set
9507                                                 (error, errno,
9508                                                  RTE_FLOW_ERROR_TYPE_ACTION,
9509                                                  NULL,
9510                                                  "cannot create jump action.");
9511                         }
9512                         dev_flow->dv.actions[actions_n++] =
9513                                         dev_flow->dv.jump->action;
9514                         action_flags |= MLX5_FLOW_ACTION_JUMP;
9515                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
9516                         break;
9517                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
9518                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
9519                         if (flow_dv_convert_action_modify_mac
9520                                         (mhdr_res, actions, error))
9521                                 return -rte_errno;
9522                         action_flags |= actions->type ==
9523                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
9524                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
9525                                         MLX5_FLOW_ACTION_SET_MAC_DST;
9526                         break;
9527                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
9528                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
9529                         if (flow_dv_convert_action_modify_ipv4
9530                                         (mhdr_res, actions, error))
9531                                 return -rte_errno;
9532                         action_flags |= actions->type ==
9533                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
9534                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
9535                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
9536                         break;
9537                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
9538                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
9539                         if (flow_dv_convert_action_modify_ipv6
9540                                         (mhdr_res, actions, error))
9541                                 return -rte_errno;
9542                         action_flags |= actions->type ==
9543                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
9544                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
9545                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
9546                         break;
9547                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
9548                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
9549                         if (flow_dv_convert_action_modify_tp
9550                                         (mhdr_res, actions, items,
9551                                          &flow_attr, dev_flow, !!(action_flags &
9552                                          MLX5_FLOW_ACTION_DECAP), error))
9553                                 return -rte_errno;
9554                         action_flags |= actions->type ==
9555                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
9556                                         MLX5_FLOW_ACTION_SET_TP_SRC :
9557                                         MLX5_FLOW_ACTION_SET_TP_DST;
9558                         break;
9559                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
9560                         if (flow_dv_convert_action_modify_dec_ttl
9561                                         (mhdr_res, items, &flow_attr, dev_flow,
9562                                          !!(action_flags &
9563                                          MLX5_FLOW_ACTION_DECAP), error))
9564                                 return -rte_errno;
9565                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
9566                         break;
9567                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
9568                         if (flow_dv_convert_action_modify_ttl
9569                                         (mhdr_res, actions, items, &flow_attr,
9570                                          dev_flow, !!(action_flags &
9571                                          MLX5_FLOW_ACTION_DECAP), error))
9572                                 return -rte_errno;
9573                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
9574                         break;
9575                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
9576                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
9577                         if (flow_dv_convert_action_modify_tcp_seq
9578                                         (mhdr_res, actions, error))
9579                                 return -rte_errno;
9580                         action_flags |= actions->type ==
9581                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
9582                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
9583                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
9584                         break;
9585
9586                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
9587                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
9588                         if (flow_dv_convert_action_modify_tcp_ack
9589                                         (mhdr_res, actions, error))
9590                                 return -rte_errno;
9591                         action_flags |= actions->type ==
9592                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
9593                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
9594                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
9595                         break;
9596                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
9597                         if (flow_dv_convert_action_set_reg
9598                                         (mhdr_res, actions, error))
9599                                 return -rte_errno;
9600                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9601                         break;
9602                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
9603                         if (flow_dv_convert_action_copy_mreg
9604                                         (dev, mhdr_res, actions, error))
9605                                 return -rte_errno;
9606                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
9607                         break;
9608                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
9609                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
9610                         dev_flow->handle->fate_action =
9611                                         MLX5_FLOW_FATE_DEFAULT_MISS;
9612                         break;
9613                 case RTE_FLOW_ACTION_TYPE_METER:
9614                         mtr = actions->conf;
9615                         if (!flow->meter) {
9616                                 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
9617                                                             attr, error);
9618                                 if (!fm)
9619                                         return rte_flow_error_set(error,
9620                                                 rte_errno,
9621                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9622                                                 NULL,
9623                                                 "meter not found "
9624                                                 "or invalid parameters");
9625                                 flow->meter = fm->idx;
9626                         }
9627                         /* Set the meter action. */
9628                         if (!fm) {
9629                                 fm = mlx5_ipool_get(priv->sh->ipool
9630                                                 [MLX5_IPOOL_MTR], flow->meter);
9631                                 if (!fm)
9632                                         return rte_flow_error_set(error,
9633                                                 rte_errno,
9634                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9635                                                 NULL,
9636                                                 "meter not found "
9637                                                 "or invalid parameters");
9638                         }
9639                         dev_flow->dv.actions[actions_n++] =
9640                                 fm->mfts->meter_action;
9641                         action_flags |= MLX5_FLOW_ACTION_METER;
9642                         break;
9643                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
9644                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
9645                                                               actions, error))
9646                                 return -rte_errno;
9647                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
9648                         break;
9649                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
9650                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
9651                                                               actions, error))
9652                                 return -rte_errno;
9653                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
9654                         break;
9655                 case RTE_FLOW_ACTION_TYPE_SAMPLE:
9656                         sample_act_pos = actions_n;
9657                         ret = flow_dv_translate_action_sample(dev,
9658                                                               actions,
9659                                                               dev_flow, attr,
9660                                                               &num_of_dest,
9661                                                               sample_actions,
9662                                                               &sample_res,
9663                                                               error);
9664                         if (ret < 0)
9665                                 return ret;
9666                         actions_n++;
9667                         action_flags |= MLX5_FLOW_ACTION_SAMPLE;
9668                         /* put encap action into group if work with port id */
9669                         if ((action_flags & MLX5_FLOW_ACTION_ENCAP) &&
9670                             (action_flags & MLX5_FLOW_ACTION_PORT_ID))
9671                                 sample_act->action_flags |=
9672                                                         MLX5_FLOW_ACTION_ENCAP;
9673                         break;
9674                 case RTE_FLOW_ACTION_TYPE_END:
9675                         actions_end = true;
9676                         if (mhdr_res->actions_num) {
9677                                 /* create modify action if needed. */
9678                                 if (flow_dv_modify_hdr_resource_register
9679                                         (dev, mhdr_res, dev_flow, error))
9680                                         return -rte_errno;
9681                                 dev_flow->dv.actions[modify_action_position] =
9682                                         handle->dvh.modify_hdr->action;
9683                         }
9684                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
9685                                 flow->counter =
9686                                         flow_dv_translate_create_counter(dev,
9687                                                 dev_flow, count, age);
9688
9689                                 if (!flow->counter)
9690                                         return rte_flow_error_set
9691                                                 (error, rte_errno,
9692                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9693                                                 NULL,
9694                                                 "cannot create counter"
9695                                                 " object.");
9696                                 dev_flow->dv.actions[actions_n] =
9697                                           (flow_dv_counter_get_by_idx(dev,
9698                                           flow->counter, NULL))->action;
9699                                 actions_n++;
9700                         }
9701                         if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
9702                                 ret = flow_dv_create_action_sample(dev,
9703                                                           dev_flow, attr,
9704                                                           num_of_dest,
9705                                                           &sample_res,
9706                                                           &mdest_res,
9707                                                           sample_actions,
9708                                                           action_flags,
9709                                                           error);
9710                                 if (ret < 0)
9711                                         return rte_flow_error_set
9712                                                 (error, rte_errno,
9713                                                 RTE_FLOW_ERROR_TYPE_ACTION,
9714                                                 NULL,
9715                                                 "cannot create sample action");
9716                                 if (num_of_dest > 1) {
9717                                         dev_flow->dv.actions[sample_act_pos] =
9718                                         dev_flow->dv.dest_array_res->action;
9719                                 } else {
9720                                         dev_flow->dv.actions[sample_act_pos] =
9721                                         dev_flow->dv.sample_res->verbs_action;
9722                                 }
9723                         }
9724                         break;
9725                 default:
9726                         break;
9727                 }
9728                 if (mhdr_res->actions_num &&
9729                     modify_action_position == UINT32_MAX)
9730                         modify_action_position = actions_n++;
9731         }
9732         /*
9733          * For multiple destination (sample action with ratio=1), the encap
9734          * action and port id action will be combined into group action.
9735          * So need remove the original these actions in the flow and only
9736          * use the sample action instead of.
9737          */
9738         if (num_of_dest > 1 && sample_act->dr_port_id_action) {
9739                 int i;
9740                 void *temp_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
9741
9742                 for (i = 0; i < actions_n; i++) {
9743                         if ((sample_act->dr_encap_action &&
9744                                 sample_act->dr_encap_action ==
9745                                 dev_flow->dv.actions[i]) ||
9746                                 (sample_act->dr_port_id_action &&
9747                                 sample_act->dr_port_id_action ==
9748                                 dev_flow->dv.actions[i]))
9749                                 continue;
9750                         temp_actions[tmp_actions_n++] = dev_flow->dv.actions[i];
9751                 }
9752                 memcpy((void *)dev_flow->dv.actions,
9753                                 (void *)temp_actions,
9754                                 tmp_actions_n * sizeof(void *));
9755                 actions_n = tmp_actions_n;
9756         }
9757         dev_flow->dv.actions_n = actions_n;
9758         dev_flow->act_flags = action_flags;
9759         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
9760                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
9761                 int item_type = items->type;
9762
9763                 if (!mlx5_flow_os_item_supported(item_type))
9764                         return rte_flow_error_set(error, ENOTSUP,
9765                                                   RTE_FLOW_ERROR_TYPE_ITEM,
9766                                                   NULL, "item not supported");
9767                 switch (item_type) {
9768                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
9769                         flow_dv_translate_item_port_id(dev, match_mask,
9770                                                        match_value, items);
9771                         last_item = MLX5_FLOW_ITEM_PORT_ID;
9772                         break;
9773                 case RTE_FLOW_ITEM_TYPE_ETH:
9774                         flow_dv_translate_item_eth(match_mask, match_value,
9775                                                    items, tunnel,
9776                                                    dev_flow->dv.group);
9777                         matcher.priority = action_flags &
9778                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
9779                                         !dev_flow->external ?
9780                                         MLX5_PRIORITY_MAP_L3 :
9781                                         MLX5_PRIORITY_MAP_L2;
9782                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
9783                                              MLX5_FLOW_LAYER_OUTER_L2;
9784                         break;
9785                 case RTE_FLOW_ITEM_TYPE_VLAN:
9786                         flow_dv_translate_item_vlan(dev_flow,
9787                                                     match_mask, match_value,
9788                                                     items, tunnel,
9789                                                     dev_flow->dv.group);
9790                         matcher.priority = MLX5_PRIORITY_MAP_L2;
9791                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
9792                                               MLX5_FLOW_LAYER_INNER_VLAN) :
9793                                              (MLX5_FLOW_LAYER_OUTER_L2 |
9794                                               MLX5_FLOW_LAYER_OUTER_VLAN);
9795                         break;
9796                 case RTE_FLOW_ITEM_TYPE_IPV4:
9797                         mlx5_flow_tunnel_ip_check(items, next_protocol,
9798                                                   &item_flags, &tunnel);
9799                         flow_dv_translate_item_ipv4(match_mask, match_value,
9800                                                     items, item_flags, tunnel,
9801                                                     dev_flow->dv.group);
9802                         matcher.priority = MLX5_PRIORITY_MAP_L3;
9803                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
9804                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
9805                         if (items->mask != NULL &&
9806                             ((const struct rte_flow_item_ipv4 *)
9807                              items->mask)->hdr.next_proto_id) {
9808                                 next_protocol =
9809                                         ((const struct rte_flow_item_ipv4 *)
9810                                          (items->spec))->hdr.next_proto_id;
9811                                 next_protocol &=
9812                                         ((const struct rte_flow_item_ipv4 *)
9813                                          (items->mask))->hdr.next_proto_id;
9814                         } else {
9815                                 /* Reset for inner layer. */
9816                                 next_protocol = 0xff;
9817                         }
9818                         break;
9819                 case RTE_FLOW_ITEM_TYPE_IPV6:
9820                         mlx5_flow_tunnel_ip_check(items, next_protocol,
9821                                                   &item_flags, &tunnel);
9822                         flow_dv_translate_item_ipv6(match_mask, match_value,
9823                                                     items, item_flags, tunnel,
9824                                                     dev_flow->dv.group);
9825                         matcher.priority = MLX5_PRIORITY_MAP_L3;
9826                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
9827                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
9828                         if (items->mask != NULL &&
9829                             ((const struct rte_flow_item_ipv6 *)
9830                              items->mask)->hdr.proto) {
9831                                 next_protocol =
9832                                         ((const struct rte_flow_item_ipv6 *)
9833                                          items->spec)->hdr.proto;
9834                                 next_protocol &=
9835                                         ((const struct rte_flow_item_ipv6 *)
9836                                          items->mask)->hdr.proto;
9837                         } else {
9838                                 /* Reset for inner layer. */
9839                                 next_protocol = 0xff;
9840                         }
9841                         break;
9842                 case RTE_FLOW_ITEM_TYPE_TCP:
9843                         flow_dv_translate_item_tcp(match_mask, match_value,
9844                                                    items, tunnel);
9845                         matcher.priority = MLX5_PRIORITY_MAP_L4;
9846                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
9847                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
9848                         break;
9849                 case RTE_FLOW_ITEM_TYPE_UDP:
9850                         flow_dv_translate_item_udp(match_mask, match_value,
9851                                                    items, tunnel);
9852                         matcher.priority = MLX5_PRIORITY_MAP_L4;
9853                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
9854                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
9855                         break;
9856                 case RTE_FLOW_ITEM_TYPE_GRE:
9857                         flow_dv_translate_item_gre(match_mask, match_value,
9858                                                    items, tunnel);
9859                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9860                         last_item = MLX5_FLOW_LAYER_GRE;
9861                         break;
9862                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
9863                         flow_dv_translate_item_gre_key(match_mask,
9864                                                        match_value, items);
9865                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
9866                         break;
9867                 case RTE_FLOW_ITEM_TYPE_NVGRE:
9868                         flow_dv_translate_item_nvgre(match_mask, match_value,
9869                                                      items, tunnel);
9870                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9871                         last_item = MLX5_FLOW_LAYER_GRE;
9872                         break;
9873                 case RTE_FLOW_ITEM_TYPE_VXLAN:
9874                         flow_dv_translate_item_vxlan(match_mask, match_value,
9875                                                      items, tunnel);
9876                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9877                         last_item = MLX5_FLOW_LAYER_VXLAN;
9878                         break;
9879                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
9880                         flow_dv_translate_item_vxlan_gpe(match_mask,
9881                                                          match_value, items,
9882                                                          tunnel);
9883                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9884                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
9885                         break;
9886                 case RTE_FLOW_ITEM_TYPE_GENEVE:
9887                         flow_dv_translate_item_geneve(match_mask, match_value,
9888                                                       items, tunnel);
9889                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9890                         last_item = MLX5_FLOW_LAYER_GENEVE;
9891                         break;
9892                 case RTE_FLOW_ITEM_TYPE_MPLS:
9893                         flow_dv_translate_item_mpls(match_mask, match_value,
9894                                                     items, last_item, tunnel);
9895                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9896                         last_item = MLX5_FLOW_LAYER_MPLS;
9897                         break;
9898                 case RTE_FLOW_ITEM_TYPE_MARK:
9899                         flow_dv_translate_item_mark(dev, match_mask,
9900                                                     match_value, items);
9901                         last_item = MLX5_FLOW_ITEM_MARK;
9902                         break;
9903                 case RTE_FLOW_ITEM_TYPE_META:
9904                         flow_dv_translate_item_meta(dev, match_mask,
9905                                                     match_value, attr, items);
9906                         last_item = MLX5_FLOW_ITEM_METADATA;
9907                         break;
9908                 case RTE_FLOW_ITEM_TYPE_ICMP:
9909                         flow_dv_translate_item_icmp(match_mask, match_value,
9910                                                     items, tunnel);
9911                         last_item = MLX5_FLOW_LAYER_ICMP;
9912                         break;
9913                 case RTE_FLOW_ITEM_TYPE_ICMP6:
9914                         flow_dv_translate_item_icmp6(match_mask, match_value,
9915                                                       items, tunnel);
9916                         last_item = MLX5_FLOW_LAYER_ICMP6;
9917                         break;
9918                 case RTE_FLOW_ITEM_TYPE_TAG:
9919                         flow_dv_translate_item_tag(dev, match_mask,
9920                                                    match_value, items);
9921                         last_item = MLX5_FLOW_ITEM_TAG;
9922                         break;
9923                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
9924                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
9925                                                         match_value, items);
9926                         last_item = MLX5_FLOW_ITEM_TAG;
9927                         break;
9928                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
9929                         flow_dv_translate_item_tx_queue(dev, match_mask,
9930                                                         match_value,
9931                                                         items);
9932                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
9933                         break;
9934                 case RTE_FLOW_ITEM_TYPE_GTP:
9935                         flow_dv_translate_item_gtp(match_mask, match_value,
9936                                                    items, tunnel);
9937                         matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
9938                         last_item = MLX5_FLOW_LAYER_GTP;
9939                         break;
9940                 case RTE_FLOW_ITEM_TYPE_ECPRI:
9941                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
9942                                 /* Create it only the first time to be used. */
9943                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
9944                                 if (ret)
9945                                         return rte_flow_error_set
9946                                                 (error, -ret,
9947                                                 RTE_FLOW_ERROR_TYPE_ITEM,
9948                                                 NULL,
9949                                                 "cannot create eCPRI parser");
9950                         }
9951                         /* Adjust the length matcher and device flow value. */
9952                         matcher.mask.size = MLX5_ST_SZ_BYTES(fte_match_param);
9953                         dev_flow->dv.value.size =
9954                                         MLX5_ST_SZ_BYTES(fte_match_param);
9955                         flow_dv_translate_item_ecpri(dev, match_mask,
9956                                                      match_value, items);
9957                         /* No other protocol should follow eCPRI layer. */
9958                         last_item = MLX5_FLOW_LAYER_ECPRI;
9959                         break;
9960                 default:
9961                         break;
9962                 }
9963                 item_flags |= last_item;
9964         }
9965         /*
9966          * When E-Switch mode is enabled, we have two cases where we need to
9967          * set the source port manually.
9968          * The first one, is in case of Nic steering rule, and the second is
9969          * E-Switch rule where no port_id item was found. In both cases
9970          * the source port is set according the current port in use.
9971          */
9972         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
9973             (priv->representor || priv->master)) {
9974                 if (flow_dv_translate_item_port_id(dev, match_mask,
9975                                                    match_value, NULL))
9976                         return -rte_errno;
9977         }
9978 #ifdef RTE_LIBRTE_MLX5_DEBUG
9979         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
9980                                               dev_flow->dv.value.buf));
9981 #endif
9982         /*
9983          * Layers may be already initialized from prefix flow if this dev_flow
9984          * is the suffix flow.
9985          */
9986         handle->layers |= item_flags;
9987         if (action_flags & MLX5_FLOW_ACTION_RSS)
9988                 flow_dv_hashfields_set(dev_flow, rss_desc);
9989         /* Register matcher. */
9990         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
9991                                     matcher.mask.size);
9992         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
9993                                                      matcher.priority);
9994         /* reserved field no needs to be set to 0 here. */
9995         tbl_key.domain = attr->transfer;
9996         tbl_key.direction = attr->egress;
9997         tbl_key.table_id = dev_flow->dv.group;
9998         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
9999                 return -rte_errno;
10000         return 0;
10001 }
10002
10003 /**
10004  * Apply the flow to the NIC, lock free,
10005  * (mutex should be acquired by caller).
10006  *
10007  * @param[in] dev
10008  *   Pointer to the Ethernet device structure.
10009  * @param[in, out] flow
10010  *   Pointer to flow structure.
10011  * @param[out] error
10012  *   Pointer to error structure.
10013  *
10014  * @return
10015  *   0 on success, a negative errno value otherwise and rte_errno is set.
10016  */
10017 static int
10018 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
10019                 struct rte_flow_error *error)
10020 {
10021         struct mlx5_flow_dv_workspace *dv;
10022         struct mlx5_flow_handle *dh;
10023         struct mlx5_flow_handle_dv *dv_h;
10024         struct mlx5_flow *dev_flow;
10025         struct mlx5_priv *priv = dev->data->dev_private;
10026         uint32_t handle_idx;
10027         int n;
10028         int err;
10029         int idx;
10030
10031         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
10032                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
10033                 dv = &dev_flow->dv;
10034                 dh = dev_flow->handle;
10035                 dv_h = &dh->dvh;
10036                 n = dv->actions_n;
10037                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
10038                         if (dv->transfer) {
10039                                 dv->actions[n++] = priv->sh->esw_drop_action;
10040                         } else {
10041                                 struct mlx5_hrxq *drop_hrxq;
10042                                 drop_hrxq = mlx5_drop_action_create(dev);
10043                                 if (!drop_hrxq) {
10044                                         rte_flow_error_set
10045                                                 (error, errno,
10046                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10047                                                  NULL,
10048                                                  "cannot get drop hash queue");
10049                                         goto error;
10050                                 }
10051                                 /*
10052                                  * Drop queues will be released by the specify
10053                                  * mlx5_drop_action_destroy() function. Assign
10054                                  * the special index to hrxq to mark the queue
10055                                  * has been allocated.
10056                                  */
10057                                 dh->rix_hrxq = UINT32_MAX;
10058                                 dv->actions[n++] = drop_hrxq->action;
10059                         }
10060                 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
10061                            !dv_h->rix_sample && !dv_h->rix_dest_array) {
10062                         struct mlx5_hrxq *hrxq;
10063                         uint32_t hrxq_idx;
10064                         struct mlx5_flow_rss_desc *rss_desc =
10065                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
10066                                 [!!priv->flow_nested_idx];
10067
10068                         MLX5_ASSERT(rss_desc->queue_num);
10069                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
10070                                                  MLX5_RSS_HASH_KEY_LEN,
10071                                                  dev_flow->hash_fields,
10072                                                  rss_desc->queue,
10073                                                  rss_desc->queue_num);
10074                         if (!hrxq_idx) {
10075                                 hrxq_idx = mlx5_hrxq_new
10076                                                 (dev, rss_desc->key,
10077                                                  MLX5_RSS_HASH_KEY_LEN,
10078                                                  dev_flow->hash_fields,
10079                                                  rss_desc->queue,
10080                                                  rss_desc->queue_num,
10081                                                  !!(dh->layers &
10082                                                  MLX5_FLOW_LAYER_TUNNEL));
10083                         }
10084                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
10085                                               hrxq_idx);
10086                         if (!hrxq) {
10087                                 rte_flow_error_set
10088                                         (error, rte_errno,
10089                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10090                                          "cannot get hash queue");
10091                                 goto error;
10092                         }
10093                         dh->rix_hrxq = hrxq_idx;
10094                         dv->actions[n++] = hrxq->action;
10095                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
10096                         if (flow_dv_default_miss_resource_register
10097                                         (dev, error)) {
10098                                 rte_flow_error_set
10099                                         (error, rte_errno,
10100                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
10101                                          "cannot create default miss resource");
10102                                 goto error_default_miss;
10103                         }
10104                         dh->rix_default_fate =  MLX5_FLOW_FATE_DEFAULT_MISS;
10105                         dv->actions[n++] = priv->sh->default_miss.action;
10106                 }
10107                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
10108                                                (void *)&dv->value, n,
10109                                                dv->actions, &dh->drv_flow);
10110                 if (err) {
10111                         rte_flow_error_set(error, errno,
10112                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10113                                            NULL,
10114                                            "hardware refuses to create flow");
10115                         goto error;
10116                 }
10117                 if (priv->vmwa_context &&
10118                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
10119                         /*
10120                          * The rule contains the VLAN pattern.
10121                          * For VF we are going to create VLAN
10122                          * interface to make hypervisor set correct
10123                          * e-Switch vport context.
10124                          */
10125                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
10126                 }
10127         }
10128         return 0;
10129 error:
10130         if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
10131                 flow_dv_default_miss_resource_release(dev);
10132 error_default_miss:
10133         err = rte_errno; /* Save rte_errno before cleanup. */
10134         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
10135                        handle_idx, dh, next) {
10136                 /* hrxq is union, don't clear it if the flag is not set. */
10137                 if (dh->rix_hrxq) {
10138                         if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
10139                                 mlx5_drop_action_destroy(dev);
10140                                 dh->rix_hrxq = 0;
10141                         } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
10142                                 mlx5_hrxq_release(dev, dh->rix_hrxq);
10143                                 dh->rix_hrxq = 0;
10144                         }
10145                 }
10146                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
10147                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
10148         }
10149         rte_errno = err; /* Restore rte_errno. */
10150         return -rte_errno;
10151 }
10152
10153 /**
10154  * Release the flow matcher.
10155  *
10156  * @param dev
10157  *   Pointer to Ethernet device.
10158  * @param handle
10159  *   Pointer to mlx5_flow_handle.
10160  *
10161  * @return
10162  *   1 while a reference on it exists, 0 when freed.
10163  */
10164 static int
10165 flow_dv_matcher_release(struct rte_eth_dev *dev,
10166                         struct mlx5_flow_handle *handle)
10167 {
10168         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
10169
10170         MLX5_ASSERT(matcher->matcher_object);
10171         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
10172                 dev->data->port_id, (void *)matcher,
10173                 rte_atomic32_read(&matcher->refcnt));
10174         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
10175                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10176                            (matcher->matcher_object));
10177                 LIST_REMOVE(matcher, next);
10178                 /* table ref-- in release interface. */
10179                 flow_dv_tbl_resource_release(dev, matcher->tbl);
10180                 mlx5_free(matcher);
10181                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
10182                         dev->data->port_id, (void *)matcher);
10183                 return 0;
10184         }
10185         return 1;
10186 }
10187
10188 /**
10189  * Release an encap/decap resource.
10190  *
10191  * @param dev
10192  *   Pointer to Ethernet device.
10193  * @param encap_decap_idx
10194  *   Index of encap decap resource.
10195  *
10196  * @return
10197  *   1 while a reference on it exists, 0 when freed.
10198  */
10199 static int
10200 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
10201                                      uint32_t encap_decap_idx)
10202 {
10203         struct mlx5_priv *priv = dev->data->dev_private;
10204         uint32_t idx = encap_decap_idx;
10205         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
10206
10207         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
10208                          idx);
10209         if (!cache_resource)
10210                 return 0;
10211         MLX5_ASSERT(cache_resource->action);
10212         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
10213                 (void *)cache_resource,
10214                 rte_atomic32_read(&cache_resource->refcnt));
10215         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10216                 claim_zero(mlx5_flow_os_destroy_flow_action
10217                                                 (cache_resource->action));
10218                 mlx5_hlist_remove(priv->sh->encaps_decaps,
10219                                   &cache_resource->entry);
10220                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
10221                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
10222                         (void *)cache_resource);
10223                 return 0;
10224         }
10225         return 1;
10226 }
10227
10228 /**
10229  * Release an jump to table action resource.
10230  *
10231  * @param dev
10232  *   Pointer to Ethernet device.
10233  * @param handle
10234  *   Pointer to mlx5_flow_handle.
10235  *
10236  * @return
10237  *   1 while a reference on it exists, 0 when freed.
10238  */
10239 static int
10240 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
10241                                   struct mlx5_flow_handle *handle)
10242 {
10243         struct mlx5_priv *priv = dev->data->dev_private;
10244         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
10245         struct mlx5_flow_tbl_data_entry *tbl_data;
10246
10247         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
10248                              handle->rix_jump);
10249         if (!tbl_data)
10250                 return 0;
10251         cache_resource = &tbl_data->jump;
10252         MLX5_ASSERT(cache_resource->action);
10253         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
10254                 (void *)cache_resource,
10255                 rte_atomic32_read(&cache_resource->refcnt));
10256         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10257                 claim_zero(mlx5_flow_os_destroy_flow_action
10258                                                 (cache_resource->action));
10259                 /* jump action memory free is inside the table release. */
10260                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
10261                 DRV_LOG(DEBUG, "jump table resource %p: removed",
10262                         (void *)cache_resource);
10263                 return 0;
10264         }
10265         return 1;
10266 }
10267
10268 /**
10269  * Release a default miss resource.
10270  *
10271  * @param dev
10272  *   Pointer to Ethernet device.
10273  * @return
10274  *   1 while a reference on it exists, 0 when freed.
10275  */
10276 static int
10277 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
10278 {
10279         struct mlx5_priv *priv = dev->data->dev_private;
10280         struct mlx5_dev_ctx_shared *sh = priv->sh;
10281         struct mlx5_flow_default_miss_resource *cache_resource =
10282                         &sh->default_miss;
10283
10284         MLX5_ASSERT(cache_resource->action);
10285         DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--",
10286                         (void *)cache_resource->action,
10287                         rte_atomic32_read(&cache_resource->refcnt));
10288         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10289                 claim_zero(mlx5_glue->destroy_flow_action
10290                                 (cache_resource->action));
10291                 DRV_LOG(DEBUG, "default miss resource %p: removed",
10292                                 (void *)cache_resource->action);
10293                 return 0;
10294         }
10295         return 1;
10296 }
10297
10298 /**
10299  * Release a modify-header resource.
10300  *
10301  * @param dev
10302  *   Pointer to Ethernet device.
10303  * @param handle
10304  *   Pointer to mlx5_flow_handle.
10305  *
10306  * @return
10307  *   1 while a reference on it exists, 0 when freed.
10308  */
10309 static int
10310 flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
10311                                     struct mlx5_flow_handle *handle)
10312 {
10313         struct mlx5_priv *priv = dev->data->dev_private;
10314         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
10315                                                         handle->dvh.modify_hdr;
10316
10317         MLX5_ASSERT(cache_resource->action);
10318         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
10319                 (void *)cache_resource,
10320                 rte_atomic32_read(&cache_resource->refcnt));
10321         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10322                 claim_zero(mlx5_flow_os_destroy_flow_action
10323                                                 (cache_resource->action));
10324                 mlx5_hlist_remove(priv->sh->modify_cmds,
10325                                   &cache_resource->entry);
10326                 mlx5_free(cache_resource);
10327                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
10328                         (void *)cache_resource);
10329                 return 0;
10330         }
10331         return 1;
10332 }
10333
10334 /**
10335  * Release port ID action resource.
10336  *
10337  * @param dev
10338  *   Pointer to Ethernet device.
10339  * @param handle
10340  *   Pointer to mlx5_flow_handle.
10341  *
10342  * @return
10343  *   1 while a reference on it exists, 0 when freed.
10344  */
10345 static int
10346 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
10347                                         uint32_t port_id)
10348 {
10349         struct mlx5_priv *priv = dev->data->dev_private;
10350         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
10351         uint32_t idx = port_id;
10352
10353         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
10354                                         idx);
10355         if (!cache_resource)
10356                 return 0;
10357         MLX5_ASSERT(cache_resource->action);
10358         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
10359                 (void *)cache_resource,
10360                 rte_atomic32_read(&cache_resource->refcnt));
10361         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10362                 claim_zero(mlx5_flow_os_destroy_flow_action
10363                                                 (cache_resource->action));
10364                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
10365                              &priv->sh->port_id_action_list, idx,
10366                              cache_resource, next);
10367                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx);
10368                 DRV_LOG(DEBUG, "port id action resource %p: removed",
10369                         (void *)cache_resource);
10370                 return 0;
10371         }
10372         return 1;
10373 }
10374
10375 /**
10376  * Release push vlan action resource.
10377  *
10378  * @param dev
10379  *   Pointer to Ethernet device.
10380  * @param handle
10381  *   Pointer to mlx5_flow_handle.
10382  *
10383  * @return
10384  *   1 while a reference on it exists, 0 when freed.
10385  */
10386 static int
10387 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
10388                                           struct mlx5_flow_handle *handle)
10389 {
10390         struct mlx5_priv *priv = dev->data->dev_private;
10391         uint32_t idx = handle->dvh.rix_push_vlan;
10392         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
10393
10394         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
10395                                         idx);
10396         if (!cache_resource)
10397                 return 0;
10398         MLX5_ASSERT(cache_resource->action);
10399         DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
10400                 (void *)cache_resource,
10401                 rte_atomic32_read(&cache_resource->refcnt));
10402         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
10403                 claim_zero(mlx5_flow_os_destroy_flow_action
10404                                                 (cache_resource->action));
10405                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
10406                              &priv->sh->push_vlan_action_list, idx,
10407                              cache_resource, next);
10408                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
10409                 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
10410                         (void *)cache_resource);
10411                 return 0;
10412         }
10413         return 1;
10414 }
10415
10416 /**
10417  * Release the fate resource.
10418  *
10419  * @param dev
10420  *   Pointer to Ethernet device.
10421  * @param handle
10422  *   Pointer to mlx5_flow_handle.
10423  */
10424 static void
10425 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
10426                                struct mlx5_flow_handle *handle)
10427 {
10428         if (!handle->rix_fate)
10429                 return;
10430         switch (handle->fate_action) {
10431         case MLX5_FLOW_FATE_DROP:
10432                 mlx5_drop_action_destroy(dev);
10433                 break;
10434         case MLX5_FLOW_FATE_QUEUE:
10435                 mlx5_hrxq_release(dev, handle->rix_hrxq);
10436                 break;
10437         case MLX5_FLOW_FATE_JUMP:
10438                 flow_dv_jump_tbl_resource_release(dev, handle);
10439                 break;
10440         case MLX5_FLOW_FATE_PORT_ID:
10441                 flow_dv_port_id_action_resource_release(dev,
10442                                 handle->rix_port_id_action);
10443                 break;
10444         case MLX5_FLOW_FATE_DEFAULT_MISS:
10445                 flow_dv_default_miss_resource_release(dev);
10446                 break;
10447         default:
10448                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
10449                 break;
10450         }
10451         handle->rix_fate = 0;
10452 }
10453
10454 /**
10455  * Release an sample resource.
10456  *
10457  * @param dev
10458  *   Pointer to Ethernet device.
10459  * @param handle
10460  *   Pointer to mlx5_flow_handle.
10461  *
10462  * @return
10463  *   1 while a reference on it exists, 0 when freed.
10464  */
10465 static int
10466 flow_dv_sample_resource_release(struct rte_eth_dev *dev,
10467                                      struct mlx5_flow_handle *handle)
10468 {
10469         struct mlx5_priv *priv = dev->data->dev_private;
10470         uint32_t idx = handle->dvh.rix_sample;
10471         struct mlx5_flow_dv_sample_resource *cache_resource;
10472
10473         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
10474                          idx);
10475         if (!cache_resource)
10476                 return 0;
10477         MLX5_ASSERT(cache_resource->verbs_action);
10478         DRV_LOG(DEBUG, "sample resource %p: refcnt %d--",
10479                 (void *)cache_resource,
10480                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10481         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10482                                __ATOMIC_RELAXED) == 0) {
10483                 if (cache_resource->verbs_action)
10484                         claim_zero(mlx5_glue->destroy_flow_action
10485                                         (cache_resource->verbs_action));
10486                 if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
10487                         if (cache_resource->default_miss)
10488                                 claim_zero(mlx5_glue->destroy_flow_action
10489                                   (cache_resource->default_miss));
10490                 }
10491                 if (cache_resource->normal_path_tbl)
10492                         flow_dv_tbl_resource_release(dev,
10493                                 cache_resource->normal_path_tbl);
10494         }
10495         if (cache_resource->sample_idx.rix_hrxq &&
10496                 !mlx5_hrxq_release(dev,
10497                         cache_resource->sample_idx.rix_hrxq))
10498                 cache_resource->sample_idx.rix_hrxq = 0;
10499         if (cache_resource->sample_idx.rix_tag &&
10500                 !flow_dv_tag_release(dev,
10501                         cache_resource->sample_idx.rix_tag))
10502                 cache_resource->sample_idx.rix_tag = 0;
10503         if (cache_resource->sample_idx.cnt) {
10504                 flow_dv_counter_release(dev,
10505                         cache_resource->sample_idx.cnt);
10506                 cache_resource->sample_idx.cnt = 0;
10507         }
10508         if (!__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)) {
10509                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
10510                              &priv->sh->sample_action_list, idx,
10511                              cache_resource, next);
10512                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], idx);
10513                 DRV_LOG(DEBUG, "sample resource %p: removed",
10514                         (void *)cache_resource);
10515                 return 0;
10516         }
10517         return 1;
10518 }
10519
10520 /**
10521  * Release an destination array resource.
10522  *
10523  * @param dev
10524  *   Pointer to Ethernet device.
10525  * @param handle
10526  *   Pointer to mlx5_flow_handle.
10527  *
10528  * @return
10529  *   1 while a reference on it exists, 0 when freed.
10530  */
10531 static int
10532 flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
10533                                      struct mlx5_flow_handle *handle)
10534 {
10535         struct mlx5_priv *priv = dev->data->dev_private;
10536         struct mlx5_flow_dv_dest_array_resource *cache_resource;
10537         struct mlx5_flow_sub_actions_idx *mdest_act_res;
10538         uint32_t idx = handle->dvh.rix_dest_array;
10539         uint32_t i = 0;
10540
10541         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10542                          idx);
10543         if (!cache_resource)
10544                 return 0;
10545         MLX5_ASSERT(cache_resource->action);
10546         DRV_LOG(DEBUG, "destination array resource %p: refcnt %d--",
10547                 (void *)cache_resource,
10548                 __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
10549         if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
10550                                __ATOMIC_RELAXED) == 0) {
10551                 if (cache_resource->action)
10552                         claim_zero(mlx5_glue->destroy_flow_action
10553                                                 (cache_resource->action));
10554                 for (; i < cache_resource->num_of_dest; i++) {
10555                         mdest_act_res = &cache_resource->sample_idx[i];
10556                         if (mdest_act_res->rix_hrxq) {
10557                                 mlx5_hrxq_release(dev,
10558                                         mdest_act_res->rix_hrxq);
10559                                 mdest_act_res->rix_hrxq = 0;
10560                         }
10561                         if (mdest_act_res->rix_encap_decap) {
10562                                 flow_dv_encap_decap_resource_release(dev,
10563                                         mdest_act_res->rix_encap_decap);
10564                                 mdest_act_res->rix_encap_decap = 0;
10565                         }
10566                         if (mdest_act_res->rix_port_id_action) {
10567                                 flow_dv_port_id_action_resource_release(dev,
10568                                         mdest_act_res->rix_port_id_action);
10569                                 mdest_act_res->rix_port_id_action = 0;
10570                         }
10571                         if (mdest_act_res->rix_tag) {
10572                                 flow_dv_tag_release(dev,
10573                                         mdest_act_res->rix_tag);
10574                                 mdest_act_res->rix_tag = 0;
10575                         }
10576                 }
10577                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
10578                              &priv->sh->dest_array_list, idx,
10579                              cache_resource, next);
10580                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], idx);
10581                 DRV_LOG(DEBUG, "destination array resource %p: removed",
10582                         (void *)cache_resource);
10583                 return 0;
10584         }
10585         return 1;
10586 }
10587
10588 /**
10589  * Remove the flow from the NIC but keeps it in memory.
10590  * Lock free, (mutex should be acquired by caller).
10591  *
10592  * @param[in] dev
10593  *   Pointer to Ethernet device.
10594  * @param[in, out] flow
10595  *   Pointer to flow structure.
10596  */
10597 static void
10598 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
10599 {
10600         struct mlx5_flow_handle *dh;
10601         uint32_t handle_idx;
10602         struct mlx5_priv *priv = dev->data->dev_private;
10603
10604         if (!flow)
10605                 return;
10606         handle_idx = flow->dev_handles;
10607         while (handle_idx) {
10608                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10609                                     handle_idx);
10610                 if (!dh)
10611                         return;
10612                 if (dh->drv_flow) {
10613                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
10614                         dh->drv_flow = NULL;
10615                 }
10616                 if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
10617                     dh->fate_action == MLX5_FLOW_FATE_QUEUE ||
10618                     dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
10619                         flow_dv_fate_resource_release(dev, dh);
10620                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
10621                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
10622                 handle_idx = dh->next.next;
10623         }
10624 }
10625
10626 /**
10627  * Remove the flow from the NIC and the memory.
10628  * Lock free, (mutex should be acquired by caller).
10629  *
10630  * @param[in] dev
10631  *   Pointer to the Ethernet device structure.
10632  * @param[in, out] flow
10633  *   Pointer to flow structure.
10634  */
10635 static void
10636 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
10637 {
10638         struct mlx5_flow_handle *dev_handle;
10639         struct mlx5_priv *priv = dev->data->dev_private;
10640
10641         if (!flow)
10642                 return;
10643         __flow_dv_remove(dev, flow);
10644         if (flow->counter) {
10645                 flow_dv_counter_release(dev, flow->counter);
10646                 flow->counter = 0;
10647         }
10648         if (flow->meter) {
10649                 struct mlx5_flow_meter *fm;
10650
10651                 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
10652                                     flow->meter);
10653                 if (fm)
10654                         mlx5_flow_meter_detach(fm);
10655                 flow->meter = 0;
10656         }
10657         while (flow->dev_handles) {
10658                 uint32_t tmp_idx = flow->dev_handles;
10659
10660                 dev_handle = mlx5_ipool_get(priv->sh->ipool
10661                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
10662                 if (!dev_handle)
10663                         return;
10664                 flow->dev_handles = dev_handle->next.next;
10665                 if (dev_handle->dvh.matcher)
10666                         flow_dv_matcher_release(dev, dev_handle);
10667                 if (dev_handle->dvh.rix_sample)
10668                         flow_dv_sample_resource_release(dev, dev_handle);
10669                 if (dev_handle->dvh.rix_dest_array)
10670                         flow_dv_dest_array_resource_release(dev, dev_handle);
10671                 if (dev_handle->dvh.rix_encap_decap)
10672                         flow_dv_encap_decap_resource_release(dev,
10673                                 dev_handle->dvh.rix_encap_decap);
10674                 if (dev_handle->dvh.modify_hdr)
10675                         flow_dv_modify_hdr_resource_release(dev, dev_handle);
10676                 if (dev_handle->dvh.rix_push_vlan)
10677                         flow_dv_push_vlan_action_resource_release(dev,
10678                                                                   dev_handle);
10679                 if (dev_handle->dvh.rix_tag)
10680                         flow_dv_tag_release(dev,
10681                                             dev_handle->dvh.rix_tag);
10682                 flow_dv_fate_resource_release(dev, dev_handle);
10683                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
10684                            tmp_idx);
10685         }
10686 }
10687
10688 /**
10689  * Query a dv flow  rule for its statistics via devx.
10690  *
10691  * @param[in] dev
10692  *   Pointer to Ethernet device.
10693  * @param[in] flow
10694  *   Pointer to the sub flow.
10695  * @param[out] data
10696  *   data retrieved by the query.
10697  * @param[out] error
10698  *   Perform verbose error reporting if not NULL.
10699  *
10700  * @return
10701  *   0 on success, a negative errno value otherwise and rte_errno is set.
10702  */
10703 static int
10704 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
10705                     void *data, struct rte_flow_error *error)
10706 {
10707         struct mlx5_priv *priv = dev->data->dev_private;
10708         struct rte_flow_query_count *qc = data;
10709
10710         if (!priv->config.devx)
10711                 return rte_flow_error_set(error, ENOTSUP,
10712                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10713                                           NULL,
10714                                           "counters are not supported");
10715         if (flow->counter) {
10716                 uint64_t pkts, bytes;
10717                 struct mlx5_flow_counter *cnt;
10718
10719                 cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
10720                                                  NULL);
10721                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
10722                                                &bytes);
10723
10724                 if (err)
10725                         return rte_flow_error_set(error, -err,
10726                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10727                                         NULL, "cannot read counters");
10728                 qc->hits_set = 1;
10729                 qc->bytes_set = 1;
10730                 qc->hits = pkts - cnt->hits;
10731                 qc->bytes = bytes - cnt->bytes;
10732                 if (qc->reset) {
10733                         cnt->hits = pkts;
10734                         cnt->bytes = bytes;
10735                 }
10736                 return 0;
10737         }
10738         return rte_flow_error_set(error, EINVAL,
10739                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
10740                                   NULL,
10741                                   "counters are not available");
10742 }
10743
10744 /**
10745  * Query a flow.
10746  *
10747  * @see rte_flow_query()
10748  * @see rte_flow_ops
10749  */
10750 static int
10751 flow_dv_query(struct rte_eth_dev *dev,
10752               struct rte_flow *flow __rte_unused,
10753               const struct rte_flow_action *actions __rte_unused,
10754               void *data __rte_unused,
10755               struct rte_flow_error *error __rte_unused)
10756 {
10757         int ret = -EINVAL;
10758
10759         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
10760                 switch (actions->type) {
10761                 case RTE_FLOW_ACTION_TYPE_VOID:
10762                         break;
10763                 case RTE_FLOW_ACTION_TYPE_COUNT:
10764                         ret = flow_dv_query_count(dev, flow, data, error);
10765                         break;
10766                 default:
10767                         return rte_flow_error_set(error, ENOTSUP,
10768                                                   RTE_FLOW_ERROR_TYPE_ACTION,
10769                                                   actions,
10770                                                   "action not supported");
10771                 }
10772         }
10773         return ret;
10774 }
10775
10776 /**
10777  * Destroy the meter table set.
10778  * Lock free, (mutex should be acquired by caller).
10779  *
10780  * @param[in] dev
10781  *   Pointer to Ethernet device.
10782  * @param[in] tbl
10783  *   Pointer to the meter table set.
10784  *
10785  * @return
10786  *   Always 0.
10787  */
10788 static int
10789 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
10790                         struct mlx5_meter_domains_infos *tbl)
10791 {
10792         struct mlx5_priv *priv = dev->data->dev_private;
10793         struct mlx5_meter_domains_infos *mtd =
10794                                 (struct mlx5_meter_domains_infos *)tbl;
10795
10796         if (!mtd || !priv->config.dv_flow_en)
10797                 return 0;
10798         if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
10799                 claim_zero(mlx5_flow_os_destroy_flow
10800                            (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
10801         if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
10802                 claim_zero(mlx5_flow_os_destroy_flow
10803                            (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
10804         if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
10805                 claim_zero(mlx5_flow_os_destroy_flow
10806                            (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
10807         if (mtd->egress.color_matcher)
10808                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10809                            (mtd->egress.color_matcher));
10810         if (mtd->egress.any_matcher)
10811                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10812                            (mtd->egress.any_matcher));
10813         if (mtd->egress.tbl)
10814                 flow_dv_tbl_resource_release(dev, mtd->egress.tbl);
10815         if (mtd->egress.sfx_tbl)
10816                 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl);
10817         if (mtd->ingress.color_matcher)
10818                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10819                            (mtd->ingress.color_matcher));
10820         if (mtd->ingress.any_matcher)
10821                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10822                            (mtd->ingress.any_matcher));
10823         if (mtd->ingress.tbl)
10824                 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl);
10825         if (mtd->ingress.sfx_tbl)
10826                 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl);
10827         if (mtd->transfer.color_matcher)
10828                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10829                            (mtd->transfer.color_matcher));
10830         if (mtd->transfer.any_matcher)
10831                 claim_zero(mlx5_flow_os_destroy_flow_matcher
10832                            (mtd->transfer.any_matcher));
10833         if (mtd->transfer.tbl)
10834                 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl);
10835         if (mtd->transfer.sfx_tbl)
10836                 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl);
10837         if (mtd->drop_actn)
10838                 claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn));
10839         mlx5_free(mtd);
10840         return 0;
10841 }
10842
10843 /* Number of meter flow actions, count and jump or count and drop. */
10844 #define METER_ACTIONS 2
10845
10846 /**
10847  * Create specify domain meter table and suffix table.
10848  *
10849  * @param[in] dev
10850  *   Pointer to Ethernet device.
10851  * @param[in,out] mtb
10852  *   Pointer to DV meter table set.
10853  * @param[in] egress
10854  *   Table attribute.
10855  * @param[in] transfer
10856  *   Table attribute.
10857  * @param[in] color_reg_c_idx
10858  *   Reg C index for color match.
10859  *
10860  * @return
10861  *   0 on success, -1 otherwise and rte_errno is set.
10862  */
10863 static int
10864 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
10865                            struct mlx5_meter_domains_infos *mtb,
10866                            uint8_t egress, uint8_t transfer,
10867                            uint32_t color_reg_c_idx)
10868 {
10869         struct mlx5_priv *priv = dev->data->dev_private;
10870         struct mlx5_dev_ctx_shared *sh = priv->sh;
10871         struct mlx5_flow_dv_match_params mask = {
10872                 .size = sizeof(mask.buf),
10873         };
10874         struct mlx5_flow_dv_match_params value = {
10875                 .size = sizeof(value.buf),
10876         };
10877         struct mlx5dv_flow_matcher_attr dv_attr = {
10878                 .type = IBV_FLOW_ATTR_NORMAL,
10879                 .priority = 0,
10880                 .match_criteria_enable = 0,
10881                 .match_mask = (void *)&mask,
10882         };
10883         void *actions[METER_ACTIONS];
10884         struct mlx5_meter_domain_info *dtb;
10885         struct rte_flow_error error;
10886         int i = 0;
10887         int ret;
10888
10889         if (transfer)
10890                 dtb = &mtb->transfer;
10891         else if (egress)
10892                 dtb = &mtb->egress;
10893         else
10894                 dtb = &mtb->ingress;
10895         /* Create the meter table with METER level. */
10896         dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
10897                                             egress, transfer, &error);
10898         if (!dtb->tbl) {
10899                 DRV_LOG(ERR, "Failed to create meter policer table.");
10900                 return -1;
10901         }
10902         /* Create the meter suffix table with SUFFIX level. */
10903         dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
10904                                             MLX5_FLOW_TABLE_LEVEL_SUFFIX,
10905                                             egress, transfer, &error);
10906         if (!dtb->sfx_tbl) {
10907                 DRV_LOG(ERR, "Failed to create meter suffix table.");
10908                 return -1;
10909         }
10910         /* Create matchers, Any and Color. */
10911         dv_attr.priority = 3;
10912         dv_attr.match_criteria_enable = 0;
10913         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
10914                                                &dtb->any_matcher);
10915         if (ret) {
10916                 DRV_LOG(ERR, "Failed to create meter"
10917                              " policer default matcher.");
10918                 goto error_exit;
10919         }
10920         dv_attr.priority = 0;
10921         dv_attr.match_criteria_enable =
10922                                 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
10923         flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
10924                                rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
10925         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
10926                                                &dtb->color_matcher);
10927         if (ret) {
10928                 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
10929                 goto error_exit;
10930         }
10931         if (mtb->count_actns[RTE_MTR_DROPPED])
10932                 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
10933         actions[i++] = mtb->drop_actn;
10934         /* Default rule: lowest priority, match any, actions: drop. */
10935         ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i,
10936                                        actions,
10937                                        &dtb->policer_rules[RTE_MTR_DROPPED]);
10938         if (ret) {
10939                 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
10940                 goto error_exit;
10941         }
10942         return 0;
10943 error_exit:
10944         return -1;
10945 }
10946
10947 /**
10948  * Create the needed meter and suffix tables.
10949  * Lock free, (mutex should be acquired by caller).
10950  *
10951  * @param[in] dev
10952  *   Pointer to Ethernet device.
10953  * @param[in] fm
10954  *   Pointer to the flow meter.
10955  *
10956  * @return
10957  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
10958  */
10959 static struct mlx5_meter_domains_infos *
10960 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
10961                        const struct mlx5_flow_meter *fm)
10962 {
10963         struct mlx5_priv *priv = dev->data->dev_private;
10964         struct mlx5_meter_domains_infos *mtb;
10965         int ret;
10966         int i;
10967
10968         if (!priv->mtr_en) {
10969                 rte_errno = ENOTSUP;
10970                 return NULL;
10971         }
10972         mtb = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*mtb), 0, SOCKET_ID_ANY);
10973         if (!mtb) {
10974                 DRV_LOG(ERR, "Failed to allocate memory for meter.");
10975                 return NULL;
10976         }
10977         /* Create meter count actions */
10978         for (i = 0; i <= RTE_MTR_DROPPED; i++) {
10979                 struct mlx5_flow_counter *cnt;
10980                 if (!fm->policer_stats.cnt[i])
10981                         continue;
10982                 cnt = flow_dv_counter_get_by_idx(dev,
10983                       fm->policer_stats.cnt[i], NULL);
10984                 mtb->count_actns[i] = cnt->action;
10985         }
10986         /* Create drop action. */
10987         ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn);
10988         if (ret) {
10989                 DRV_LOG(ERR, "Failed to create drop action.");
10990                 goto error_exit;
10991         }
10992         /* Egress meter table. */
10993         ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
10994         if (ret) {
10995                 DRV_LOG(ERR, "Failed to prepare egress meter table.");
10996                 goto error_exit;
10997         }
10998         /* Ingress meter table. */
10999         ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
11000         if (ret) {
11001                 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
11002                 goto error_exit;
11003         }
11004         /* FDB meter table. */
11005         if (priv->config.dv_esw_en) {
11006                 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
11007                                                  priv->mtr_color_reg);
11008                 if (ret) {
11009                         DRV_LOG(ERR, "Failed to prepare fdb meter table.");
11010                         goto error_exit;
11011                 }
11012         }
11013         return mtb;
11014 error_exit:
11015         flow_dv_destroy_mtr_tbl(dev, mtb);
11016         return NULL;
11017 }
11018
11019 /**
11020  * Destroy domain policer rule.
11021  *
11022  * @param[in] dt
11023  *   Pointer to domain table.
11024  */
11025 static void
11026 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
11027 {
11028         int i;
11029
11030         for (i = 0; i < RTE_MTR_DROPPED; i++) {
11031                 if (dt->policer_rules[i]) {
11032                         claim_zero(mlx5_flow_os_destroy_flow
11033                                    (dt->policer_rules[i]));
11034                         dt->policer_rules[i] = NULL;
11035                 }
11036         }
11037         if (dt->jump_actn) {
11038                 claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn));
11039                 dt->jump_actn = NULL;
11040         }
11041 }
11042
11043 /**
11044  * Destroy policer rules.
11045  *
11046  * @param[in] dev
11047  *   Pointer to Ethernet device.
11048  * @param[in] fm
11049  *   Pointer to flow meter structure.
11050  * @param[in] attr
11051  *   Pointer to flow attributes.
11052  *
11053  * @return
11054  *   Always 0.
11055  */
11056 static int
11057 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
11058                               const struct mlx5_flow_meter *fm,
11059                               const struct rte_flow_attr *attr)
11060 {
11061         struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
11062
11063         if (!mtb)
11064                 return 0;
11065         if (attr->egress)
11066                 flow_dv_destroy_domain_policer_rule(&mtb->egress);
11067         if (attr->ingress)
11068                 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
11069         if (attr->transfer)
11070                 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
11071         return 0;
11072 }
11073
11074 /**
11075  * Create specify domain meter policer rule.
11076  *
11077  * @param[in] fm
11078  *   Pointer to flow meter structure.
11079  * @param[in] mtb
11080  *   Pointer to DV meter table set.
11081  * @param[in] mtr_reg_c
11082  *   Color match REG_C.
11083  *
11084  * @return
11085  *   0 on success, -1 otherwise.
11086  */
11087 static int
11088 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
11089                                     struct mlx5_meter_domain_info *dtb,
11090                                     uint8_t mtr_reg_c)
11091 {
11092         struct mlx5_flow_dv_match_params matcher = {
11093                 .size = sizeof(matcher.buf),
11094         };
11095         struct mlx5_flow_dv_match_params value = {
11096                 .size = sizeof(value.buf),
11097         };
11098         struct mlx5_meter_domains_infos *mtb = fm->mfts;
11099         void *actions[METER_ACTIONS];
11100         int i;
11101         int ret = 0;
11102
11103         /* Create jump action. */
11104         if (!dtb->jump_actn)
11105                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
11106                                 (dtb->sfx_tbl->obj, &dtb->jump_actn);
11107         if (ret) {
11108                 DRV_LOG(ERR, "Failed to create policer jump action.");
11109                 goto error;
11110         }
11111         for (i = 0; i < RTE_MTR_DROPPED; i++) {
11112                 int j = 0;
11113
11114                 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
11115                                        rte_col_2_mlx5_col(i), UINT8_MAX);
11116                 if (mtb->count_actns[i])
11117                         actions[j++] = mtb->count_actns[i];
11118                 if (fm->action[i] == MTR_POLICER_ACTION_DROP)
11119                         actions[j++] = mtb->drop_actn;
11120                 else
11121                         actions[j++] = dtb->jump_actn;
11122                 ret = mlx5_flow_os_create_flow(dtb->color_matcher,
11123                                                (void *)&value, j, actions,
11124                                                &dtb->policer_rules[i]);
11125                 if (ret) {
11126                         DRV_LOG(ERR, "Failed to create policer rule.");
11127                         goto error;
11128                 }
11129         }
11130         return 0;
11131 error:
11132         rte_errno = errno;
11133         return -1;
11134 }
11135
11136 /**
11137  * Create policer rules.
11138  *
11139  * @param[in] dev
11140  *   Pointer to Ethernet device.
11141  * @param[in] fm
11142  *   Pointer to flow meter structure.
11143  * @param[in] attr
11144  *   Pointer to flow attributes.
11145  *
11146  * @return
11147  *   0 on success, -1 otherwise.
11148  */
11149 static int
11150 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
11151                              struct mlx5_flow_meter *fm,
11152                              const struct rte_flow_attr *attr)
11153 {
11154         struct mlx5_priv *priv = dev->data->dev_private;
11155         struct mlx5_meter_domains_infos *mtb = fm->mfts;
11156         int ret;
11157
11158         if (attr->egress) {
11159                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
11160                                                 priv->mtr_color_reg);
11161                 if (ret) {
11162                         DRV_LOG(ERR, "Failed to create egress policer.");
11163                         goto error;
11164                 }
11165         }
11166         if (attr->ingress) {
11167                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
11168                                                 priv->mtr_color_reg);
11169                 if (ret) {
11170                         DRV_LOG(ERR, "Failed to create ingress policer.");
11171                         goto error;
11172                 }
11173         }
11174         if (attr->transfer) {
11175                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
11176                                                 priv->mtr_color_reg);
11177                 if (ret) {
11178                         DRV_LOG(ERR, "Failed to create transfer policer.");
11179                         goto error;
11180                 }
11181         }
11182         return 0;
11183 error:
11184         flow_dv_destroy_policer_rules(dev, fm, attr);
11185         return -1;
11186 }
11187
11188 /**
11189  * Query a devx counter.
11190  *
11191  * @param[in] dev
11192  *   Pointer to the Ethernet device structure.
11193  * @param[in] cnt
11194  *   Index to the flow counter.
11195  * @param[in] clear
11196  *   Set to clear the counter statistics.
11197  * @param[out] pkts
11198  *   The statistics value of packets.
11199  * @param[out] bytes
11200  *   The statistics value of bytes.
11201  *
11202  * @return
11203  *   0 on success, otherwise return -1.
11204  */
11205 static int
11206 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
11207                       uint64_t *pkts, uint64_t *bytes)
11208 {
11209         struct mlx5_priv *priv = dev->data->dev_private;
11210         struct mlx5_flow_counter *cnt;
11211         uint64_t inn_pkts, inn_bytes;
11212         int ret;
11213
11214         if (!priv->config.devx)
11215                 return -1;
11216
11217         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
11218         if (ret)
11219                 return -1;
11220         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
11221         *pkts = inn_pkts - cnt->hits;
11222         *bytes = inn_bytes - cnt->bytes;
11223         if (clear) {
11224                 cnt->hits = inn_pkts;
11225                 cnt->bytes = inn_bytes;
11226         }
11227         return 0;
11228 }
11229
11230 /**
11231  * Get aged-out flows.
11232  *
11233  * @param[in] dev
11234  *   Pointer to the Ethernet device structure.
11235  * @param[in] context
11236  *   The address of an array of pointers to the aged-out flows contexts.
11237  * @param[in] nb_contexts
11238  *   The length of context array pointers.
11239  * @param[out] error
11240  *   Perform verbose error reporting if not NULL. Initialized in case of
11241  *   error only.
11242  *
11243  * @return
11244  *   how many contexts get in success, otherwise negative errno value.
11245  *   if nb_contexts is 0, return the amount of all aged contexts.
11246  *   if nb_contexts is not 0 , return the amount of aged flows reported
11247  *   in the context array.
11248  * @note: only stub for now
11249  */
11250 static int
11251 flow_get_aged_flows(struct rte_eth_dev *dev,
11252                     void **context,
11253                     uint32_t nb_contexts,
11254                     struct rte_flow_error *error)
11255 {
11256         struct mlx5_priv *priv = dev->data->dev_private;
11257         struct mlx5_age_info *age_info;
11258         struct mlx5_age_param *age_param;
11259         struct mlx5_flow_counter *counter;
11260         int nb_flows = 0;
11261
11262         if (nb_contexts && !context)
11263                 return rte_flow_error_set(error, EINVAL,
11264                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
11265                                           NULL,
11266                                           "Should assign at least one flow or"
11267                                           " context to get if nb_contexts != 0");
11268         age_info = GET_PORT_AGE_INFO(priv);
11269         rte_spinlock_lock(&age_info->aged_sl);
11270         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
11271                 nb_flows++;
11272                 if (nb_contexts) {
11273                         age_param = MLX5_CNT_TO_AGE(counter);
11274                         context[nb_flows - 1] = age_param->context;
11275                         if (!(--nb_contexts))
11276                                 break;
11277                 }
11278         }
11279         rte_spinlock_unlock(&age_info->aged_sl);
11280         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
11281         return nb_flows;
11282 }
11283
11284 /*
11285  * Mutex-protected thunk to lock-free  __flow_dv_translate().
11286  */
11287 static int
11288 flow_dv_translate(struct rte_eth_dev *dev,
11289                   struct mlx5_flow *dev_flow,
11290                   const struct rte_flow_attr *attr,
11291                   const struct rte_flow_item items[],
11292                   const struct rte_flow_action actions[],
11293                   struct rte_flow_error *error)
11294 {
11295         int ret;
11296
11297         flow_dv_shared_lock(dev);
11298         ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
11299         flow_dv_shared_unlock(dev);
11300         return ret;
11301 }
11302
11303 /*
11304  * Mutex-protected thunk to lock-free  __flow_dv_apply().
11305  */
11306 static int
11307 flow_dv_apply(struct rte_eth_dev *dev,
11308               struct rte_flow *flow,
11309               struct rte_flow_error *error)
11310 {
11311         int ret;
11312
11313         flow_dv_shared_lock(dev);
11314         ret = __flow_dv_apply(dev, flow, error);
11315         flow_dv_shared_unlock(dev);
11316         return ret;
11317 }
11318
11319 /*
11320  * Mutex-protected thunk to lock-free __flow_dv_remove().
11321  */
11322 static void
11323 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
11324 {
11325         flow_dv_shared_lock(dev);
11326         __flow_dv_remove(dev, flow);
11327         flow_dv_shared_unlock(dev);
11328 }
11329
11330 /*
11331  * Mutex-protected thunk to lock-free __flow_dv_destroy().
11332  */
11333 static void
11334 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
11335 {
11336         flow_dv_shared_lock(dev);
11337         __flow_dv_destroy(dev, flow);
11338         flow_dv_shared_unlock(dev);
11339 }
11340
11341 /*
11342  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
11343  */
11344 static uint32_t
11345 flow_dv_counter_allocate(struct rte_eth_dev *dev)
11346 {
11347         uint32_t cnt;
11348
11349         flow_dv_shared_lock(dev);
11350         cnt = flow_dv_counter_alloc(dev, 0, 0, 1, 0);
11351         flow_dv_shared_unlock(dev);
11352         return cnt;
11353 }
11354
11355 /*
11356  * Mutex-protected thunk to lock-free flow_dv_counter_release().
11357  */
11358 static void
11359 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
11360 {
11361         flow_dv_shared_lock(dev);
11362         flow_dv_counter_release(dev, cnt);
11363         flow_dv_shared_unlock(dev);
11364 }
11365
11366 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
11367         .validate = flow_dv_validate,
11368         .prepare = flow_dv_prepare,
11369         .translate = flow_dv_translate,
11370         .apply = flow_dv_apply,
11371         .remove = flow_dv_remove,
11372         .destroy = flow_dv_destroy,
11373         .query = flow_dv_query,
11374         .create_mtr_tbls = flow_dv_create_mtr_tbl,
11375         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
11376         .create_policer_rules = flow_dv_create_policer_rules,
11377         .destroy_policer_rules = flow_dv_destroy_policer_rules,
11378         .counter_alloc = flow_dv_counter_allocate,
11379         .counter_free = flow_dv_counter_free,
11380         .counter_query = flow_dv_counter_query,
11381         .get_aged_flows = flow_get_aged_flows,
11382 };
11383
11384 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
11385