drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_cycles.h>
  28 #include <rte_ip.h>
  29 #include <rte_gre.h>
  30 #include <rte_vxlan.h>
  31 #include <rte_gtp.h>
  32
  33 #include <mlx5_glue.h>
  34 #include <mlx5_devx_cmds.h>
  35 #include <mlx5_prm.h>
  36
  37 #include "mlx5_defs.h"
  38 #include "mlx5.h"
  39 #include "mlx5_common_os.h"
  40 #include "mlx5_flow.h"
  41 #include "mlx5_rxtx.h"
  42
  43 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  44
  45 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  46 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR_ESWITCH
  50 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  51 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  52 #endif
  53 #endif
  54
  55 #ifndef HAVE_MLX5DV_DR
  56 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  57 #endif
  58
  59 /* VLAN header definitions */
  60 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  61 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  62 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  63 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  64 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  65
  66 union flow_dv_attr {
  67         struct {
  68                 uint32_t valid:1;
  69                 uint32_t ipv4:1;
  70                 uint32_t ipv6:1;
  71                 uint32_t tcp:1;
  72                 uint32_t udp:1;
  73                 uint32_t reserved:27;
  74         };
  75         uint32_t attr;
  76 };
  77
  78 static int
  79 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
  80                              struct mlx5_flow_tbl_resource *tbl);
  81
  82 /**
  83  * Initialize flow attributes structure according to flow items' types.
  84  *
  85  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  86  * mode. For tunnel mode, the items to be modified are the outermost ones.
  87  *
  88  * @param[in] item
  89  *   Pointer to item specification.
  90  * @param[out] attr
  91  *   Pointer to flow attributes structure.
  92  * @param[in] dev_flow
  93  *   Pointer to the sub flow.
  94  * @param[in] tunnel_decap
  95  *   Whether action is after tunnel decapsulation.
  96  */
  97 static void
  98 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
  99                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 100 {
 101         uint64_t layers = dev_flow->handle->layers;
 102
 103         /*
 104          * If layers is already initialized, it means this dev_flow is the
 105          * suffix flow, the layers flags is set by the prefix flow. Need to
 106          * use the layer flags from prefix flow as the suffix flow may not
 107          * have the user defined items as the flow is split.
 108          */
 109         if (layers) {
 110                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 111                         attr->ipv4 = 1;
 112                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 113                         attr->ipv6 = 1;
 114                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 115                         attr->tcp = 1;
 116                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 117                         attr->udp = 1;
 118                 attr->valid = 1;
 119                 return;
 120         }
 121         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 122                 uint8_t next_protocol = 0xff;
 123                 switch (item->type) {
 124                 case RTE_FLOW_ITEM_TYPE_GRE:
 125                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 126                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 127                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 128                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 129                 case RTE_FLOW_ITEM_TYPE_MPLS:
 130                         if (tunnel_decap)
 131                                 attr->attr = 0;
 132                         break;
 133                 case RTE_FLOW_ITEM_TYPE_IPV4:
 134                         if (!attr->ipv6)
 135                                 attr->ipv4 = 1;
 136                         if (item->mask != NULL &&
 137                             ((const struct rte_flow_item_ipv4 *)
 138                             item->mask)->hdr.next_proto_id)
 139                                 next_protocol =
 140                                     ((const struct rte_flow_item_ipv4 *)
 141                                       (item->spec))->hdr.next_proto_id &
 142                                     ((const struct rte_flow_item_ipv4 *)
 143                                       (item->mask))->hdr.next_proto_id;
 144                         if ((next_protocol == IPPROTO_IPIP ||
 145                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 146                                 attr->attr = 0;
 147                         break;
 148                 case RTE_FLOW_ITEM_TYPE_IPV6:
 149                         if (!attr->ipv4)
 150                                 attr->ipv6 = 1;
 151                         if (item->mask != NULL &&
 152                             ((const struct rte_flow_item_ipv6 *)
 153                             item->mask)->hdr.proto)
 154                                 next_protocol =
 155                                     ((const struct rte_flow_item_ipv6 *)
 156                                       (item->spec))->hdr.proto &
 157                                     ((const struct rte_flow_item_ipv6 *)
 158                                       (item->mask))->hdr.proto;
 159                         if ((next_protocol == IPPROTO_IPIP ||
 160                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 161                                 attr->attr = 0;
 162                         break;
 163                 case RTE_FLOW_ITEM_TYPE_UDP:
 164                         if (!attr->tcp)
 165                                 attr->udp = 1;
 166                         break;
 167                 case RTE_FLOW_ITEM_TYPE_TCP:
 168                         if (!attr->udp)
 169                                 attr->tcp = 1;
 170                         break;
 171                 default:
 172                         break;
 173                 }
 174         }
 175         attr->valid = 1;
 176 }
 177
 178 /**
 179  * Convert rte_mtr_color to mlx5 color.
 180  *
 181  * @param[in] rcol
 182  *   rte_mtr_color.
 183  *
 184  * @return
 185  *   mlx5 color.
 186  */
 187 static int
 188 rte_col_2_mlx5_col(enum rte_color rcol)
 189 {
 190         switch (rcol) {
 191         case RTE_COLOR_GREEN:
 192                 return MLX5_FLOW_COLOR_GREEN;
 193         case RTE_COLOR_YELLOW:
 194                 return MLX5_FLOW_COLOR_YELLOW;
 195         case RTE_COLOR_RED:
 196                 return MLX5_FLOW_COLOR_RED;
 197         default:
 198                 break;
 199         }
 200         return MLX5_FLOW_COLOR_UNDEFINED;
 201 }
 202
 203 struct field_modify_info {
 204         uint32_t size; /* Size of field in protocol header, in bytes. */
 205         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 206         enum mlx5_modification_field id;
 207 };
 208
 209 struct field_modify_info modify_eth[] = {
 210         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 211         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 212         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 213         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 214         {0, 0, 0},
 215 };
 216
 217 struct field_modify_info modify_vlan_out_first_vid[] = {
 218         /* Size in bits !!! */
 219         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 220         {0, 0, 0},
 221 };
 222
 223 struct field_modify_info modify_ipv4[] = {
 224         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 225         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 226         {4, 12, MLX5_MODI_OUT_SIPV4},
 227         {4, 16, MLX5_MODI_OUT_DIPV4},
 228         {0, 0, 0},
 229 };
 230
 231 struct field_modify_info modify_ipv6[] = {
 232         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 233         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 234         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 235         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 236         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 237         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 238         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 239         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 240         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 241         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 242         {0, 0, 0},
 243 };
 244
 245 struct field_modify_info modify_udp[] = {
 246         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 247         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 248         {0, 0, 0},
 249 };
 250
 251 struct field_modify_info modify_tcp[] = {
 252         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 253         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 254         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 255         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 256         {0, 0, 0},
 257 };
 258
 259 static void
 260 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 261                           uint8_t next_protocol, uint64_t *item_flags,
 262                           int *tunnel)
 263 {
 264         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 265                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 266         if (next_protocol == IPPROTO_IPIP) {
 267                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 268                 *tunnel = 1;
 269         }
 270         if (next_protocol == IPPROTO_IPV6) {
 271                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 272                 *tunnel = 1;
 273         }
 274 }
 275
 276 /**
 277  * Acquire the synchronizing object to protect multithreaded access
 278  * to shared dv context. Lock occurs only if context is actually
 279  * shared, i.e. we have multiport IB device and representors are
 280  * created.
 281  *
 282  * @param[in] dev
 283  *   Pointer to the rte_eth_dev structure.
 284  */
 285 static void
 286 flow_dv_shared_lock(struct rte_eth_dev *dev)
 287 {
 288         struct mlx5_priv *priv = dev->data->dev_private;
 289         struct mlx5_dev_ctx_shared *sh = priv->sh;
 290
 291         if (sh->dv_refcnt > 1) {
 292                 int ret;
 293
 294                 ret = pthread_mutex_lock(&sh->dv_mutex);
 295                 MLX5_ASSERT(!ret);
 296                 (void)ret;
 297         }
 298 }
 299
 300 static void
 301 flow_dv_shared_unlock(struct rte_eth_dev *dev)
 302 {
 303         struct mlx5_priv *priv = dev->data->dev_private;
 304         struct mlx5_dev_ctx_shared *sh = priv->sh;
 305
 306         if (sh->dv_refcnt > 1) {
 307                 int ret;
 308
 309                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 310                 MLX5_ASSERT(!ret);
 311                 (void)ret;
 312         }
 313 }
 314
 315 /* Update VLAN's VID/PCP based on input rte_flow_action.
 316  *
 317  * @param[in] action
 318  *   Pointer to struct rte_flow_action.
 319  * @param[out] vlan
 320  *   Pointer to struct rte_vlan_hdr.
 321  */
 322 static void
 323 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 324                          struct rte_vlan_hdr *vlan)
 325 {
 326         uint16_t vlan_tci;
 327         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 328                 vlan_tci =
 329                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 330                                                action->conf)->vlan_pcp;
 331                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 332                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 333                 vlan->vlan_tci |= vlan_tci;
 334         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 335                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 336                 vlan->vlan_tci |= rte_be_to_cpu_16
 337                     (((const struct rte_flow_action_of_set_vlan_vid *)
 338                                              action->conf)->vlan_vid);
 339         }
 340 }
 341
 342 /**
 343  * Fetch 1, 2, 3 or 4 byte field from the byte array
 344  * and return as unsigned integer in host-endian format.
 345  *
 346  * @param[in] data
 347  *   Pointer to data array.
 348  * @param[in] size
 349  *   Size of field to extract.
 350  *
 351  * @return
 352  *   converted field in host endian format.
 353  */
 354 static inline uint32_t
 355 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 356 {
 357         uint32_t ret;
 358
 359         switch (size) {
 360         case 1:
 361                 ret = *data;
 362                 break;
 363         case 2:
 364                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 365                 break;
 366         case 3:
 367                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 368                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 369                 break;
 370         case 4:
 371                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 372                 break;
 373         default:
 374                 MLX5_ASSERT(false);
 375                 ret = 0;
 376                 break;
 377         }
 378         return ret;
 379 }
 380
 381 /**
 382  * Convert modify-header action to DV specification.
 383  *
 384  * Data length of each action is determined by provided field description
 385  * and the item mask. Data bit offset and width of each action is determined
 386  * by provided item mask.
 387  *
 388  * @param[in] item
 389  *   Pointer to item specification.
 390  * @param[in] field
 391  *   Pointer to field modification information.
 392  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 393  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 394  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 395  * @param[in] dcopy
 396  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 397  *   Negative offset value sets the same offset as source offset.
 398  *   size field is ignored, value is taken from source field.
 399  * @param[in,out] resource
 400  *   Pointer to the modify-header resource.
 401  * @param[in] type
 402  *   Type of modification.
 403  * @param[out] error
 404  *   Pointer to the error structure.
 405  *
 406  * @return
 407  *   0 on success, a negative errno value otherwise and rte_errno is set.
 408  */
 409 static int
 410 flow_dv_convert_modify_action(struct rte_flow_item *item,
 411                               struct field_modify_info *field,
 412                               struct field_modify_info *dcopy,
 413                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 414                               uint32_t type, struct rte_flow_error *error)
 415 {
 416         uint32_t i = resource->actions_num;
 417         struct mlx5_modification_cmd *actions = resource->actions;
 418
 419         /*
 420          * The item and mask are provided in big-endian format.
 421          * The fields should be presented as in big-endian format either.
 422          * Mask must be always present, it defines the actual field width.
 423          */
 424         MLX5_ASSERT(item->mask);
 425         MLX5_ASSERT(field->size);
 426         do {
 427                 unsigned int size_b;
 428                 unsigned int off_b;
 429                 uint32_t mask;
 430                 uint32_t data;
 431
 432                 if (i >= MLX5_MAX_MODIFY_NUM)
 433                         return rte_flow_error_set(error, EINVAL,
 434                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 435                                  "too many items to modify");
 436                 /* Fetch variable byte size mask from the array. */
 437                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 438                                            field->offset, field->size);
 439                 if (!mask) {
 440                         ++field;
 441                         continue;
 442                 }
 443                 /* Deduce actual data width in bits from mask value. */
 444                 off_b = rte_bsf32(mask);
 445                 size_b = sizeof(uint32_t) * CHAR_BIT -
 446                          off_b - __builtin_clz(mask);
 447                 MLX5_ASSERT(size_b);
 448                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 449                 actions[i] = (struct mlx5_modification_cmd) {
 450                         .action_type = type,
 451                         .field = field->id,
 452                         .offset = off_b,
 453                         .length = size_b,
 454                 };
 455                 /* Convert entire record to expected big-endian format. */
 456                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 457                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 458                         MLX5_ASSERT(dcopy);
 459                         actions[i].dst_field = dcopy->id;
 460                         actions[i].dst_offset =
 461                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 462                         /* Convert entire record to big-endian format. */
 463                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 464                 } else {
 465                         MLX5_ASSERT(item->spec);
 466                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 467                                                    field->offset, field->size);
 468                         /* Shift out the trailing masked bits from data. */
 469                         data = (data & mask) >> off_b;
 470                         actions[i].data1 = rte_cpu_to_be_32(data);
 471                 }
 472                 ++i;
 473                 ++field;
 474         } while (field->size);
 475         if (resource->actions_num == i)
 476                 return rte_flow_error_set(error, EINVAL,
 477                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 478                                           "invalid modification flow item");
 479         resource->actions_num = i;
 480         return 0;
 481 }
 482
 483 /**
 484  * Convert modify-header set IPv4 address action to DV specification.
 485  *
 486  * @param[in,out] resource
 487  *   Pointer to the modify-header resource.
 488  * @param[in] action
 489  *   Pointer to action specification.
 490  * @param[out] error
 491  *   Pointer to the error structure.
 492  *
 493  * @return
 494  *   0 on success, a negative errno value otherwise and rte_errno is set.
 495  */
 496 static int
 497 flow_dv_convert_action_modify_ipv4
 498                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 499                          const struct rte_flow_action *action,
 500                          struct rte_flow_error *error)
 501 {
 502         const struct rte_flow_action_set_ipv4 *conf =
 503                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 504         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 505         struct rte_flow_item_ipv4 ipv4;
 506         struct rte_flow_item_ipv4 ipv4_mask;
 507
 508         memset(&ipv4, 0, sizeof(ipv4));
 509         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 510         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 511                 ipv4.hdr.src_addr = conf->ipv4_addr;
 512                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 513         } else {
 514                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 515                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 516         }
 517         item.spec = &ipv4;
 518         item.mask = &ipv4_mask;
 519         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 520                                              MLX5_MODIFICATION_TYPE_SET, error);
 521 }
 522
 523 /**
 524  * Convert modify-header set IPv6 address action to DV specification.
 525  *
 526  * @param[in,out] resource
 527  *   Pointer to the modify-header resource.
 528  * @param[in] action
 529  *   Pointer to action specification.
 530  * @param[out] error
 531  *   Pointer to the error structure.
 532  *
 533  * @return
 534  *   0 on success, a negative errno value otherwise and rte_errno is set.
 535  */
 536 static int
 537 flow_dv_convert_action_modify_ipv6
 538                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 539                          const struct rte_flow_action *action,
 540                          struct rte_flow_error *error)
 541 {
 542         const struct rte_flow_action_set_ipv6 *conf =
 543                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 544         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 545         struct rte_flow_item_ipv6 ipv6;
 546         struct rte_flow_item_ipv6 ipv6_mask;
 547
 548         memset(&ipv6, 0, sizeof(ipv6));
 549         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 550         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 551                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 552                        sizeof(ipv6.hdr.src_addr));
 553                 memcpy(&ipv6_mask.hdr.src_addr,
 554                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 555                        sizeof(ipv6.hdr.src_addr));
 556         } else {
 557                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 558                        sizeof(ipv6.hdr.dst_addr));
 559                 memcpy(&ipv6_mask.hdr.dst_addr,
 560                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 561                        sizeof(ipv6.hdr.dst_addr));
 562         }
 563         item.spec = &ipv6;
 564         item.mask = &ipv6_mask;
 565         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 566                                              MLX5_MODIFICATION_TYPE_SET, error);
 567 }
 568
 569 /**
 570  * Convert modify-header set MAC address action to DV specification.
 571  *
 572  * @param[in,out] resource
 573  *   Pointer to the modify-header resource.
 574  * @param[in] action
 575  *   Pointer to action specification.
 576  * @param[out] error
 577  *   Pointer to the error structure.
 578  *
 579  * @return
 580  *   0 on success, a negative errno value otherwise and rte_errno is set.
 581  */
 582 static int
 583 flow_dv_convert_action_modify_mac
 584                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 585                          const struct rte_flow_action *action,
 586                          struct rte_flow_error *error)
 587 {
 588         const struct rte_flow_action_set_mac *conf =
 589                 (const struct rte_flow_action_set_mac *)(action->conf);
 590         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 591         struct rte_flow_item_eth eth;
 592         struct rte_flow_item_eth eth_mask;
 593
 594         memset(&eth, 0, sizeof(eth));
 595         memset(&eth_mask, 0, sizeof(eth_mask));
 596         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 597                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 598                        sizeof(eth.src.addr_bytes));
 599                 memcpy(&eth_mask.src.addr_bytes,
 600                        &rte_flow_item_eth_mask.src.addr_bytes,
 601                        sizeof(eth_mask.src.addr_bytes));
 602         } else {
 603                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 604                        sizeof(eth.dst.addr_bytes));
 605                 memcpy(&eth_mask.dst.addr_bytes,
 606                        &rte_flow_item_eth_mask.dst.addr_bytes,
 607                        sizeof(eth_mask.dst.addr_bytes));
 608         }
 609         item.spec = &eth;
 610         item.mask = &eth_mask;
 611         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 612                                              MLX5_MODIFICATION_TYPE_SET, error);
 613 }
 614
 615 /**
 616  * Convert modify-header set VLAN VID action to DV specification.
 617  *
 618  * @param[in,out] resource
 619  *   Pointer to the modify-header resource.
 620  * @param[in] action
 621  *   Pointer to action specification.
 622  * @param[out] error
 623  *   Pointer to the error structure.
 624  *
 625  * @return
 626  *   0 on success, a negative errno value otherwise and rte_errno is set.
 627  */
 628 static int
 629 flow_dv_convert_action_modify_vlan_vid
 630                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 631                          const struct rte_flow_action *action,
 632                          struct rte_flow_error *error)
 633 {
 634         const struct rte_flow_action_of_set_vlan_vid *conf =
 635                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 636         int i = resource->actions_num;
 637         struct mlx5_modification_cmd *actions = resource->actions;
 638         struct field_modify_info *field = modify_vlan_out_first_vid;
 639
 640         if (i >= MLX5_MAX_MODIFY_NUM)
 641                 return rte_flow_error_set(error, EINVAL,
 642                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 643                          "too many items to modify");
 644         actions[i] = (struct mlx5_modification_cmd) {
 645                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 646                 .field = field->id,
 647                 .length = field->size,
 648                 .offset = field->offset,
 649         };
 650         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 651         actions[i].data1 = conf->vlan_vid;
 652         actions[i].data1 = actions[i].data1 << 16;
 653         resource->actions_num = ++i;
 654         return 0;
 655 }
 656
 657 /**
 658  * Convert modify-header set TP action to DV specification.
 659  *
 660  * @param[in,out] resource
 661  *   Pointer to the modify-header resource.
 662  * @param[in] action
 663  *   Pointer to action specification.
 664  * @param[in] items
 665  *   Pointer to rte_flow_item objects list.
 666  * @param[in] attr
 667  *   Pointer to flow attributes structure.
 668  * @param[in] dev_flow
 669  *   Pointer to the sub flow.
 670  * @param[in] tunnel_decap
 671  *   Whether action is after tunnel decapsulation.
 672  * @param[out] error
 673  *   Pointer to the error structure.
 674  *
 675  * @return
 676  *   0 on success, a negative errno value otherwise and rte_errno is set.
 677  */
 678 static int
 679 flow_dv_convert_action_modify_tp
 680                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 681                          const struct rte_flow_action *action,
 682                          const struct rte_flow_item *items,
 683                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 684                          bool tunnel_decap, struct rte_flow_error *error)
 685 {
 686         const struct rte_flow_action_set_tp *conf =
 687                 (const struct rte_flow_action_set_tp *)(action->conf);
 688         struct rte_flow_item item;
 689         struct rte_flow_item_udp udp;
 690         struct rte_flow_item_udp udp_mask;
 691         struct rte_flow_item_tcp tcp;
 692         struct rte_flow_item_tcp tcp_mask;
 693         struct field_modify_info *field;
 694
 695         if (!attr->valid)
 696                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 697         if (attr->udp) {
 698                 memset(&udp, 0, sizeof(udp));
 699                 memset(&udp_mask, 0, sizeof(udp_mask));
 700                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 701                         udp.hdr.src_port = conf->port;
 702                         udp_mask.hdr.src_port =
 703                                         rte_flow_item_udp_mask.hdr.src_port;
 704                 } else {
 705                         udp.hdr.dst_port = conf->port;
 706                         udp_mask.hdr.dst_port =
 707                                         rte_flow_item_udp_mask.hdr.dst_port;
 708                 }
 709                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 710                 item.spec = &udp;
 711                 item.mask = &udp_mask;
 712                 field = modify_udp;
 713         } else {
 714                 MLX5_ASSERT(attr->tcp);
 715                 memset(&tcp, 0, sizeof(tcp));
 716                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 717                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 718                         tcp.hdr.src_port = conf->port;
 719                         tcp_mask.hdr.src_port =
 720                                         rte_flow_item_tcp_mask.hdr.src_port;
 721                 } else {
 722                         tcp.hdr.dst_port = conf->port;
 723                         tcp_mask.hdr.dst_port =
 724                                         rte_flow_item_tcp_mask.hdr.dst_port;
 725                 }
 726                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 727                 item.spec = &tcp;
 728                 item.mask = &tcp_mask;
 729                 field = modify_tcp;
 730         }
 731         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 732                                              MLX5_MODIFICATION_TYPE_SET, error);
 733 }
 734
 735 /**
 736  * Convert modify-header set TTL action to DV specification.
 737  *
 738  * @param[in,out] resource
 739  *   Pointer to the modify-header resource.
 740  * @param[in] action
 741  *   Pointer to action specification.
 742  * @param[in] items
 743  *   Pointer to rte_flow_item objects list.
 744  * @param[in] attr
 745  *   Pointer to flow attributes structure.
 746  * @param[in] dev_flow
 747  *   Pointer to the sub flow.
 748  * @param[in] tunnel_decap
 749  *   Whether action is after tunnel decapsulation.
 750  * @param[out] error
 751  *   Pointer to the error structure.
 752  *
 753  * @return
 754  *   0 on success, a negative errno value otherwise and rte_errno is set.
 755  */
 756 static int
 757 flow_dv_convert_action_modify_ttl
 758                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 759                          const struct rte_flow_action *action,
 760                          const struct rte_flow_item *items,
 761                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 762                          bool tunnel_decap, struct rte_flow_error *error)
 763 {
 764         const struct rte_flow_action_set_ttl *conf =
 765                 (const struct rte_flow_action_set_ttl *)(action->conf);
 766         struct rte_flow_item item;
 767         struct rte_flow_item_ipv4 ipv4;
 768         struct rte_flow_item_ipv4 ipv4_mask;
 769         struct rte_flow_item_ipv6 ipv6;
 770         struct rte_flow_item_ipv6 ipv6_mask;
 771         struct field_modify_info *field;
 772
 773         if (!attr->valid)
 774                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 775         if (attr->ipv4) {
 776                 memset(&ipv4, 0, sizeof(ipv4));
 777                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 778                 ipv4.hdr.time_to_live = conf->ttl_value;
 779                 ipv4_mask.hdr.time_to_live = 0xFF;
 780                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 781                 item.spec = &ipv4;
 782                 item.mask = &ipv4_mask;
 783                 field = modify_ipv4;
 784         } else {
 785                 MLX5_ASSERT(attr->ipv6);
 786                 memset(&ipv6, 0, sizeof(ipv6));
 787                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 788                 ipv6.hdr.hop_limits = conf->ttl_value;
 789                 ipv6_mask.hdr.hop_limits = 0xFF;
 790                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 791                 item.spec = &ipv6;
 792                 item.mask = &ipv6_mask;
 793                 field = modify_ipv6;
 794         }
 795         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 796                                              MLX5_MODIFICATION_TYPE_SET, error);
 797 }
 798
 799 /**
 800  * Convert modify-header decrement TTL action to DV specification.
 801  *
 802  * @param[in,out] resource
 803  *   Pointer to the modify-header resource.
 804  * @param[in] action
 805  *   Pointer to action specification.
 806  * @param[in] items
 807  *   Pointer to rte_flow_item objects list.
 808  * @param[in] attr
 809  *   Pointer to flow attributes structure.
 810  * @param[in] dev_flow
 811  *   Pointer to the sub flow.
 812  * @param[in] tunnel_decap
 813  *   Whether action is after tunnel decapsulation.
 814  * @param[out] error
 815  *   Pointer to the error structure.
 816  *
 817  * @return
 818  *   0 on success, a negative errno value otherwise and rte_errno is set.
 819  */
 820 static int
 821 flow_dv_convert_action_modify_dec_ttl
 822                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 823                          const struct rte_flow_item *items,
 824                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 825                          bool tunnel_decap, struct rte_flow_error *error)
 826 {
 827         struct rte_flow_item item;
 828         struct rte_flow_item_ipv4 ipv4;
 829         struct rte_flow_item_ipv4 ipv4_mask;
 830         struct rte_flow_item_ipv6 ipv6;
 831         struct rte_flow_item_ipv6 ipv6_mask;
 832         struct field_modify_info *field;
 833
 834         if (!attr->valid)
 835                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 836         if (attr->ipv4) {
 837                 memset(&ipv4, 0, sizeof(ipv4));
 838                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 839                 ipv4.hdr.time_to_live = 0xFF;
 840                 ipv4_mask.hdr.time_to_live = 0xFF;
 841                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 842                 item.spec = &ipv4;
 843                 item.mask = &ipv4_mask;
 844                 field = modify_ipv4;
 845         } else {
 846                 MLX5_ASSERT(attr->ipv6);
 847                 memset(&ipv6, 0, sizeof(ipv6));
 848                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 849                 ipv6.hdr.hop_limits = 0xFF;
 850                 ipv6_mask.hdr.hop_limits = 0xFF;
 851                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 852                 item.spec = &ipv6;
 853                 item.mask = &ipv6_mask;
 854                 field = modify_ipv6;
 855         }
 856         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 857                                              MLX5_MODIFICATION_TYPE_ADD, error);
 858 }
 859
 860 /**
 861  * Convert modify-header increment/decrement TCP Sequence number
 862  * to DV specification.
 863  *
 864  * @param[in,out] resource
 865  *   Pointer to the modify-header resource.
 866  * @param[in] action
 867  *   Pointer to action specification.
 868  * @param[out] error
 869  *   Pointer to the error structure.
 870  *
 871  * @return
 872  *   0 on success, a negative errno value otherwise and rte_errno is set.
 873  */
 874 static int
 875 flow_dv_convert_action_modify_tcp_seq
 876                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 877                          const struct rte_flow_action *action,
 878                          struct rte_flow_error *error)
 879 {
 880         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 881         uint64_t value = rte_be_to_cpu_32(*conf);
 882         struct rte_flow_item item;
 883         struct rte_flow_item_tcp tcp;
 884         struct rte_flow_item_tcp tcp_mask;
 885
 886         memset(&tcp, 0, sizeof(tcp));
 887         memset(&tcp_mask, 0, sizeof(tcp_mask));
 888         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 889                 /*
 890                  * The HW has no decrement operation, only increment operation.
 891                  * To simulate decrement X from Y using increment operation
 892                  * we need to add UINT32_MAX X times to Y.
 893                  * Each adding of UINT32_MAX decrements Y by 1.
 894                  */
 895                 value *= UINT32_MAX;
 896         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 897         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 898         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 899         item.spec = &tcp;
 900         item.mask = &tcp_mask;
 901         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 902                                              MLX5_MODIFICATION_TYPE_ADD, error);
 903 }
 904
 905 /**
 906  * Convert modify-header increment/decrement TCP Acknowledgment number
 907  * to DV specification.
 908  *
 909  * @param[in,out] resource
 910  *   Pointer to the modify-header resource.
 911  * @param[in] action
 912  *   Pointer to action specification.
 913  * @param[out] error
 914  *   Pointer to the error structure.
 915  *
 916  * @return
 917  *   0 on success, a negative errno value otherwise and rte_errno is set.
 918  */
 919 static int
 920 flow_dv_convert_action_modify_tcp_ack
 921                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 922                          const struct rte_flow_action *action,
 923                          struct rte_flow_error *error)
 924 {
 925         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 926         uint64_t value = rte_be_to_cpu_32(*conf);
 927         struct rte_flow_item item;
 928         struct rte_flow_item_tcp tcp;
 929         struct rte_flow_item_tcp tcp_mask;
 930
 931         memset(&tcp, 0, sizeof(tcp));
 932         memset(&tcp_mask, 0, sizeof(tcp_mask));
 933         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 934                 /*
 935                  * The HW has no decrement operation, only increment operation.
 936                  * To simulate decrement X from Y using increment operation
 937                  * we need to add UINT32_MAX X times to Y.
 938                  * Each adding of UINT32_MAX decrements Y by 1.
 939                  */
 940                 value *= UINT32_MAX;
 941         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 942         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 943         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 944         item.spec = &tcp;
 945         item.mask = &tcp_mask;
 946         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 947                                              MLX5_MODIFICATION_TYPE_ADD, error);
 948 }
 949
 950 static enum mlx5_modification_field reg_to_field[] = {
 951         [REG_NONE] = MLX5_MODI_OUT_NONE,
 952         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 953         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 954         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 955         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 956         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 957         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 958         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 959         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 960         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 961         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 962 };
 963
 964 /**
 965  * Convert register set to DV specification.
 966  *
 967  * @param[in,out] resource
 968  *   Pointer to the modify-header resource.
 969  * @param[in] action
 970  *   Pointer to action specification.
 971  * @param[out] error
 972  *   Pointer to the error structure.
 973  *
 974  * @return
 975  *   0 on success, a negative errno value otherwise and rte_errno is set.
 976  */
 977 static int
 978 flow_dv_convert_action_set_reg
 979                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 980                          const struct rte_flow_action *action,
 981                          struct rte_flow_error *error)
 982 {
 983         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 984         struct mlx5_modification_cmd *actions = resource->actions;
 985         uint32_t i = resource->actions_num;
 986
 987         if (i >= MLX5_MAX_MODIFY_NUM)
 988                 return rte_flow_error_set(error, EINVAL,
 989                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 990                                           "too many items to modify");
 991         MLX5_ASSERT(conf->id != REG_NONE);
 992         MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field));
 993         actions[i] = (struct mlx5_modification_cmd) {
 994                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 995                 .field = reg_to_field[conf->id],
 996         };
 997         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 998         actions[i].data1 = rte_cpu_to_be_32(conf->data);
 999         ++i;
1000         resource->actions_num = i;
1001         return 0;
1002 }
1003
1004 /**
1005  * Convert SET_TAG action to DV specification.
1006  *
1007  * @param[in] dev
1008  *   Pointer to the rte_eth_dev structure.
1009  * @param[in,out] resource
1010  *   Pointer to the modify-header resource.
1011  * @param[in] conf
1012  *   Pointer to action specification.
1013  * @param[out] error
1014  *   Pointer to the error structure.
1015  *
1016  * @return
1017  *   0 on success, a negative errno value otherwise and rte_errno is set.
1018  */
1019 static int
1020 flow_dv_convert_action_set_tag
1021                         (struct rte_eth_dev *dev,
1022                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1023                          const struct rte_flow_action_set_tag *conf,
1024                          struct rte_flow_error *error)
1025 {
1026         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1027         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1028         struct rte_flow_item item = {
1029                 .spec = &data,
1030                 .mask = &mask,
1031         };
1032         struct field_modify_info reg_c_x[] = {
1033                 [1] = {0, 0, 0},
1034         };
1035         enum mlx5_modification_field reg_type;
1036         int ret;
1037
1038         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1039         if (ret < 0)
1040                 return ret;
1041         MLX5_ASSERT(ret != REG_NONE);
1042         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1043         reg_type = reg_to_field[ret];
1044         MLX5_ASSERT(reg_type > 0);
1045         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1046         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1047                                              MLX5_MODIFICATION_TYPE_SET, error);
1048 }
1049
1050 /**
1051  * Convert internal COPY_REG action to DV specification.
1052  *
1053  * @param[in] dev
1054  *   Pointer to the rte_eth_dev structure.
1055  * @param[in,out] res
1056  *   Pointer to the modify-header resource.
1057  * @param[in] action
1058  *   Pointer to action specification.
1059  * @param[out] error
1060  *   Pointer to the error structure.
1061  *
1062  * @return
1063  *   0 on success, a negative errno value otherwise and rte_errno is set.
1064  */
1065 static int
1066 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1067                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1068                                  const struct rte_flow_action *action,
1069                                  struct rte_flow_error *error)
1070 {
1071         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1072         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1073         struct rte_flow_item item = {
1074                 .spec = NULL,
1075                 .mask = &mask,
1076         };
1077         struct field_modify_info reg_src[] = {
1078                 {4, 0, reg_to_field[conf->src]},
1079                 {0, 0, 0},
1080         };
1081         struct field_modify_info reg_dst = {
1082                 .offset = 0,
1083                 .id = reg_to_field[conf->dst],
1084         };
1085         /* Adjust reg_c[0] usage according to reported mask. */
1086         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1087                 struct mlx5_priv *priv = dev->data->dev_private;
1088                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1089
1090                 MLX5_ASSERT(reg_c0);
1091                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1092                 if (conf->dst == REG_C_0) {
1093                         /* Copy to reg_c[0], within mask only. */
1094                         reg_dst.offset = rte_bsf32(reg_c0);
1095                         /*
1096                          * Mask is ignoring the enianness, because
1097                          * there is no conversion in datapath.
1098                          */
1099 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1100                         /* Copy from destination lower bits to reg_c[0]. */
1101                         mask = reg_c0 >> reg_dst.offset;
1102 #else
1103                         /* Copy from destination upper bits to reg_c[0]. */
1104                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1105                                           rte_fls_u32(reg_c0));
1106 #endif
1107                 } else {
1108                         mask = rte_cpu_to_be_32(reg_c0);
1109 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1110                         /* Copy from reg_c[0] to destination lower bits. */
1111                         reg_dst.offset = 0;
1112 #else
1113                         /* Copy from reg_c[0] to destination upper bits. */
1114                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1115                                          (rte_fls_u32(reg_c0) -
1116                                           rte_bsf32(reg_c0));
1117 #endif
1118                 }
1119         }
1120         return flow_dv_convert_modify_action(&item,
1121                                              reg_src, &reg_dst, res,
1122                                              MLX5_MODIFICATION_TYPE_COPY,
1123                                              error);
1124 }
1125
1126 /**
1127  * Convert MARK action to DV specification. This routine is used
1128  * in extensive metadata only and requires metadata register to be
1129  * handled. In legacy mode hardware tag resource is engaged.
1130  *
1131  * @param[in] dev
1132  *   Pointer to the rte_eth_dev structure.
1133  * @param[in] conf
1134  *   Pointer to MARK action specification.
1135  * @param[in,out] resource
1136  *   Pointer to the modify-header resource.
1137  * @param[out] error
1138  *   Pointer to the error structure.
1139  *
1140  * @return
1141  *   0 on success, a negative errno value otherwise and rte_errno is set.
1142  */
1143 static int
1144 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1145                             const struct rte_flow_action_mark *conf,
1146                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1147                             struct rte_flow_error *error)
1148 {
1149         struct mlx5_priv *priv = dev->data->dev_private;
1150         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1151                                            priv->sh->dv_mark_mask);
1152         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1153         struct rte_flow_item item = {
1154                 .spec = &data,
1155                 .mask = &mask,
1156         };
1157         struct field_modify_info reg_c_x[] = {
1158                 {4, 0, 0}, /* dynamic instead of MLX5_MODI_META_REG_C_1. */
1159                 {0, 0, 0},
1160         };
1161         int reg;
1162
1163         if (!mask)
1164                 return rte_flow_error_set(error, EINVAL,
1165                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1166                                           NULL, "zero mark action mask");
1167         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1168         if (reg < 0)
1169                 return reg;
1170         MLX5_ASSERT(reg > 0);
1171         if (reg == REG_C_0) {
1172                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1173                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1174
1175                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1176                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1177                 mask = rte_cpu_to_be_32(mask << shl_c0);
1178         }
1179         reg_c_x[0].id = reg_to_field[reg];
1180         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1181                                              MLX5_MODIFICATION_TYPE_SET, error);
1182 }
1183
1184 /**
1185  * Get metadata register index for specified steering domain.
1186  *
1187  * @param[in] dev
1188  *   Pointer to the rte_eth_dev structure.
1189  * @param[in] attr
1190  *   Attributes of flow to determine steering domain.
1191  * @param[out] error
1192  *   Pointer to the error structure.
1193  *
1194  * @return
1195  *   positive index on success, a negative errno value otherwise
1196  *   and rte_errno is set.
1197  */
1198 static enum modify_reg
1199 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1200                          const struct rte_flow_attr *attr,
1201                          struct rte_flow_error *error)
1202 {
1203         int reg =
1204                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1205                                           MLX5_METADATA_FDB :
1206                                             attr->egress ?
1207                                             MLX5_METADATA_TX :
1208                                             MLX5_METADATA_RX, 0, error);
1209         if (reg < 0)
1210                 return rte_flow_error_set(error,
1211                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1212                                           NULL, "unavailable "
1213                                           "metadata register");
1214         return reg;
1215 }
1216
1217 /**
1218  * Convert SET_META action to DV specification.
1219  *
1220  * @param[in] dev
1221  *   Pointer to the rte_eth_dev structure.
1222  * @param[in,out] resource
1223  *   Pointer to the modify-header resource.
1224  * @param[in] attr
1225  *   Attributes of flow that includes this item.
1226  * @param[in] conf
1227  *   Pointer to action specification.
1228  * @param[out] error
1229  *   Pointer to the error structure.
1230  *
1231  * @return
1232  *   0 on success, a negative errno value otherwise and rte_errno is set.
1233  */
1234 static int
1235 flow_dv_convert_action_set_meta
1236                         (struct rte_eth_dev *dev,
1237                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1238                          const struct rte_flow_attr *attr,
1239                          const struct rte_flow_action_set_meta *conf,
1240                          struct rte_flow_error *error)
1241 {
1242         uint32_t data = conf->data;
1243         uint32_t mask = conf->mask;
1244         struct rte_flow_item item = {
1245                 .spec = &data,
1246                 .mask = &mask,
1247         };
1248         struct field_modify_info reg_c_x[] = {
1249                 [1] = {0, 0, 0},
1250         };
1251         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1252
1253         if (reg < 0)
1254                 return reg;
1255         /*
1256          * In datapath code there is no endianness
1257          * coversions for perfromance reasons, all
1258          * pattern conversions are done in rte_flow.
1259          */
1260         if (reg == REG_C_0) {
1261                 struct mlx5_priv *priv = dev->data->dev_private;
1262                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1263                 uint32_t shl_c0;
1264
1265                 MLX5_ASSERT(msk_c0);
1266 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1267                 shl_c0 = rte_bsf32(msk_c0);
1268 #else
1269                 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1270 #endif
1271                 mask <<= shl_c0;
1272                 data <<= shl_c0;
1273                 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1274         }
1275         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1276         /* The routine expects parameters in memory as big-endian ones. */
1277         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1278                                              MLX5_MODIFICATION_TYPE_SET, error);
1279 }
1280
1281 /**
1282  * Convert modify-header set IPv4 DSCP action to DV specification.
1283  *
1284  * @param[in,out] resource
1285  *   Pointer to the modify-header resource.
1286  * @param[in] action
1287  *   Pointer to action specification.
1288  * @param[out] error
1289  *   Pointer to the error structure.
1290  *
1291  * @return
1292  *   0 on success, a negative errno value otherwise and rte_errno is set.
1293  */
1294 static int
1295 flow_dv_convert_action_modify_ipv4_dscp
1296                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1297                          const struct rte_flow_action *action,
1298                          struct rte_flow_error *error)
1299 {
1300         const struct rte_flow_action_set_dscp *conf =
1301                 (const struct rte_flow_action_set_dscp *)(action->conf);
1302         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1303         struct rte_flow_item_ipv4 ipv4;
1304         struct rte_flow_item_ipv4 ipv4_mask;
1305
1306         memset(&ipv4, 0, sizeof(ipv4));
1307         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1308         ipv4.hdr.type_of_service = conf->dscp;
1309         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1310         item.spec = &ipv4;
1311         item.mask = &ipv4_mask;
1312         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1313                                              MLX5_MODIFICATION_TYPE_SET, error);
1314 }
1315
1316 /**
1317  * Convert modify-header set IPv6 DSCP action to DV specification.
1318  *
1319  * @param[in,out] resource
1320  *   Pointer to the modify-header resource.
1321  * @param[in] action
1322  *   Pointer to action specification.
1323  * @param[out] error
1324  *   Pointer to the error structure.
1325  *
1326  * @return
1327  *   0 on success, a negative errno value otherwise and rte_errno is set.
1328  */
1329 static int
1330 flow_dv_convert_action_modify_ipv6_dscp
1331                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1332                          const struct rte_flow_action *action,
1333                          struct rte_flow_error *error)
1334 {
1335         const struct rte_flow_action_set_dscp *conf =
1336                 (const struct rte_flow_action_set_dscp *)(action->conf);
1337         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1338         struct rte_flow_item_ipv6 ipv6;
1339         struct rte_flow_item_ipv6 ipv6_mask;
1340
1341         memset(&ipv6, 0, sizeof(ipv6));
1342         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1343         /*
1344          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1345          * rdma-core only accept the DSCP bits byte aligned start from
1346          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1347          * bits in IPv6 case as rdma-core requires byte aligned value.
1348          */
1349         ipv6.hdr.vtc_flow = conf->dscp;
1350         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1351         item.spec = &ipv6;
1352         item.mask = &ipv6_mask;
1353         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1354                                              MLX5_MODIFICATION_TYPE_SET, error);
1355 }
1356
1357 /**
1358  * Validate MARK item.
1359  *
1360  * @param[in] dev
1361  *   Pointer to the rte_eth_dev structure.
1362  * @param[in] item
1363  *   Item specification.
1364  * @param[in] attr
1365  *   Attributes of flow that includes this item.
1366  * @param[out] error
1367  *   Pointer to error structure.
1368  *
1369  * @return
1370  *   0 on success, a negative errno value otherwise and rte_errno is set.
1371  */
1372 static int
1373 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1374                            const struct rte_flow_item *item,
1375                            const struct rte_flow_attr *attr __rte_unused,
1376                            struct rte_flow_error *error)
1377 {
1378         struct mlx5_priv *priv = dev->data->dev_private;
1379         struct mlx5_dev_config *config = &priv->config;
1380         const struct rte_flow_item_mark *spec = item->spec;
1381         const struct rte_flow_item_mark *mask = item->mask;
1382         const struct rte_flow_item_mark nic_mask = {
1383                 .id = priv->sh->dv_mark_mask,
1384         };
1385         int ret;
1386
1387         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1388                 return rte_flow_error_set(error, ENOTSUP,
1389                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1390                                           "extended metadata feature"
1391                                           " isn't enabled");
1392         if (!mlx5_flow_ext_mreg_supported(dev))
1393                 return rte_flow_error_set(error, ENOTSUP,
1394                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1395                                           "extended metadata register"
1396                                           " isn't supported");
1397         if (!nic_mask.id)
1398                 return rte_flow_error_set(error, ENOTSUP,
1399                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1400                                           "extended metadata register"
1401                                           " isn't available");
1402         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1403         if (ret < 0)
1404                 return ret;
1405         if (!spec)
1406                 return rte_flow_error_set(error, EINVAL,
1407                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1408                                           item->spec,
1409                                           "data cannot be empty");
1410         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1411                 return rte_flow_error_set(error, EINVAL,
1412                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1413                                           &spec->id,
1414                                           "mark id exceeds the limit");
1415         if (!mask)
1416                 mask = &nic_mask;
1417         if (!mask->id)
1418                 return rte_flow_error_set(error, EINVAL,
1419                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1420                                         "mask cannot be zero");
1421
1422         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1423                                         (const uint8_t *)&nic_mask,
1424                                         sizeof(struct rte_flow_item_mark),
1425                                         error);
1426         if (ret < 0)
1427                 return ret;
1428         return 0;
1429 }
1430
1431 /**
1432  * Validate META item.
1433  *
1434  * @param[in] dev
1435  *   Pointer to the rte_eth_dev structure.
1436  * @param[in] item
1437  *   Item specification.
1438  * @param[in] attr
1439  *   Attributes of flow that includes this item.
1440  * @param[out] error
1441  *   Pointer to error structure.
1442  *
1443  * @return
1444  *   0 on success, a negative errno value otherwise and rte_errno is set.
1445  */
1446 static int
1447 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1448                            const struct rte_flow_item *item,
1449                            const struct rte_flow_attr *attr,
1450                            struct rte_flow_error *error)
1451 {
1452         struct mlx5_priv *priv = dev->data->dev_private;
1453         struct mlx5_dev_config *config = &priv->config;
1454         const struct rte_flow_item_meta *spec = item->spec;
1455         const struct rte_flow_item_meta *mask = item->mask;
1456         struct rte_flow_item_meta nic_mask = {
1457                 .data = UINT32_MAX
1458         };
1459         int reg;
1460         int ret;
1461
1462         if (!spec)
1463                 return rte_flow_error_set(error, EINVAL,
1464                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1465                                           item->spec,
1466                                           "data cannot be empty");
1467         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1468                 if (!mlx5_flow_ext_mreg_supported(dev))
1469                         return rte_flow_error_set(error, ENOTSUP,
1470                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1471                                           "extended metadata register"
1472                                           " isn't supported");
1473                 reg = flow_dv_get_metadata_reg(dev, attr, error);
1474                 if (reg < 0)
1475                         return reg;
1476                 if (reg == REG_B)
1477                         return rte_flow_error_set(error, ENOTSUP,
1478                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1479                                           "match on reg_b "
1480                                           "isn't supported");
1481                 if (reg != REG_A)
1482                         nic_mask.data = priv->sh->dv_meta_mask;
1483         }
1484         if (!mask)
1485                 mask = &rte_flow_item_meta_mask;
1486         if (!mask->data)
1487                 return rte_flow_error_set(error, EINVAL,
1488                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1489                                         "mask cannot be zero");
1490
1491         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1492                                         (const uint8_t *)&nic_mask,
1493                                         sizeof(struct rte_flow_item_meta),
1494                                         error);
1495         return ret;
1496 }
1497
1498 /**
1499  * Validate TAG item.
1500  *
1501  * @param[in] dev
1502  *   Pointer to the rte_eth_dev structure.
1503  * @param[in] item
1504  *   Item specification.
1505  * @param[in] attr
1506  *   Attributes of flow that includes this item.
1507  * @param[out] error
1508  *   Pointer to error structure.
1509  *
1510  * @return
1511  *   0 on success, a negative errno value otherwise and rte_errno is set.
1512  */
1513 static int
1514 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1515                           const struct rte_flow_item *item,
1516                           const struct rte_flow_attr *attr __rte_unused,
1517                           struct rte_flow_error *error)
1518 {
1519         const struct rte_flow_item_tag *spec = item->spec;
1520         const struct rte_flow_item_tag *mask = item->mask;
1521         const struct rte_flow_item_tag nic_mask = {
1522                 .data = RTE_BE32(UINT32_MAX),
1523                 .index = 0xff,
1524         };
1525         int ret;
1526
1527         if (!mlx5_flow_ext_mreg_supported(dev))
1528                 return rte_flow_error_set(error, ENOTSUP,
1529                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1530                                           "extensive metadata register"
1531                                           " isn't supported");
1532         if (!spec)
1533                 return rte_flow_error_set(error, EINVAL,
1534                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1535                                           item->spec,
1536                                           "data cannot be empty");
1537         if (!mask)
1538                 mask = &rte_flow_item_tag_mask;
1539         if (!mask->data)
1540                 return rte_flow_error_set(error, EINVAL,
1541                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1542                                         "mask cannot be zero");
1543
1544         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1545                                         (const uint8_t *)&nic_mask,
1546                                         sizeof(struct rte_flow_item_tag),
1547                                         error);
1548         if (ret < 0)
1549                 return ret;
1550         if (mask->index != 0xff)
1551                 return rte_flow_error_set(error, EINVAL,
1552                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1553                                           "partial mask for tag index"
1554                                           " is not supported");
1555         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1556         if (ret < 0)
1557                 return ret;
1558         MLX5_ASSERT(ret != REG_NONE);
1559         return 0;
1560 }
1561
1562 /**
1563  * Validate vport item.
1564  *
1565  * @param[in] dev
1566  *   Pointer to the rte_eth_dev structure.
1567  * @param[in] item
1568  *   Item specification.
1569  * @param[in] attr
1570  *   Attributes of flow that includes this item.
1571  * @param[in] item_flags
1572  *   Bit-fields that holds the items detected until now.
1573  * @param[out] error
1574  *   Pointer to error structure.
1575  *
1576  * @return
1577  *   0 on success, a negative errno value otherwise and rte_errno is set.
1578  */
1579 static int
1580 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1581                               const struct rte_flow_item *item,
1582                               const struct rte_flow_attr *attr,
1583                               uint64_t item_flags,
1584                               struct rte_flow_error *error)
1585 {
1586         const struct rte_flow_item_port_id *spec = item->spec;
1587         const struct rte_flow_item_port_id *mask = item->mask;
1588         const struct rte_flow_item_port_id switch_mask = {
1589                         .id = 0xffffffff,
1590         };
1591         struct mlx5_priv *esw_priv;
1592         struct mlx5_priv *dev_priv;
1593         int ret;
1594
1595         if (!attr->transfer)
1596                 return rte_flow_error_set(error, EINVAL,
1597                                           RTE_FLOW_ERROR_TYPE_ITEM,
1598                                           NULL,
1599                                           "match on port id is valid only"
1600                                           " when transfer flag is enabled");
1601         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1602                 return rte_flow_error_set(error, ENOTSUP,
1603                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1604                                           "multiple source ports are not"
1605                                           " supported");
1606         if (!mask)
1607                 mask = &switch_mask;
1608         if (mask->id != 0xffffffff)
1609                 return rte_flow_error_set(error, ENOTSUP,
1610                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1611                                            mask,
1612                                            "no support for partial mask on"
1613                                            " \"id\" field");
1614         ret = mlx5_flow_item_acceptable
1615                                 (item, (const uint8_t *)mask,
1616                                  (const uint8_t *)&rte_flow_item_port_id_mask,
1617                                  sizeof(struct rte_flow_item_port_id),
1618                                  error);
1619         if (ret)
1620                 return ret;
1621         if (!spec)
1622                 return 0;
1623         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1624         if (!esw_priv)
1625                 return rte_flow_error_set(error, rte_errno,
1626                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1627                                           "failed to obtain E-Switch info for"
1628                                           " port");
1629         dev_priv = mlx5_dev_to_eswitch_info(dev);
1630         if (!dev_priv)
1631                 return rte_flow_error_set(error, rte_errno,
1632                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1633                                           NULL,
1634                                           "failed to obtain E-Switch info");
1635         if (esw_priv->domain_id != dev_priv->domain_id)
1636                 return rte_flow_error_set(error, EINVAL,
1637                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1638                                           "cannot match on a port from a"
1639                                           " different E-Switch");
1640         return 0;
1641 }
1642
1643 /**
1644  * Validate VLAN item.
1645  *
1646  * @param[in] item
1647  *   Item specification.
1648  * @param[in] item_flags
1649  *   Bit-fields that holds the items detected until now.
1650  * @param[in] dev
1651  *   Ethernet device flow is being created on.
1652  * @param[out] error
1653  *   Pointer to error structure.
1654  *
1655  * @return
1656  *   0 on success, a negative errno value otherwise and rte_errno is set.
1657  */
1658 static int
1659 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
1660                            uint64_t item_flags,
1661                            struct rte_eth_dev *dev,
1662                            struct rte_flow_error *error)
1663 {
1664         const struct rte_flow_item_vlan *mask = item->mask;
1665         const struct rte_flow_item_vlan nic_mask = {
1666                 .tci = RTE_BE16(UINT16_MAX),
1667                 .inner_type = RTE_BE16(UINT16_MAX),
1668         };
1669         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1670         int ret;
1671         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1672                                         MLX5_FLOW_LAYER_INNER_L4) :
1673                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1674                                         MLX5_FLOW_LAYER_OUTER_L4);
1675         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1676                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1677
1678         if (item_flags & vlanm)
1679                 return rte_flow_error_set(error, EINVAL,
1680                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1681                                           "multiple VLAN layers not supported");
1682         else if ((item_flags & l34m) != 0)
1683                 return rte_flow_error_set(error, EINVAL,
1684                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1685                                           "VLAN cannot follow L3/L4 layer");
1686         if (!mask)
1687                 mask = &rte_flow_item_vlan_mask;
1688         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1689                                         (const uint8_t *)&nic_mask,
1690                                         sizeof(struct rte_flow_item_vlan),
1691                                         error);
1692         if (ret)
1693                 return ret;
1694         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1695                 struct mlx5_priv *priv = dev->data->dev_private;
1696
1697                 if (priv->vmwa_context) {
1698                         /*
1699                          * Non-NULL context means we have a virtual machine
1700                          * and SR-IOV enabled, we have to create VLAN interface
1701                          * to make hypervisor to setup E-Switch vport
1702                          * context correctly. We avoid creating the multiple
1703                          * VLAN interfaces, so we cannot support VLAN tag mask.
1704                          */
1705                         return rte_flow_error_set(error, EINVAL,
1706                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1707                                                   item,
1708                                                   "VLAN tag mask is not"
1709                                                   " supported in virtual"
1710                                                   " environment");
1711                 }
1712         }
1713         return 0;
1714 }
1715
1716 /*
1717  * GTP flags are contained in 1 byte of the format:
1718  * -------------------------------------------
1719  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
1720  * |-----------------------------------------|
1721  * | value | Version | PT | Res | E | S | PN |
1722  * -------------------------------------------
1723  *
1724  * Matching is supported only for GTP flags E, S, PN.
1725  */
1726 #define MLX5_GTP_FLAGS_MASK     0x07
1727
1728 /**
1729  * Validate GTP item.
1730  *
1731  * @param[in] dev
1732  *   Pointer to the rte_eth_dev structure.
1733  * @param[in] item
1734  *   Item specification.
1735  * @param[in] item_flags
1736  *   Bit-fields that holds the items detected until now.
1737  * @param[out] error
1738  *   Pointer to error structure.
1739  *
1740  * @return
1741  *   0 on success, a negative errno value otherwise and rte_errno is set.
1742  */
1743 static int
1744 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
1745                           const struct rte_flow_item *item,
1746                           uint64_t item_flags,
1747                           struct rte_flow_error *error)
1748 {
1749         struct mlx5_priv *priv = dev->data->dev_private;
1750         const struct rte_flow_item_gtp *spec = item->spec;
1751         const struct rte_flow_item_gtp *mask = item->mask;
1752         const struct rte_flow_item_gtp nic_mask = {
1753                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
1754                 .msg_type = 0xff,
1755                 .teid = RTE_BE32(0xffffffff),
1756         };
1757
1758         if (!priv->config.hca_attr.tunnel_stateless_gtp)
1759                 return rte_flow_error_set(error, ENOTSUP,
1760                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1761                                           "GTP support is not enabled");
1762         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1763                 return rte_flow_error_set(error, ENOTSUP,
1764                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1765                                           "multiple tunnel layers not"
1766                                           " supported");
1767         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1768                 return rte_flow_error_set(error, EINVAL,
1769                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1770                                           "no outer UDP layer found");
1771         if (!mask)
1772                 mask = &rte_flow_item_gtp_mask;
1773         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
1774                 return rte_flow_error_set(error, ENOTSUP,
1775                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1776                                           "Match is supported for GTP"
1777                                           " flags only");
1778         return mlx5_flow_item_acceptable
1779                 (item, (const uint8_t *)mask,
1780                  (const uint8_t *)&nic_mask,
1781                  sizeof(struct rte_flow_item_gtp),
1782                  error);
1783 }
1784
1785 /**
1786  * Validate the pop VLAN action.
1787  *
1788  * @param[in] dev
1789  *   Pointer to the rte_eth_dev structure.
1790  * @param[in] action_flags
1791  *   Holds the actions detected until now.
1792  * @param[in] action
1793  *   Pointer to the pop vlan action.
1794  * @param[in] item_flags
1795  *   The items found in this flow rule.
1796  * @param[in] attr
1797  *   Pointer to flow attributes.
1798  * @param[out] error
1799  *   Pointer to error structure.
1800  *
1801  * @return
1802  *   0 on success, a negative errno value otherwise and rte_errno is set.
1803  */
1804 static int
1805 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
1806                                  uint64_t action_flags,
1807                                  const struct rte_flow_action *action,
1808                                  uint64_t item_flags,
1809                                  const struct rte_flow_attr *attr,
1810                                  struct rte_flow_error *error)
1811 {
1812         const struct mlx5_priv *priv = dev->data->dev_private;
1813
1814         (void)action;
1815         (void)attr;
1816         if (!priv->sh->pop_vlan_action)
1817                 return rte_flow_error_set(error, ENOTSUP,
1818                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1819                                           NULL,
1820                                           "pop vlan action is not supported");
1821         if (attr->egress)
1822                 return rte_flow_error_set(error, ENOTSUP,
1823                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1824                                           NULL,
1825                                           "pop vlan action not supported for "
1826                                           "egress");
1827         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
1828                 return rte_flow_error_set(error, ENOTSUP,
1829                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1830                                           "no support for multiple VLAN "
1831                                           "actions");
1832         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
1833                 return rte_flow_error_set(error, ENOTSUP,
1834                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1835                                           NULL,
1836                                           "cannot pop vlan without a "
1837                                           "match on (outer) vlan in the flow");
1838         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1839                 return rte_flow_error_set(error, EINVAL,
1840                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1841                                           "wrong action order, port_id should "
1842                                           "be after pop VLAN action");
1843         if (!attr->transfer && priv->representor)
1844                 return rte_flow_error_set(error, ENOTSUP,
1845                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1846                                           "pop vlan action for VF representor "
1847                                           "not supported on NIC table");
1848         return 0;
1849 }
1850
1851 /**
1852  * Get VLAN default info from vlan match info.
1853  *
1854  * @param[in] items
1855  *   the list of item specifications.
1856  * @param[out] vlan
1857  *   pointer VLAN info to fill to.
1858  *
1859  * @return
1860  *   0 on success, a negative errno value otherwise and rte_errno is set.
1861  */
1862 static void
1863 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
1864                                   struct rte_vlan_hdr *vlan)
1865 {
1866         const struct rte_flow_item_vlan nic_mask = {
1867                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
1868                                 MLX5DV_FLOW_VLAN_VID_MASK),
1869                 .inner_type = RTE_BE16(0xffff),
1870         };
1871
1872         if (items == NULL)
1873                 return;
1874         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1875                 int type = items->type;
1876
1877                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
1878                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
1879                         break;
1880         }
1881         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
1882                 const struct rte_flow_item_vlan *vlan_m = items->mask;
1883                 const struct rte_flow_item_vlan *vlan_v = items->spec;
1884
1885                 /* If VLAN item in pattern doesn't contain data, return here. */
1886                 if (!vlan_v)
1887                         return;
1888                 if (!vlan_m)
1889                         vlan_m = &nic_mask;
1890                 /* Only full match values are accepted */
1891                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
1892                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
1893                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
1894                         vlan->vlan_tci |=
1895                                 rte_be_to_cpu_16(vlan_v->tci &
1896                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
1897                 }
1898                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
1899                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
1900                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
1901                         vlan->vlan_tci |=
1902                                 rte_be_to_cpu_16(vlan_v->tci &
1903                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
1904                 }
1905                 if (vlan_m->inner_type == nic_mask.inner_type)
1906                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
1907                                                            vlan_m->inner_type);
1908         }
1909 }
1910
1911 /**
1912  * Validate the push VLAN action.
1913  *
1914  * @param[in] dev
1915  *   Pointer to the rte_eth_dev structure.
1916  * @param[in] action_flags
1917  *   Holds the actions detected until now.
1918  * @param[in] item_flags
1919  *   The items found in this flow rule.
1920  * @param[in] action
1921  *   Pointer to the action structure.
1922  * @param[in] attr
1923  *   Pointer to flow attributes
1924  * @param[out] error
1925  *   Pointer to error structure.
1926  *
1927  * @return
1928  *   0 on success, a negative errno value otherwise and rte_errno is set.
1929  */
1930 static int
1931 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
1932                                   uint64_t action_flags,
1933                                   const struct rte_flow_item_vlan *vlan_m,
1934                                   const struct rte_flow_action *action,
1935                                   const struct rte_flow_attr *attr,
1936                                   struct rte_flow_error *error)
1937 {
1938         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
1939         const struct mlx5_priv *priv = dev->data->dev_private;
1940
1941         if (!attr->transfer && attr->ingress)
1942                 return rte_flow_error_set(error, ENOTSUP,
1943                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1944                                           NULL,
1945                                           "push VLAN action not supported for "
1946                                           "ingress");
1947         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
1948             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
1949                 return rte_flow_error_set(error, EINVAL,
1950                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1951                                           "invalid vlan ethertype");
1952         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
1953                 return rte_flow_error_set(error, ENOTSUP,
1954                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1955                                           "no support for multiple VLAN "
1956                                           "actions");
1957         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1958                 return rte_flow_error_set(error, EINVAL,
1959                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1960                                           "wrong action order, port_id should "
1961                                           "be after push VLAN");
1962         if (!attr->transfer && priv->representor)
1963                 return rte_flow_error_set(error, ENOTSUP,
1964                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1965                                           "push vlan action for VF representor "
1966                                           "not supported on NIC table");
1967         if (vlan_m &&
1968             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
1969             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
1970                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
1971             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
1972             !(mlx5_flow_find_action
1973                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
1974                 return rte_flow_error_set(error, EINVAL,
1975                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1976                                           "not full match mask on VLAN PCP and "
1977                                           "there is no of_set_vlan_pcp action, "
1978                                           "push VLAN action cannot figure out "
1979                                           "PCP value");
1980         if (vlan_m &&
1981             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
1982             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
1983                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
1984             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
1985             !(mlx5_flow_find_action
1986                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
1987                 return rte_flow_error_set(error, EINVAL,
1988                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1989                                           "not full match mask on VLAN VID and "
1990                                           "there is no of_set_vlan_vid action, "
1991                                           "push VLAN action cannot figure out "
1992                                           "VID value");
1993         (void)attr;
1994         return 0;
1995 }
1996
1997 /**
1998  * Validate the set VLAN PCP.
1999  *
2000  * @param[in] action_flags
2001  *   Holds the actions detected until now.
2002  * @param[in] actions
2003  *   Pointer to the list of actions remaining in the flow rule.
2004  * @param[out] error
2005  *   Pointer to error structure.
2006  *
2007  * @return
2008  *   0 on success, a negative errno value otherwise and rte_errno is set.
2009  */
2010 static int
2011 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2012                                      const struct rte_flow_action actions[],
2013                                      struct rte_flow_error *error)
2014 {
2015         const struct rte_flow_action *action = actions;
2016         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2017
2018         if (conf->vlan_pcp > 7)
2019                 return rte_flow_error_set(error, EINVAL,
2020                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2021                                           "VLAN PCP value is too big");
2022         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2023                 return rte_flow_error_set(error, ENOTSUP,
2024                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2025                                           "set VLAN PCP action must follow "
2026                                           "the push VLAN action");
2027         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2028                 return rte_flow_error_set(error, ENOTSUP,
2029                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2030                                           "Multiple VLAN PCP modification are "
2031                                           "not supported");
2032         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2033                 return rte_flow_error_set(error, EINVAL,
2034                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2035                                           "wrong action order, port_id should "
2036                                           "be after set VLAN PCP");
2037         return 0;
2038 }
2039
2040 /**
2041  * Validate the set VLAN VID.
2042  *
2043  * @param[in] item_flags
2044  *   Holds the items detected in this rule.
2045  * @param[in] action_flags
2046  *   Holds the actions detected until now.
2047  * @param[in] actions
2048  *   Pointer to the list of actions remaining in the flow rule.
2049  * @param[out] error
2050  *   Pointer to error structure.
2051  *
2052  * @return
2053  *   0 on success, a negative errno value otherwise and rte_errno is set.
2054  */
2055 static int
2056 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2057                                      uint64_t action_flags,
2058                                      const struct rte_flow_action actions[],
2059                                      struct rte_flow_error *error)
2060 {
2061         const struct rte_flow_action *action = actions;
2062         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2063
2064         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2065                 return rte_flow_error_set(error, EINVAL,
2066                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2067                                           "VLAN VID value is too big");
2068         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2069             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2070                 return rte_flow_error_set(error, ENOTSUP,
2071                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2072                                           "set VLAN VID action must follow push"
2073                                           " VLAN action or match on VLAN item");
2074         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2075                 return rte_flow_error_set(error, ENOTSUP,
2076                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2077                                           "Multiple VLAN VID modifications are "
2078                                           "not supported");
2079         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2080                 return rte_flow_error_set(error, EINVAL,
2081                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2082                                           "wrong action order, port_id should "
2083                                           "be after set VLAN VID");
2084         return 0;
2085 }
2086
2087 /*
2088  * Validate the FLAG action.
2089  *
2090  * @param[in] dev
2091  *   Pointer to the rte_eth_dev structure.
2092  * @param[in] action_flags
2093  *   Holds the actions detected until now.
2094  * @param[in] attr
2095  *   Pointer to flow attributes
2096  * @param[out] error
2097  *   Pointer to error structure.
2098  *
2099  * @return
2100  *   0 on success, a negative errno value otherwise and rte_errno is set.
2101  */
2102 static int
2103 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
2104                              uint64_t action_flags,
2105                              const struct rte_flow_attr *attr,
2106                              struct rte_flow_error *error)
2107 {
2108         struct mlx5_priv *priv = dev->data->dev_private;
2109         struct mlx5_dev_config *config = &priv->config;
2110         int ret;
2111
2112         /* Fall back if no extended metadata register support. */
2113         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2114                 return mlx5_flow_validate_action_flag(action_flags, attr,
2115                                                       error);
2116         /* Extensive metadata mode requires registers. */
2117         if (!mlx5_flow_ext_mreg_supported(dev))
2118                 return rte_flow_error_set(error, ENOTSUP,
2119                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2120                                           "no metadata registers "
2121                                           "to support flag action");
2122         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
2123                 return rte_flow_error_set(error, ENOTSUP,
2124                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2125                                           "extended metadata register"
2126                                           " isn't available");
2127         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2128         if (ret < 0)
2129                 return ret;
2130         MLX5_ASSERT(ret > 0);
2131         if (action_flags & MLX5_FLOW_ACTION_MARK)
2132                 return rte_flow_error_set(error, EINVAL,
2133                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2134                                           "can't mark and flag in same flow");
2135         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2136                 return rte_flow_error_set(error, EINVAL,
2137                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2138                                           "can't have 2 flag"
2139                                           " actions in same flow");
2140         return 0;
2141 }
2142
2143 /**
2144  * Validate MARK action.
2145  *
2146  * @param[in] dev
2147  *   Pointer to the rte_eth_dev structure.
2148  * @param[in] action
2149  *   Pointer to action.
2150  * @param[in] action_flags
2151  *   Holds the actions detected until now.
2152  * @param[in] attr
2153  *   Pointer to flow attributes
2154  * @param[out] error
2155  *   Pointer to error structure.
2156  *
2157  * @return
2158  *   0 on success, a negative errno value otherwise and rte_errno is set.
2159  */
2160 static int
2161 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
2162                              const struct rte_flow_action *action,
2163                              uint64_t action_flags,
2164                              const struct rte_flow_attr *attr,
2165                              struct rte_flow_error *error)
2166 {
2167         struct mlx5_priv *priv = dev->data->dev_private;
2168         struct mlx5_dev_config *config = &priv->config;
2169         const struct rte_flow_action_mark *mark = action->conf;
2170         int ret;
2171
2172         /* Fall back if no extended metadata register support. */
2173         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2174                 return mlx5_flow_validate_action_mark(action, action_flags,
2175                                                       attr, error);
2176         /* Extensive metadata mode requires registers. */
2177         if (!mlx5_flow_ext_mreg_supported(dev))
2178                 return rte_flow_error_set(error, ENOTSUP,
2179                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2180                                           "no metadata registers "
2181                                           "to support mark action");
2182         if (!priv->sh->dv_mark_mask)
2183                 return rte_flow_error_set(error, ENOTSUP,
2184                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2185                                           "extended metadata register"
2186                                           " isn't available");
2187         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2188         if (ret < 0)
2189                 return ret;
2190         MLX5_ASSERT(ret > 0);
2191         if (!mark)
2192                 return rte_flow_error_set(error, EINVAL,
2193                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2194                                           "configuration cannot be null");
2195         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
2196                 return rte_flow_error_set(error, EINVAL,
2197                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2198                                           &mark->id,
2199                                           "mark id exceeds the limit");
2200         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2201                 return rte_flow_error_set(error, EINVAL,
2202                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2203                                           "can't flag and mark in same flow");
2204         if (action_flags & MLX5_FLOW_ACTION_MARK)
2205                 return rte_flow_error_set(error, EINVAL,
2206                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2207                                           "can't have 2 mark actions in same"
2208                                           " flow");
2209         return 0;
2210 }
2211
2212 /**
2213  * Validate SET_META action.
2214  *
2215  * @param[in] dev
2216  *   Pointer to the rte_eth_dev structure.
2217  * @param[in] action
2218  *   Pointer to the action structure.
2219  * @param[in] action_flags
2220  *   Holds the actions detected until now.
2221  * @param[in] attr
2222  *   Pointer to flow attributes
2223  * @param[out] error
2224  *   Pointer to error structure.
2225  *
2226  * @return
2227  *   0 on success, a negative errno value otherwise and rte_errno is set.
2228  */
2229 static int
2230 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
2231                                  const struct rte_flow_action *action,
2232                                  uint64_t action_flags __rte_unused,
2233                                  const struct rte_flow_attr *attr,
2234                                  struct rte_flow_error *error)
2235 {
2236         const struct rte_flow_action_set_meta *conf;
2237         uint32_t nic_mask = UINT32_MAX;
2238         int reg;
2239
2240         if (!mlx5_flow_ext_mreg_supported(dev))
2241                 return rte_flow_error_set(error, ENOTSUP,
2242                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2243                                           "extended metadata register"
2244                                           " isn't supported");
2245         reg = flow_dv_get_metadata_reg(dev, attr, error);
2246         if (reg < 0)
2247                 return reg;
2248         if (reg != REG_A && reg != REG_B) {
2249                 struct mlx5_priv *priv = dev->data->dev_private;
2250
2251                 nic_mask = priv->sh->dv_meta_mask;
2252         }
2253         if (!(action->conf))
2254                 return rte_flow_error_set(error, EINVAL,
2255                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2256                                           "configuration cannot be null");
2257         conf = (const struct rte_flow_action_set_meta *)action->conf;
2258         if (!conf->mask)
2259                 return rte_flow_error_set(error, EINVAL,
2260                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2261                                           "zero mask doesn't have any effect");
2262         if (conf->mask & ~nic_mask)
2263                 return rte_flow_error_set(error, EINVAL,
2264                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2265                                           "meta data must be within reg C0");
2266         return 0;
2267 }
2268
2269 /**
2270  * Validate SET_TAG action.
2271  *
2272  * @param[in] dev
2273  *   Pointer to the rte_eth_dev structure.
2274  * @param[in] action
2275  *   Pointer to the action structure.
2276  * @param[in] action_flags
2277  *   Holds the actions detected until now.
2278  * @param[in] attr
2279  *   Pointer to flow attributes
2280  * @param[out] error
2281  *   Pointer to error structure.
2282  *
2283  * @return
2284  *   0 on success, a negative errno value otherwise and rte_errno is set.
2285  */
2286 static int
2287 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
2288                                 const struct rte_flow_action *action,
2289                                 uint64_t action_flags,
2290                                 const struct rte_flow_attr *attr,
2291                                 struct rte_flow_error *error)
2292 {
2293         const struct rte_flow_action_set_tag *conf;
2294         const uint64_t terminal_action_flags =
2295                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
2296                 MLX5_FLOW_ACTION_RSS;
2297         int ret;
2298
2299         if (!mlx5_flow_ext_mreg_supported(dev))
2300                 return rte_flow_error_set(error, ENOTSUP,
2301                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2302                                           "extensive metadata register"
2303                                           " isn't supported");
2304         if (!(action->conf))
2305                 return rte_flow_error_set(error, EINVAL,
2306                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2307                                           "configuration cannot be null");
2308         conf = (const struct rte_flow_action_set_tag *)action->conf;
2309         if (!conf->mask)
2310                 return rte_flow_error_set(error, EINVAL,
2311                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2312                                           "zero mask doesn't have any effect");
2313         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
2314         if (ret < 0)
2315                 return ret;
2316         if (!attr->transfer && attr->ingress &&
2317             (action_flags & terminal_action_flags))
2318                 return rte_flow_error_set(error, EINVAL,
2319                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2320                                           "set_tag has no effect"
2321                                           " with terminal actions");
2322         return 0;
2323 }
2324
2325 /**
2326  * Validate count action.
2327  *
2328  * @param[in] dev
2329  *   Pointer to rte_eth_dev structure.
2330  * @param[out] error
2331  *   Pointer to error structure.
2332  *
2333  * @return
2334  *   0 on success, a negative errno value otherwise and rte_errno is set.
2335  */
2336 static int
2337 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2338                               struct rte_flow_error *error)
2339 {
2340         struct mlx5_priv *priv = dev->data->dev_private;
2341
2342         if (!priv->config.devx)
2343                 goto notsup_err;
2344 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2345         return 0;
2346 #endif
2347 notsup_err:
2348         return rte_flow_error_set
2349                       (error, ENOTSUP,
2350                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2351                        NULL,
2352                        "count action not supported");
2353 }
2354
2355 /**
2356  * Validate the L2 encap action.
2357  *
2358  * @param[in] dev
2359  *   Pointer to the rte_eth_dev structure.
2360  * @param[in] action_flags
2361  *   Holds the actions detected until now.
2362  * @param[in] action
2363  *   Pointer to the action structure.
2364  * @param[in] attr
2365  *   Pointer to flow attributes.
2366  * @param[out] error
2367  *   Pointer to error structure.
2368  *
2369  * @return
2370  *   0 on success, a negative errno value otherwise and rte_errno is set.
2371  */
2372 static int
2373 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
2374                                  uint64_t action_flags,
2375                                  const struct rte_flow_action *action,
2376                                  const struct rte_flow_attr *attr,
2377                                  struct rte_flow_error *error)
2378 {
2379         const struct mlx5_priv *priv = dev->data->dev_private;
2380
2381         if (!(action->conf))
2382                 return rte_flow_error_set(error, EINVAL,
2383                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2384                                           "configuration cannot be null");
2385         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
2386                 return rte_flow_error_set(error, EINVAL,
2387                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2388                                           "can only have a single encap action "
2389                                           "in a flow");
2390         if (!attr->transfer && priv->representor)
2391                 return rte_flow_error_set(error, ENOTSUP,
2392                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2393                                           "encap action for VF representor "
2394                                           "not supported on NIC table");
2395         return 0;
2396 }
2397
2398 /**
2399  * Validate a decap action.
2400  *
2401  * @param[in] dev
2402  *   Pointer to the rte_eth_dev structure.
2403  * @param[in] action_flags
2404  *   Holds the actions detected until now.
2405  * @param[in] attr
2406  *   Pointer to flow attributes
2407  * @param[out] error
2408  *   Pointer to error structure.
2409  *
2410  * @return
2411  *   0 on success, a negative errno value otherwise and rte_errno is set.
2412  */
2413 static int
2414 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
2415                               uint64_t action_flags,
2416                               const struct rte_flow_attr *attr,
2417                               struct rte_flow_error *error)
2418 {
2419         const struct mlx5_priv *priv = dev->data->dev_private;
2420
2421         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
2422                 return rte_flow_error_set(error, ENOTSUP,
2423                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2424                                           action_flags &
2425                                           MLX5_FLOW_ACTION_DECAP ? "can only "
2426                                           "have a single decap action" : "decap "
2427                                           "after encap is not supported");
2428         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2429                 return rte_flow_error_set(error, EINVAL,
2430                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2431                                           "can't have decap action after"
2432                                           " modify action");
2433         if (attr->egress)
2434                 return rte_flow_error_set(error, ENOTSUP,
2435                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2436                                           NULL,
2437                                           "decap action not supported for "
2438                                           "egress");
2439         if (!attr->transfer && priv->representor)
2440                 return rte_flow_error_set(error, ENOTSUP,
2441                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2442                                           "decap action for VF representor "
2443                                           "not supported on NIC table");
2444         return 0;
2445 }
2446
2447 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
2448
2449 /**
2450  * Validate the raw encap and decap actions.
2451  *
2452  * @param[in] dev
2453  *   Pointer to the rte_eth_dev structure.
2454  * @param[in] decap
2455  *   Pointer to the decap action.
2456  * @param[in] encap
2457  *   Pointer to the encap action.
2458  * @param[in] attr
2459  *   Pointer to flow attributes
2460  * @param[in/out] action_flags
2461  *   Holds the actions detected until now.
2462  * @param[out] actions_n
2463  *   pointer to the number of actions counter.
2464  * @param[out] error
2465  *   Pointer to error structure.
2466  *
2467  * @return
2468  *   0 on success, a negative errno value otherwise and rte_errno is set.
2469  */
2470 static int
2471 flow_dv_validate_action_raw_encap_decap
2472         (struct rte_eth_dev *dev,
2473          const struct rte_flow_action_raw_decap *decap,
2474          const struct rte_flow_action_raw_encap *encap,
2475          const struct rte_flow_attr *attr, uint64_t *action_flags,
2476          int *actions_n, struct rte_flow_error *error)
2477 {
2478         const struct mlx5_priv *priv = dev->data->dev_private;
2479         int ret;
2480
2481         if (encap && (!encap->size || !encap->data))
2482                 return rte_flow_error_set(error, EINVAL,
2483                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2484                                           "raw encap data cannot be empty");
2485         if (decap && encap) {
2486                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
2487                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2488                         /* L3 encap. */
2489                         decap = NULL;
2490                 else if (encap->size <=
2491                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2492                            decap->size >
2493                            MLX5_ENCAPSULATION_DECISION_SIZE)
2494                         /* L3 decap. */
2495                         encap = NULL;
2496                 else if (encap->size >
2497                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2498                            decap->size >
2499                            MLX5_ENCAPSULATION_DECISION_SIZE)
2500                         /* 2 L2 actions: encap and decap. */
2501                         ;
2502                 else
2503                         return rte_flow_error_set(error,
2504                                 ENOTSUP,
2505                                 RTE_FLOW_ERROR_TYPE_ACTION,
2506                                 NULL, "unsupported too small "
2507                                 "raw decap and too small raw "
2508                                 "encap combination");
2509         }
2510         if (decap) {
2511                 ret = flow_dv_validate_action_decap(dev, *action_flags, attr,
2512                                                     error);
2513                 if (ret < 0)
2514                         return ret;
2515                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
2516                 ++(*actions_n);
2517         }
2518         if (encap) {
2519                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
2520                         return rte_flow_error_set(error, ENOTSUP,
2521                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2522                                                   NULL,
2523                                                   "small raw encap size");
2524                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
2525                         return rte_flow_error_set(error, EINVAL,
2526                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2527                                                   NULL,
2528                                                   "more than one encap action");
2529                 if (!attr->transfer && priv->representor)
2530                         return rte_flow_error_set
2531                                         (error, ENOTSUP,
2532                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2533                                          "encap action for VF representor "
2534                                          "not supported on NIC table");
2535                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
2536                 ++(*actions_n);
2537         }
2538         return 0;
2539 }
2540
2541 /**
2542  * Find existing encap/decap resource or create and register a new one.
2543  *
2544  * @param[in, out] dev
2545  *   Pointer to rte_eth_dev structure.
2546  * @param[in, out] resource
2547  *   Pointer to encap/decap resource.
2548  * @parm[in, out] dev_flow
2549  *   Pointer to the dev_flow.
2550  * @param[out] error
2551  *   pointer to error structure.
2552  *
2553  * @return
2554  *   0 on success otherwise -errno and errno is set.
2555  */
2556 static int
2557 flow_dv_encap_decap_resource_register
2558                         (struct rte_eth_dev *dev,
2559                          struct mlx5_flow_dv_encap_decap_resource *resource,
2560                          struct mlx5_flow *dev_flow,
2561                          struct rte_flow_error *error)
2562 {
2563         struct mlx5_priv *priv = dev->data->dev_private;
2564         struct mlx5_dev_ctx_shared *sh = priv->sh;
2565         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2566         struct mlx5dv_dr_domain *domain;
2567         uint32_t idx = 0;
2568
2569         resource->flags = dev_flow->dv.group ? 0 : 1;
2570         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2571                 domain = sh->fdb_domain;
2572         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2573                 domain = sh->rx_domain;
2574         else
2575                 domain = sh->tx_domain;
2576         /* Lookup a matching resource from cache. */
2577         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], sh->encaps_decaps, idx,
2578                       cache_resource, next) {
2579                 if (resource->reformat_type == cache_resource->reformat_type &&
2580                     resource->ft_type == cache_resource->ft_type &&
2581                     resource->flags == cache_resource->flags &&
2582                     resource->size == cache_resource->size &&
2583                     !memcmp((const void *)resource->buf,
2584                             (const void *)cache_resource->buf,
2585                             resource->size)) {
2586                         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2587                                 (void *)cache_resource,
2588                                 rte_atomic32_read(&cache_resource->refcnt));
2589                         rte_atomic32_inc(&cache_resource->refcnt);
2590                         dev_flow->handle->dvh.rix_encap_decap = idx;
2591                         dev_flow->dv.encap_decap = cache_resource;
2592                         return 0;
2593                 }
2594         }
2595         /* Register new encap/decap resource. */
2596         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2597                                        &dev_flow->handle->dvh.rix_encap_decap);
2598         if (!cache_resource)
2599                 return rte_flow_error_set(error, ENOMEM,
2600                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2601                                           "cannot allocate resource memory");
2602         *cache_resource = *resource;
2603         cache_resource->verbs_action =
2604                 mlx5_glue->dv_create_flow_action_packet_reformat
2605                         (sh->ctx, cache_resource->reformat_type,
2606                          cache_resource->ft_type, domain, cache_resource->flags,
2607                          cache_resource->size,
2608                          (cache_resource->size ? cache_resource->buf : NULL));
2609         if (!cache_resource->verbs_action) {
2610                 rte_free(cache_resource);
2611                 return rte_flow_error_set(error, ENOMEM,
2612                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2613                                           NULL, "cannot create action");
2614         }
2615         rte_atomic32_init(&cache_resource->refcnt);
2616         rte_atomic32_inc(&cache_resource->refcnt);
2617         ILIST_INSERT(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &sh->encaps_decaps,
2618                      dev_flow->handle->dvh.rix_encap_decap, cache_resource,
2619                      next);
2620         dev_flow->dv.encap_decap = cache_resource;
2621         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2622                 (void *)cache_resource,
2623                 rte_atomic32_read(&cache_resource->refcnt));
2624         return 0;
2625 }
2626
2627 /**
2628  * Find existing table jump resource or create and register a new one.
2629  *
2630  * @param[in, out] dev
2631  *   Pointer to rte_eth_dev structure.
2632  * @param[in, out] tbl
2633  *   Pointer to flow table resource.
2634  * @parm[in, out] dev_flow
2635  *   Pointer to the dev_flow.
2636  * @param[out] error
2637  *   pointer to error structure.
2638  *
2639  * @return
2640  *   0 on success otherwise -errno and errno is set.
2641  */
2642 static int
2643 flow_dv_jump_tbl_resource_register
2644                         (struct rte_eth_dev *dev __rte_unused,
2645                          struct mlx5_flow_tbl_resource *tbl,
2646                          struct mlx5_flow *dev_flow,
2647                          struct rte_flow_error *error)
2648 {
2649         struct mlx5_flow_tbl_data_entry *tbl_data =
2650                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2651         int cnt;
2652
2653         MLX5_ASSERT(tbl);
2654         cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
2655         if (!cnt) {
2656                 tbl_data->jump.action =
2657                         mlx5_glue->dr_create_flow_action_dest_flow_tbl
2658                         (tbl->obj);
2659                 if (!tbl_data->jump.action)
2660                         return rte_flow_error_set(error, ENOMEM,
2661                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2662                                         NULL, "cannot create jump action");
2663                 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2664                         (void *)&tbl_data->jump, cnt);
2665         } else {
2666                 /* old jump should not make the table ref++. */
2667                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
2668                 MLX5_ASSERT(tbl_data->jump.action);
2669                 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2670                         (void *)&tbl_data->jump, cnt);
2671         }
2672         rte_atomic32_inc(&tbl_data->jump.refcnt);
2673         dev_flow->handle->rix_jump = tbl_data->idx;
2674         dev_flow->dv.jump = &tbl_data->jump;
2675         return 0;
2676 }
2677
2678 /**
2679  * Find existing table port ID resource or create and register a new one.
2680  *
2681  * @param[in, out] dev
2682  *   Pointer to rte_eth_dev structure.
2683  * @param[in, out] resource
2684  *   Pointer to port ID action resource.
2685  * @parm[in, out] dev_flow
2686  *   Pointer to the dev_flow.
2687  * @param[out] error
2688  *   pointer to error structure.
2689  *
2690  * @return
2691  *   0 on success otherwise -errno and errno is set.
2692  */
2693 static int
2694 flow_dv_port_id_action_resource_register
2695                         (struct rte_eth_dev *dev,
2696                          struct mlx5_flow_dv_port_id_action_resource *resource,
2697                          struct mlx5_flow *dev_flow,
2698                          struct rte_flow_error *error)
2699 {
2700         struct mlx5_priv *priv = dev->data->dev_private;
2701         struct mlx5_dev_ctx_shared *sh = priv->sh;
2702         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
2703         uint32_t idx = 0;
2704
2705         /* Lookup a matching resource from cache. */
2706         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list,
2707                       idx, cache_resource, next) {
2708                 if (resource->port_id == cache_resource->port_id) {
2709                         DRV_LOG(DEBUG, "port id action resource resource %p: "
2710                                 "refcnt %d++",
2711                                 (void *)cache_resource,
2712                                 rte_atomic32_read(&cache_resource->refcnt));
2713                         rte_atomic32_inc(&cache_resource->refcnt);
2714                         dev_flow->handle->rix_port_id_action = idx;
2715                         dev_flow->dv.port_id_action = cache_resource;
2716                         return 0;
2717                 }
2718         }
2719         /* Register new port id action resource. */
2720         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID],
2721                                        &dev_flow->handle->rix_port_id_action);
2722         if (!cache_resource)
2723                 return rte_flow_error_set(error, ENOMEM,
2724                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2725                                           "cannot allocate resource memory");
2726         *cache_resource = *resource;
2727         /*
2728          * Depending on rdma_core version the glue routine calls
2729          * either mlx5dv_dr_action_create_dest_ib_port(domain, dev_port)
2730          * or mlx5dv_dr_action_create_dest_vport(domain, vport_id).
2731          */
2732         cache_resource->action =
2733                 mlx5_glue->dr_create_flow_action_dest_port
2734                         (priv->sh->fdb_domain, resource->port_id);
2735         if (!cache_resource->action) {
2736                 rte_free(cache_resource);
2737                 return rte_flow_error_set(error, ENOMEM,
2738                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2739                                           NULL, "cannot create action");
2740         }
2741         rte_atomic32_init(&cache_resource->refcnt);
2742         rte_atomic32_inc(&cache_resource->refcnt);
2743         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
2744                      dev_flow->handle->rix_port_id_action, cache_resource,
2745                      next);
2746         dev_flow->dv.port_id_action = cache_resource;
2747         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
2748                 (void *)cache_resource,
2749                 rte_atomic32_read(&cache_resource->refcnt));
2750         return 0;
2751 }
2752
2753 /**
2754  * Find existing push vlan resource or create and register a new one.
2755  *
2756  * @param [in, out] dev
2757  *   Pointer to rte_eth_dev structure.
2758  * @param[in, out] resource
2759  *   Pointer to port ID action resource.
2760  * @parm[in, out] dev_flow
2761  *   Pointer to the dev_flow.
2762  * @param[out] error
2763  *   pointer to error structure.
2764  *
2765  * @return
2766  *   0 on success otherwise -errno and errno is set.
2767  */
2768 static int
2769 flow_dv_push_vlan_action_resource_register
2770                        (struct rte_eth_dev *dev,
2771                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
2772                         struct mlx5_flow *dev_flow,
2773                         struct rte_flow_error *error)
2774 {
2775         struct mlx5_priv *priv = dev->data->dev_private;
2776         struct mlx5_dev_ctx_shared *sh = priv->sh;
2777         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
2778         struct mlx5dv_dr_domain *domain;
2779         uint32_t idx = 0;
2780
2781         /* Lookup a matching resource from cache. */
2782         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2783                       sh->push_vlan_action_list, idx, cache_resource, next) {
2784                 if (resource->vlan_tag == cache_resource->vlan_tag &&
2785                     resource->ft_type == cache_resource->ft_type) {
2786                         DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
2787                                 "refcnt %d++",
2788                                 (void *)cache_resource,
2789                                 rte_atomic32_read(&cache_resource->refcnt));
2790                         rte_atomic32_inc(&cache_resource->refcnt);
2791                         dev_flow->handle->dvh.rix_push_vlan = idx;
2792                         dev_flow->dv.push_vlan_res = cache_resource;
2793                         return 0;
2794                 }
2795         }
2796         /* Register new push_vlan action resource. */
2797         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2798                                        &dev_flow->handle->dvh.rix_push_vlan);
2799         if (!cache_resource)
2800                 return rte_flow_error_set(error, ENOMEM,
2801                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2802                                           "cannot allocate resource memory");
2803         *cache_resource = *resource;
2804         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2805                 domain = sh->fdb_domain;
2806         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2807                 domain = sh->rx_domain;
2808         else
2809                 domain = sh->tx_domain;
2810         cache_resource->action =
2811                 mlx5_glue->dr_create_flow_action_push_vlan(domain,
2812                                                            resource->vlan_tag);
2813         if (!cache_resource->action) {
2814                 rte_free(cache_resource);
2815                 return rte_flow_error_set(error, ENOMEM,
2816                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2817                                           NULL, "cannot create action");
2818         }
2819         rte_atomic32_init(&cache_resource->refcnt);
2820         rte_atomic32_inc(&cache_resource->refcnt);
2821         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2822                      &sh->push_vlan_action_list,
2823                      dev_flow->handle->dvh.rix_push_vlan,
2824                      cache_resource, next);
2825         dev_flow->dv.push_vlan_res = cache_resource;
2826         DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
2827                 (void *)cache_resource,
2828                 rte_atomic32_read(&cache_resource->refcnt));
2829         return 0;
2830 }
2831 /**
2832  * Get the size of specific rte_flow_item_type
2833  *
2834  * @param[in] item_type
2835  *   Tested rte_flow_item_type.
2836  *
2837  * @return
2838  *   sizeof struct item_type, 0 if void or irrelevant.
2839  */
2840 static size_t
2841 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
2842 {
2843         size_t retval;
2844
2845         switch (item_type) {
2846         case RTE_FLOW_ITEM_TYPE_ETH:
2847                 retval = sizeof(struct rte_flow_item_eth);
2848                 break;
2849         case RTE_FLOW_ITEM_TYPE_VLAN:
2850                 retval = sizeof(struct rte_flow_item_vlan);
2851                 break;
2852         case RTE_FLOW_ITEM_TYPE_IPV4:
2853                 retval = sizeof(struct rte_flow_item_ipv4);
2854                 break;
2855         case RTE_FLOW_ITEM_TYPE_IPV6:
2856                 retval = sizeof(struct rte_flow_item_ipv6);
2857                 break;
2858         case RTE_FLOW_ITEM_TYPE_UDP:
2859                 retval = sizeof(struct rte_flow_item_udp);
2860                 break;
2861         case RTE_FLOW_ITEM_TYPE_TCP:
2862                 retval = sizeof(struct rte_flow_item_tcp);
2863                 break;
2864         case RTE_FLOW_ITEM_TYPE_VXLAN:
2865                 retval = sizeof(struct rte_flow_item_vxlan);
2866                 break;
2867         case RTE_FLOW_ITEM_TYPE_GRE:
2868                 retval = sizeof(struct rte_flow_item_gre);
2869                 break;
2870         case RTE_FLOW_ITEM_TYPE_NVGRE:
2871                 retval = sizeof(struct rte_flow_item_nvgre);
2872                 break;
2873         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
2874                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
2875                 break;
2876         case RTE_FLOW_ITEM_TYPE_MPLS:
2877                 retval = sizeof(struct rte_flow_item_mpls);
2878                 break;
2879         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
2880         default:
2881                 retval = 0;
2882                 break;
2883         }
2884         return retval;
2885 }
2886
2887 #define MLX5_ENCAP_IPV4_VERSION         0x40
2888 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
2889 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
2890 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
2891 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
2892 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
2893 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
2894
2895 /**
2896  * Convert the encap action data from list of rte_flow_item to raw buffer
2897  *
2898  * @param[in] items
2899  *   Pointer to rte_flow_item objects list.
2900  * @param[out] buf
2901  *   Pointer to the output buffer.
2902  * @param[out] size
2903  *   Pointer to the output buffer size.
2904  * @param[out] error
2905  *   Pointer to the error structure.
2906  *
2907  * @return
2908  *   0 on success, a negative errno value otherwise and rte_errno is set.
2909  */
2910 static int
2911 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
2912                            size_t *size, struct rte_flow_error *error)
2913 {
2914         struct rte_ether_hdr *eth = NULL;
2915         struct rte_vlan_hdr *vlan = NULL;
2916         struct rte_ipv4_hdr *ipv4 = NULL;
2917         struct rte_ipv6_hdr *ipv6 = NULL;
2918         struct rte_udp_hdr *udp = NULL;
2919         struct rte_vxlan_hdr *vxlan = NULL;
2920         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
2921         struct rte_gre_hdr *gre = NULL;
2922         size_t len;
2923         size_t temp_size = 0;
2924
2925         if (!items)
2926                 return rte_flow_error_set(error, EINVAL,
2927                                           RTE_FLOW_ERROR_TYPE_ACTION,
2928                                           NULL, "invalid empty data");
2929         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2930                 len = flow_dv_get_item_len(items->type);
2931                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
2932                         return rte_flow_error_set(error, EINVAL,
2933                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2934                                                   (void *)items->type,
2935                                                   "items total size is too big"
2936                                                   " for encap action");
2937                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
2938                 switch (items->type) {
2939                 case RTE_FLOW_ITEM_TYPE_ETH:
2940                         eth = (struct rte_ether_hdr *)&buf[temp_size];
2941                         break;
2942                 case RTE_FLOW_ITEM_TYPE_VLAN:
2943                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
2944                         if (!eth)
2945                                 return rte_flow_error_set(error, EINVAL,
2946                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2947                                                 (void *)items->type,
2948                                                 "eth header not found");
2949                         if (!eth->ether_type)
2950                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
2951                         break;
2952                 case RTE_FLOW_ITEM_TYPE_IPV4:
2953                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
2954                         if (!vlan && !eth)
2955                                 return rte_flow_error_set(error, EINVAL,
2956                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2957                                                 (void *)items->type,
2958                                                 "neither eth nor vlan"
2959                                                 " header found");
2960                         if (vlan && !vlan->eth_proto)
2961                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
2962                         else if (eth && !eth->ether_type)
2963                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
2964                         if (!ipv4->version_ihl)
2965                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
2966                                                     MLX5_ENCAP_IPV4_IHL_MIN;
2967                         if (!ipv4->time_to_live)
2968                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
2969                         break;
2970                 case RTE_FLOW_ITEM_TYPE_IPV6:
2971                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
2972                         if (!vlan && !eth)
2973                                 return rte_flow_error_set(error, EINVAL,
2974                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2975                                                 (void *)items->type,
2976                                                 "neither eth nor vlan"
2977                                                 " header found");
2978                         if (vlan && !vlan->eth_proto)
2979                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
2980                         else if (eth && !eth->ether_type)
2981                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
2982                         if (!ipv6->vtc_flow)
2983                                 ipv6->vtc_flow =
2984                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
2985                         if (!ipv6->hop_limits)
2986                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
2987                         break;
2988                 case RTE_FLOW_ITEM_TYPE_UDP:
2989                         udp = (struct rte_udp_hdr *)&buf[temp_size];
2990                         if (!ipv4 && !ipv6)
2991                                 return rte_flow_error_set(error, EINVAL,
2992                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2993                                                 (void *)items->type,
2994                                                 "ip header not found");
2995                         if (ipv4 && !ipv4->next_proto_id)
2996                                 ipv4->next_proto_id = IPPROTO_UDP;
2997                         else if (ipv6 && !ipv6->proto)
2998                                 ipv6->proto = IPPROTO_UDP;
2999                         break;
3000                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3001                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
3002                         if (!udp)
3003                                 return rte_flow_error_set(error, EINVAL,
3004                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3005                                                 (void *)items->type,
3006                                                 "udp header not found");
3007                         if (!udp->dst_port)
3008                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
3009                         if (!vxlan->vx_flags)
3010                                 vxlan->vx_flags =
3011                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
3012                         break;
3013                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3014                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
3015                         if (!udp)
3016                                 return rte_flow_error_set(error, EINVAL,
3017                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3018                                                 (void *)items->type,
3019                                                 "udp header not found");
3020                         if (!vxlan_gpe->proto)
3021                                 return rte_flow_error_set(error, EINVAL,
3022                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3023                                                 (void *)items->type,
3024                                                 "next protocol not found");
3025                         if (!udp->dst_port)
3026                                 udp->dst_port =
3027                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
3028                         if (!vxlan_gpe->vx_flags)
3029                                 vxlan_gpe->vx_flags =
3030                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
3031                         break;
3032                 case RTE_FLOW_ITEM_TYPE_GRE:
3033                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3034                         gre = (struct rte_gre_hdr *)&buf[temp_size];
3035                         if (!gre->proto)
3036                                 return rte_flow_error_set(error, EINVAL,
3037                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3038                                                 (void *)items->type,
3039                                                 "next protocol not found");
3040                         if (!ipv4 && !ipv6)
3041                                 return rte_flow_error_set(error, EINVAL,
3042                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3043                                                 (void *)items->type,
3044                                                 "ip header not found");
3045                         if (ipv4 && !ipv4->next_proto_id)
3046                                 ipv4->next_proto_id = IPPROTO_GRE;
3047                         else if (ipv6 && !ipv6->proto)
3048                                 ipv6->proto = IPPROTO_GRE;
3049                         break;
3050                 case RTE_FLOW_ITEM_TYPE_VOID:
3051                         break;
3052                 default:
3053                         return rte_flow_error_set(error, EINVAL,
3054                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3055                                                   (void *)items->type,
3056                                                   "unsupported item type");
3057                         break;
3058                 }
3059                 temp_size += len;
3060         }
3061         *size = temp_size;
3062         return 0;
3063 }
3064
3065 static int
3066 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
3067 {
3068         struct rte_ether_hdr *eth = NULL;
3069         struct rte_vlan_hdr *vlan = NULL;
3070         struct rte_ipv6_hdr *ipv6 = NULL;
3071         struct rte_udp_hdr *udp = NULL;
3072         char *next_hdr;
3073         uint16_t proto;
3074
3075         eth = (struct rte_ether_hdr *)data;
3076         next_hdr = (char *)(eth + 1);
3077         proto = RTE_BE16(eth->ether_type);
3078
3079         /* VLAN skipping */
3080         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
3081                 vlan = (struct rte_vlan_hdr *)next_hdr;
3082                 proto = RTE_BE16(vlan->eth_proto);
3083                 next_hdr += sizeof(struct rte_vlan_hdr);
3084         }
3085
3086         /* HW calculates IPv4 csum. no need to proceed */
3087         if (proto == RTE_ETHER_TYPE_IPV4)
3088                 return 0;
3089
3090         /* non IPv4/IPv6 header. not supported */
3091         if (proto != RTE_ETHER_TYPE_IPV6) {
3092                 return rte_flow_error_set(error, ENOTSUP,
3093                                           RTE_FLOW_ERROR_TYPE_ACTION,
3094                                           NULL, "Cannot offload non IPv4/IPv6");
3095         }
3096
3097         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
3098
3099         /* ignore non UDP */
3100         if (ipv6->proto != IPPROTO_UDP)
3101                 return 0;
3102
3103         udp = (struct rte_udp_hdr *)(ipv6 + 1);
3104         udp->dgram_cksum = 0;
3105
3106         return 0;
3107 }
3108
3109 /**
3110  * Convert L2 encap action to DV specification.
3111  *
3112  * @param[in] dev
3113  *   Pointer to rte_eth_dev structure.
3114  * @param[in] action
3115  *   Pointer to action structure.
3116  * @param[in, out] dev_flow
3117  *   Pointer to the mlx5_flow.
3118  * @param[in] transfer
3119  *   Mark if the flow is E-Switch flow.
3120  * @param[out] error
3121  *   Pointer to the error structure.
3122  *
3123  * @return
3124  *   0 on success, a negative errno value otherwise and rte_errno is set.
3125  */
3126 static int
3127 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
3128                                const struct rte_flow_action *action,
3129                                struct mlx5_flow *dev_flow,
3130                                uint8_t transfer,
3131                                struct rte_flow_error *error)
3132 {
3133         const struct rte_flow_item *encap_data;
3134         const struct rte_flow_action_raw_encap *raw_encap_data;
3135         struct mlx5_flow_dv_encap_decap_resource res = {
3136                 .reformat_type =
3137                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
3138                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3139                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
3140         };
3141
3142         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3143                 raw_encap_data =
3144                         (const struct rte_flow_action_raw_encap *)action->conf;
3145                 res.size = raw_encap_data->size;
3146                 memcpy(res.buf, raw_encap_data->data, res.size);
3147         } else {
3148                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
3149                         encap_data =
3150                                 ((const struct rte_flow_action_vxlan_encap *)
3151                                                 action->conf)->definition;
3152                 else
3153                         encap_data =
3154                                 ((const struct rte_flow_action_nvgre_encap *)
3155                                                 action->conf)->definition;
3156                 if (flow_dv_convert_encap_data(encap_data, res.buf,
3157                                                &res.size, error))
3158                         return -rte_errno;
3159         }
3160         if (flow_dv_zero_encap_udp_csum(res.buf, error))
3161                 return -rte_errno;
3162         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3163                 return rte_flow_error_set(error, EINVAL,
3164                                           RTE_FLOW_ERROR_TYPE_ACTION,
3165                                           NULL, "can't create L2 encap action");
3166         return 0;
3167 }
3168
3169 /**
3170  * Convert L2 decap action to DV specification.
3171  *
3172  * @param[in] dev
3173  *   Pointer to rte_eth_dev structure.
3174  * @param[in, out] dev_flow
3175  *   Pointer to the mlx5_flow.
3176  * @param[in] transfer
3177  *   Mark if the flow is E-Switch flow.
3178  * @param[out] error
3179  *   Pointer to the error structure.
3180  *
3181  * @return
3182  *   0 on success, a negative errno value otherwise and rte_errno is set.
3183  */
3184 static int
3185 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
3186                                struct mlx5_flow *dev_flow,
3187                                uint8_t transfer,
3188                                struct rte_flow_error *error)
3189 {
3190         struct mlx5_flow_dv_encap_decap_resource res = {
3191                 .size = 0,
3192                 .reformat_type =
3193                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
3194                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3195                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
3196         };
3197
3198         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3199                 return rte_flow_error_set(error, EINVAL,
3200                                           RTE_FLOW_ERROR_TYPE_ACTION,
3201                                           NULL, "can't create L2 decap action");
3202         return 0;
3203 }
3204
3205 /**
3206  * Convert raw decap/encap (L3 tunnel) action to DV specification.
3207  *
3208  * @param[in] dev
3209  *   Pointer to rte_eth_dev structure.
3210  * @param[in] action
3211  *   Pointer to action structure.
3212  * @param[in, out] dev_flow
3213  *   Pointer to the mlx5_flow.
3214  * @param[in] attr
3215  *   Pointer to the flow attributes.
3216  * @param[out] error
3217  *   Pointer to the error structure.
3218  *
3219  * @return
3220  *   0 on success, a negative errno value otherwise and rte_errno is set.
3221  */
3222 static int
3223 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
3224                                 const struct rte_flow_action *action,
3225                                 struct mlx5_flow *dev_flow,
3226                                 const struct rte_flow_attr *attr,
3227                                 struct rte_flow_error *error)
3228 {
3229         const struct rte_flow_action_raw_encap *encap_data;
3230         struct mlx5_flow_dv_encap_decap_resource res;
3231
3232         memset(&res, 0, sizeof(res));
3233         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
3234         res.size = encap_data->size;
3235         memcpy(res.buf, encap_data->data, res.size);
3236         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
3237                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
3238                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3239         if (attr->transfer)
3240                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3241         else
3242                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3243                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3244         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3245                 return rte_flow_error_set(error, EINVAL,
3246                                           RTE_FLOW_ERROR_TYPE_ACTION,
3247                                           NULL, "can't create encap action");
3248         return 0;
3249 }
3250
3251 /**
3252  * Create action push VLAN.
3253  *
3254  * @param[in] dev
3255  *   Pointer to rte_eth_dev structure.
3256  * @param[in] attr
3257  *   Pointer to the flow attributes.
3258  * @param[in] vlan
3259  *   Pointer to the vlan to push to the Ethernet header.
3260  * @param[in, out] dev_flow
3261  *   Pointer to the mlx5_flow.
3262  * @param[out] error
3263  *   Pointer to the error structure.
3264  *
3265  * @return
3266  *   0 on success, a negative errno value otherwise and rte_errno is set.
3267  */
3268 static int
3269 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
3270                                 const struct rte_flow_attr *attr,
3271                                 const struct rte_vlan_hdr *vlan,
3272                                 struct mlx5_flow *dev_flow,
3273                                 struct rte_flow_error *error)
3274 {
3275         struct mlx5_flow_dv_push_vlan_action_resource res;
3276
3277         memset(&res, 0, sizeof(res));
3278         res.vlan_tag =
3279                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
3280                                  vlan->vlan_tci);
3281         if (attr->transfer)
3282                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3283         else
3284                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3285                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3286         return flow_dv_push_vlan_action_resource_register
3287                                             (dev, &res, dev_flow, error);
3288 }
3289
3290 /**
3291  * Validate the modify-header actions.
3292  *
3293  * @param[in] action_flags
3294  *   Holds the actions detected until now.
3295  * @param[in] action
3296  *   Pointer to the modify action.
3297  * @param[out] error
3298  *   Pointer to error structure.
3299  *
3300  * @return
3301  *   0 on success, a negative errno value otherwise and rte_errno is set.
3302  */
3303 static int
3304 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
3305                                    const struct rte_flow_action *action,
3306                                    struct rte_flow_error *error)
3307 {
3308         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
3309                 return rte_flow_error_set(error, EINVAL,
3310                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3311                                           NULL, "action configuration not set");
3312         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3313                 return rte_flow_error_set(error, EINVAL,
3314                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3315                                           "can't have encap action before"
3316                                           " modify action");
3317         return 0;
3318 }
3319
3320 /**
3321  * Validate the modify-header MAC address actions.
3322  *
3323  * @param[in] action_flags
3324  *   Holds the actions detected until now.
3325  * @param[in] action
3326  *   Pointer to the modify action.
3327  * @param[in] item_flags
3328  *   Holds the items detected.
3329  * @param[out] error
3330  *   Pointer to error structure.
3331  *
3332  * @return
3333  *   0 on success, a negative errno value otherwise and rte_errno is set.
3334  */
3335 static int
3336 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
3337                                    const struct rte_flow_action *action,
3338                                    const uint64_t item_flags,
3339                                    struct rte_flow_error *error)
3340 {
3341         int ret = 0;
3342
3343         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3344         if (!ret) {
3345                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
3346                         return rte_flow_error_set(error, EINVAL,
3347                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3348                                                   NULL,
3349                                                   "no L2 item in pattern");
3350         }
3351         return ret;
3352 }
3353
3354 /**
3355  * Validate the modify-header IPv4 address actions.
3356  *
3357  * @param[in] action_flags
3358  *   Holds the actions detected until now.
3359  * @param[in] action
3360  *   Pointer to the modify action.
3361  * @param[in] item_flags
3362  *   Holds the items detected.
3363  * @param[out] error
3364  *   Pointer to error structure.
3365  *
3366  * @return
3367  *   0 on success, a negative errno value otherwise and rte_errno is set.
3368  */
3369 static int
3370 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3371                                     const struct rte_flow_action *action,
3372                                     const uint64_t item_flags,
3373                                     struct rte_flow_error *error)
3374 {
3375         int ret = 0;
3376         uint64_t layer;
3377
3378         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3379         if (!ret) {
3380                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3381                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3382                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3383                 if (!(item_flags & layer))
3384                         return rte_flow_error_set(error, EINVAL,
3385                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3386                                                   NULL,
3387                                                   "no ipv4 item in pattern");
3388         }
3389         return ret;
3390 }
3391
3392 /**
3393  * Validate the modify-header IPv6 address actions.
3394  *
3395  * @param[in] action_flags
3396  *   Holds the actions detected until now.
3397  * @param[in] action
3398  *   Pointer to the modify action.
3399  * @param[in] item_flags
3400  *   Holds the items detected.
3401  * @param[out] error
3402  *   Pointer to error structure.
3403  *
3404  * @return
3405  *   0 on success, a negative errno value otherwise and rte_errno is set.
3406  */
3407 static int
3408 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3409                                     const struct rte_flow_action *action,
3410                                     const uint64_t item_flags,
3411                                     struct rte_flow_error *error)
3412 {
3413         int ret = 0;
3414         uint64_t layer;
3415
3416         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3417         if (!ret) {
3418                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3419                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3420                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3421                 if (!(item_flags & layer))
3422                         return rte_flow_error_set(error, EINVAL,
3423                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3424                                                   NULL,
3425                                                   "no ipv6 item in pattern");
3426         }
3427         return ret;
3428 }
3429
3430 /**
3431  * Validate the modify-header TP actions.
3432  *
3433  * @param[in] action_flags
3434  *   Holds the actions detected until now.
3435  * @param[in] action
3436  *   Pointer to the modify action.
3437  * @param[in] item_flags
3438  *   Holds the items detected.
3439  * @param[out] error
3440  *   Pointer to error structure.
3441  *
3442  * @return
3443  *   0 on success, a negative errno value otherwise and rte_errno is set.
3444  */
3445 static int
3446 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3447                                   const struct rte_flow_action *action,
3448                                   const uint64_t item_flags,
3449                                   struct rte_flow_error *error)
3450 {
3451         int ret = 0;
3452         uint64_t layer;
3453
3454         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3455         if (!ret) {
3456                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3457                                  MLX5_FLOW_LAYER_INNER_L4 :
3458                                  MLX5_FLOW_LAYER_OUTER_L4;
3459                 if (!(item_flags & layer))
3460                         return rte_flow_error_set(error, EINVAL,
3461                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3462                                                   NULL, "no transport layer "
3463                                                   "in pattern");
3464         }
3465         return ret;
3466 }
3467
3468 /**
3469  * Validate the modify-header actions of increment/decrement
3470  * TCP Sequence-number.
3471  *
3472  * @param[in] action_flags
3473  *   Holds the actions detected until now.
3474  * @param[in] action
3475  *   Pointer to the modify action.
3476  * @param[in] item_flags
3477  *   Holds the items detected.
3478  * @param[out] error
3479  *   Pointer to error structure.
3480  *
3481  * @return
3482  *   0 on success, a negative errno value otherwise and rte_errno is set.
3483  */
3484 static int
3485 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3486                                        const struct rte_flow_action *action,
3487                                        const uint64_t item_flags,
3488                                        struct rte_flow_error *error)
3489 {
3490         int ret = 0;
3491         uint64_t layer;
3492
3493         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3494         if (!ret) {
3495                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3496                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3497                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3498                 if (!(item_flags & layer))
3499                         return rte_flow_error_set(error, EINVAL,
3500                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3501                                                   NULL, "no TCP item in"
3502                                                   " pattern");
3503                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3504                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3505                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3506                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3507                         return rte_flow_error_set(error, EINVAL,
3508                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3509                                                   NULL,
3510                                                   "cannot decrease and increase"
3511                                                   " TCP sequence number"
3512                                                   " at the same time");
3513         }
3514         return ret;
3515 }
3516
3517 /**
3518  * Validate the modify-header actions of increment/decrement
3519  * TCP Acknowledgment number.
3520  *
3521  * @param[in] action_flags
3522  *   Holds the actions detected until now.
3523  * @param[in] action
3524  *   Pointer to the modify action.
3525  * @param[in] item_flags
3526  *   Holds the items detected.
3527  * @param[out] error
3528  *   Pointer to error structure.
3529  *
3530  * @return
3531  *   0 on success, a negative errno value otherwise and rte_errno is set.
3532  */
3533 static int
3534 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3535                                        const struct rte_flow_action *action,
3536                                        const uint64_t item_flags,
3537                                        struct rte_flow_error *error)
3538 {
3539         int ret = 0;
3540         uint64_t layer;
3541
3542         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3543         if (!ret) {
3544                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3545                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3546                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3547                 if (!(item_flags & layer))
3548                         return rte_flow_error_set(error, EINVAL,
3549                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3550                                                   NULL, "no TCP item in"
3551                                                   " pattern");
3552                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3553                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3554                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3555                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3556                         return rte_flow_error_set(error, EINVAL,
3557                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3558                                                   NULL,
3559                                                   "cannot decrease and increase"
3560                                                   " TCP acknowledgment number"
3561                                                   " at the same time");
3562         }
3563         return ret;
3564 }
3565
3566 /**
3567  * Validate the modify-header TTL actions.
3568  *
3569  * @param[in] action_flags
3570  *   Holds the actions detected until now.
3571  * @param[in] action
3572  *   Pointer to the modify action.
3573  * @param[in] item_flags
3574  *   Holds the items detected.
3575  * @param[out] error
3576  *   Pointer to error structure.
3577  *
3578  * @return
3579  *   0 on success, a negative errno value otherwise and rte_errno is set.
3580  */
3581 static int
3582 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3583                                    const struct rte_flow_action *action,
3584                                    const uint64_t item_flags,
3585                                    struct rte_flow_error *error)
3586 {
3587         int ret = 0;
3588         uint64_t layer;
3589
3590         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3591         if (!ret) {
3592                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3593                                  MLX5_FLOW_LAYER_INNER_L3 :
3594                                  MLX5_FLOW_LAYER_OUTER_L3;
3595                 if (!(item_flags & layer))
3596                         return rte_flow_error_set(error, EINVAL,
3597                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3598                                                   NULL,
3599                                                   "no IP protocol in pattern");
3600         }
3601         return ret;
3602 }
3603
3604 /**
3605  * Validate jump action.
3606  *
3607  * @param[in] action
3608  *   Pointer to the jump action.
3609  * @param[in] action_flags
3610  *   Holds the actions detected until now.
3611  * @param[in] attributes
3612  *   Pointer to flow attributes
3613  * @param[in] external
3614  *   Action belongs to flow rule created by request external to PMD.
3615  * @param[out] error
3616  *   Pointer to error structure.
3617  *
3618  * @return
3619  *   0 on success, a negative errno value otherwise and rte_errno is set.
3620  */
3621 static int
3622 flow_dv_validate_action_jump(const struct rte_flow_action *action,
3623                              uint64_t action_flags,
3624                              const struct rte_flow_attr *attributes,
3625                              bool external, struct rte_flow_error *error)
3626 {
3627         uint32_t target_group, table;
3628         int ret = 0;
3629
3630         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3631                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3632                 return rte_flow_error_set(error, EINVAL,
3633                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3634                                           "can't have 2 fate actions in"
3635                                           " same flow");
3636         if (action_flags & MLX5_FLOW_ACTION_METER)
3637                 return rte_flow_error_set(error, ENOTSUP,
3638                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3639                                           "jump with meter not support");
3640         if (!action->conf)
3641                 return rte_flow_error_set(error, EINVAL,
3642                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3643                                           NULL, "action configuration not set");
3644         target_group =
3645                 ((const struct rte_flow_action_jump *)action->conf)->group;
3646         ret = mlx5_flow_group_to_table(attributes, external, target_group,
3647                                        true, &table, error);
3648         if (ret)
3649                 return ret;
3650         if (attributes->group == target_group)
3651                 return rte_flow_error_set(error, EINVAL,
3652                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3653                                           "target group must be other than"
3654                                           " the current flow group");
3655         return 0;
3656 }
3657
3658 /*
3659  * Validate the port_id action.
3660  *
3661  * @param[in] dev
3662  *   Pointer to rte_eth_dev structure.
3663  * @param[in] action_flags
3664  *   Bit-fields that holds the actions detected until now.
3665  * @param[in] action
3666  *   Port_id RTE action structure.
3667  * @param[in] attr
3668  *   Attributes of flow that includes this action.
3669  * @param[out] error
3670  *   Pointer to error structure.
3671  *
3672  * @return
3673  *   0 on success, a negative errno value otherwise and rte_errno is set.
3674  */
3675 static int
3676 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
3677                                 uint64_t action_flags,
3678                                 const struct rte_flow_action *action,
3679                                 const struct rte_flow_attr *attr,
3680                                 struct rte_flow_error *error)
3681 {
3682         const struct rte_flow_action_port_id *port_id;
3683         struct mlx5_priv *act_priv;
3684         struct mlx5_priv *dev_priv;
3685         uint16_t port;
3686
3687         if (!attr->transfer)
3688                 return rte_flow_error_set(error, ENOTSUP,
3689                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3690                                           NULL,
3691                                           "port id action is valid in transfer"
3692                                           " mode only");
3693         if (!action || !action->conf)
3694                 return rte_flow_error_set(error, ENOTSUP,
3695                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3696                                           NULL,
3697                                           "port id action parameters must be"
3698                                           " specified");
3699         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3700                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3701                 return rte_flow_error_set(error, EINVAL,
3702                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3703                                           "can have only one fate actions in"
3704                                           " a flow");
3705         dev_priv = mlx5_dev_to_eswitch_info(dev);
3706         if (!dev_priv)
3707                 return rte_flow_error_set(error, rte_errno,
3708                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3709                                           NULL,
3710                                           "failed to obtain E-Switch info");
3711         port_id = action->conf;
3712         port = port_id->original ? dev->data->port_id : port_id->id;
3713         act_priv = mlx5_port_to_eswitch_info(port, false);
3714         if (!act_priv)
3715                 return rte_flow_error_set
3716                                 (error, rte_errno,
3717                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
3718                                  "failed to obtain E-Switch port id for port");
3719         if (act_priv->domain_id != dev_priv->domain_id)
3720                 return rte_flow_error_set
3721                                 (error, EINVAL,
3722                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3723                                  "port does not belong to"
3724                                  " E-Switch being configured");
3725         return 0;
3726 }
3727
3728 /**
3729  * Get the maximum number of modify header actions.
3730  *
3731  * @param dev
3732  *   Pointer to rte_eth_dev structure.
3733  * @param flags
3734  *   Flags bits to check if root level.
3735  *
3736  * @return
3737  *   Max number of modify header actions device can support.
3738  */
3739 static inline unsigned int
3740 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
3741                               uint64_t flags)
3742 {
3743         /*
3744          * There's no way to directly query the max capacity from FW.
3745          * The maximal value on root table should be assumed to be supported.
3746          */
3747         if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
3748                 return MLX5_MAX_MODIFY_NUM;
3749         else
3750                 return MLX5_ROOT_TBL_MODIFY_NUM;
3751 }
3752
3753 /**
3754  * Validate the meter action.
3755  *
3756  * @param[in] dev
3757  *   Pointer to rte_eth_dev structure.
3758  * @param[in] action_flags
3759  *   Bit-fields that holds the actions detected until now.
3760  * @param[in] action
3761  *   Pointer to the meter action.
3762  * @param[in] attr
3763  *   Attributes of flow that includes this action.
3764  * @param[out] error
3765  *   Pointer to error structure.
3766  *
3767  * @return
3768  *   0 on success, a negative errno value otherwise and rte_ernno is set.
3769  */
3770 static int
3771 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
3772                                 uint64_t action_flags,
3773                                 const struct rte_flow_action *action,
3774                                 const struct rte_flow_attr *attr,
3775                                 struct rte_flow_error *error)
3776 {
3777         struct mlx5_priv *priv = dev->data->dev_private;
3778         const struct rte_flow_action_meter *am = action->conf;
3779         struct mlx5_flow_meter *fm;
3780
3781         if (!am)
3782                 return rte_flow_error_set(error, EINVAL,
3783                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3784                                           "meter action conf is NULL");
3785
3786         if (action_flags & MLX5_FLOW_ACTION_METER)
3787                 return rte_flow_error_set(error, ENOTSUP,
3788                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3789                                           "meter chaining not support");
3790         if (action_flags & MLX5_FLOW_ACTION_JUMP)
3791                 return rte_flow_error_set(error, ENOTSUP,
3792                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3793                                           "meter with jump not support");
3794         if (!priv->mtr_en)
3795                 return rte_flow_error_set(error, ENOTSUP,
3796                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3797                                           NULL,
3798                                           "meter action not supported");
3799         fm = mlx5_flow_meter_find(priv, am->mtr_id);
3800         if (!fm)
3801                 return rte_flow_error_set(error, EINVAL,
3802                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3803                                           "Meter not found");
3804         if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
3805               (!fm->ingress && !attr->ingress && attr->egress) ||
3806               (!fm->egress && !attr->egress && attr->ingress))))
3807                 return rte_flow_error_set(error, EINVAL,
3808                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3809                                           "Flow attributes are either invalid "
3810                                           "or have a conflict with current "
3811                                           "meter attributes");
3812         return 0;
3813 }
3814
3815 /**
3816  * Validate the age action.
3817  *
3818  * @param[in] action_flags
3819  *   Holds the actions detected until now.
3820  * @param[in] action
3821  *   Pointer to the age action.
3822  * @param[in] dev
3823  *   Pointer to the Ethernet device structure.
3824  * @param[out] error
3825  *   Pointer to error structure.
3826  *
3827  * @return
3828  *   0 on success, a negative errno value otherwise and rte_errno is set.
3829  */
3830 static int
3831 flow_dv_validate_action_age(uint64_t action_flags,
3832                             const struct rte_flow_action *action,
3833                             struct rte_eth_dev *dev,
3834                             struct rte_flow_error *error)
3835 {
3836         struct mlx5_priv *priv = dev->data->dev_private;
3837         const struct rte_flow_action_age *age = action->conf;
3838
3839         if (!priv->config.devx || priv->counter_fallback)
3840                 return rte_flow_error_set(error, ENOTSUP,
3841                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3842                                           NULL,
3843                                           "age action not supported");
3844         if (!(action->conf))
3845                 return rte_flow_error_set(error, EINVAL,
3846                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3847                                           "configuration cannot be null");
3848         if (age->timeout >= UINT16_MAX / 2 / 10)
3849                 return rte_flow_error_set(error, ENOTSUP,
3850                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3851                                           "Max age time: 3275 seconds");
3852         if (action_flags & MLX5_FLOW_ACTION_AGE)
3853                 return rte_flow_error_set(error, EINVAL,
3854                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3855                                           "Duplicate age ctions set");
3856         return 0;
3857 }
3858
3859 /**
3860  * Validate the modify-header IPv4 DSCP actions.
3861  *
3862  * @param[in] action_flags
3863  *   Holds the actions detected until now.
3864  * @param[in] action
3865  *   Pointer to the modify action.
3866  * @param[in] item_flags
3867  *   Holds the items detected.
3868  * @param[out] error
3869  *   Pointer to error structure.
3870  *
3871  * @return
3872  *   0 on success, a negative errno value otherwise and rte_errno is set.
3873  */
3874 static int
3875 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
3876                                          const struct rte_flow_action *action,
3877                                          const uint64_t item_flags,
3878                                          struct rte_flow_error *error)
3879 {
3880         int ret = 0;
3881
3882         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3883         if (!ret) {
3884                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
3885                         return rte_flow_error_set(error, EINVAL,
3886                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3887                                                   NULL,
3888                                                   "no ipv4 item in pattern");
3889         }
3890         return ret;
3891 }
3892
3893 /**
3894  * Validate the modify-header IPv6 DSCP actions.
3895  *
3896  * @param[in] action_flags
3897  *   Holds the actions detected until now.
3898  * @param[in] action
3899  *   Pointer to the modify action.
3900  * @param[in] item_flags
3901  *   Holds the items detected.
3902  * @param[out] error
3903  *   Pointer to error structure.
3904  *
3905  * @return
3906  *   0 on success, a negative errno value otherwise and rte_errno is set.
3907  */
3908 static int
3909 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
3910                                          const struct rte_flow_action *action,
3911                                          const uint64_t item_flags,
3912                                          struct rte_flow_error *error)
3913 {
3914         int ret = 0;
3915
3916         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3917         if (!ret) {
3918                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
3919                         return rte_flow_error_set(error, EINVAL,
3920                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3921                                                   NULL,
3922                                                   "no ipv6 item in pattern");
3923         }
3924         return ret;
3925 }
3926
3927 /**
3928  * Find existing modify-header resource or create and register a new one.
3929  *
3930  * @param dev[in, out]
3931  *   Pointer to rte_eth_dev structure.
3932  * @param[in, out] resource
3933  *   Pointer to modify-header resource.
3934  * @parm[in, out] dev_flow
3935  *   Pointer to the dev_flow.
3936  * @param[out] error
3937  *   pointer to error structure.
3938  *
3939  * @return
3940  *   0 on success otherwise -errno and errno is set.
3941  */
3942 static int
3943 flow_dv_modify_hdr_resource_register
3944                         (struct rte_eth_dev *dev,
3945                          struct mlx5_flow_dv_modify_hdr_resource *resource,
3946                          struct mlx5_flow *dev_flow,
3947                          struct rte_flow_error *error)
3948 {
3949         struct mlx5_priv *priv = dev->data->dev_private;
3950         struct mlx5_dev_ctx_shared *sh = priv->sh;
3951         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
3952         struct mlx5dv_dr_domain *ns;
3953         uint32_t actions_len;
3954
3955         resource->flags = dev_flow->dv.group ? 0 :
3956                           MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
3957         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
3958                                     resource->flags))
3959                 return rte_flow_error_set(error, EOVERFLOW,
3960                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3961                                           "too many modify header items");
3962         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
3963                 ns = sh->fdb_domain;
3964         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
3965                 ns = sh->tx_domain;
3966         else
3967                 ns = sh->rx_domain;
3968         /* Lookup a matching resource from cache. */
3969         actions_len = resource->actions_num * sizeof(resource->actions[0]);
3970         LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
3971                 if (resource->ft_type == cache_resource->ft_type &&
3972                     resource->actions_num == cache_resource->actions_num &&
3973                     resource->flags == cache_resource->flags &&
3974                     !memcmp((const void *)resource->actions,
3975                             (const void *)cache_resource->actions,
3976                             actions_len)) {
3977                         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
3978                                 (void *)cache_resource,
3979                                 rte_atomic32_read(&cache_resource->refcnt));
3980                         rte_atomic32_inc(&cache_resource->refcnt);
3981                         dev_flow->handle->dvh.modify_hdr = cache_resource;
3982                         return 0;
3983                 }
3984         }
3985         /* Register new modify-header resource. */
3986         cache_resource = rte_calloc(__func__, 1,
3987                                     sizeof(*cache_resource) + actions_len, 0);
3988         if (!cache_resource)
3989                 return rte_flow_error_set(error, ENOMEM,
3990                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
3991                                           "cannot allocate resource memory");
3992         *cache_resource = *resource;
3993         rte_memcpy(cache_resource->actions, resource->actions, actions_len);
3994         cache_resource->verbs_action =
3995                 mlx5_glue->dv_create_flow_action_modify_header
3996                                         (sh->ctx, cache_resource->ft_type, ns,
3997                                          cache_resource->flags, actions_len,
3998                                          (uint64_t *)cache_resource->actions);
3999         if (!cache_resource->verbs_action) {
4000                 rte_free(cache_resource);
4001                 return rte_flow_error_set(error, ENOMEM,
4002                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4003                                           NULL, "cannot create action");
4004         }
4005         rte_atomic32_init(&cache_resource->refcnt);
4006         rte_atomic32_inc(&cache_resource->refcnt);
4007         LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
4008         dev_flow->handle->dvh.modify_hdr = cache_resource;
4009         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
4010                 (void *)cache_resource,
4011                 rte_atomic32_read(&cache_resource->refcnt));
4012         return 0;
4013 }
4014
4015 /**
4016  * Get DV flow counter by index.
4017  *
4018  * @param[in] dev
4019  *   Pointer to the Ethernet device structure.
4020  * @param[in] idx
4021  *   mlx5 flow counter index in the container.
4022  * @param[out] ppool
4023  *   mlx5 flow counter pool in the container,
4024  *
4025  * @return
4026  *   Pointer to the counter, NULL otherwise.
4027  */
4028 static struct mlx5_flow_counter *
4029 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
4030                            uint32_t idx,
4031                            struct mlx5_flow_counter_pool **ppool)
4032 {
4033         struct mlx5_priv *priv = dev->data->dev_private;
4034         struct mlx5_pools_container *cont;
4035         struct mlx5_flow_counter_pool *pool;
4036         uint32_t batch = 0, age = 0;
4037
4038         idx--;
4039         age = MLX_CNT_IS_AGE(idx);
4040         idx = age ? idx - MLX5_CNT_AGE_OFFSET : idx;
4041         if (idx >= MLX5_CNT_BATCH_OFFSET) {
4042                 idx -= MLX5_CNT_BATCH_OFFSET;
4043                 batch = 1;
4044         }
4045         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4046         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n);
4047         pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
4048         MLX5_ASSERT(pool);
4049         if (ppool)
4050                 *ppool = pool;
4051         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
4052 }
4053
4054 /**
4055  * Check the devx counter belongs to the pool.
4056  *
4057  * @param[in] pool
4058  *   Pointer to the counter pool.
4059  * @param[in] id
4060  *   The counter devx ID.
4061  *
4062  * @return
4063  *   True if counter belongs to the pool, false otherwise.
4064  */
4065 static bool
4066 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
4067 {
4068         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
4069                    MLX5_COUNTERS_PER_POOL;
4070
4071         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
4072                 return true;
4073         return false;
4074 }
4075
4076 /**
4077  * Get a pool by devx counter ID.
4078  *
4079  * @param[in] cont
4080  *   Pointer to the counter container.
4081  * @param[in] id
4082  *   The counter devx ID.
4083  *
4084  * @return
4085  *   The counter pool pointer if exists, NULL otherwise,
4086  */
4087 static struct mlx5_flow_counter_pool *
4088 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
4089 {
4090         uint32_t i;
4091
4092         /* Check last used pool. */
4093         if (cont->last_pool_idx != POOL_IDX_INVALID &&
4094             flow_dv_is_counter_in_pool(cont->pools[cont->last_pool_idx], id))
4095                 return cont->pools[cont->last_pool_idx];
4096         /* ID out of range means no suitable pool in the container. */
4097         if (id > cont->max_id || id < cont->min_id)
4098                 return NULL;
4099         /*
4100          * Find the pool from the end of the container, since mostly counter
4101          * ID is sequence increasing, and the last pool should be the needed
4102          * one.
4103          */
4104         i = rte_atomic16_read(&cont->n_valid);
4105         while (i--) {
4106                 struct mlx5_flow_counter_pool *pool = cont->pools[i];
4107
4108                 if (flow_dv_is_counter_in_pool(pool, id))
4109                         return pool;
4110         }
4111         return NULL;
4112 }
4113
4114 /**
4115  * Allocate a new memory for the counter values wrapped by all the needed
4116  * management.
4117  *
4118  * @param[in] dev
4119  *   Pointer to the Ethernet device structure.
4120  * @param[in] raws_n
4121  *   The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
4122  *
4123  * @return
4124  *   The new memory management pointer on success, otherwise NULL and rte_errno
4125  *   is set.
4126  */
4127 static struct mlx5_counter_stats_mem_mng *
4128 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
4129 {
4130         struct mlx5_priv *priv = dev->data->dev_private;
4131         struct mlx5_dev_ctx_shared *sh = priv->sh;
4132         struct mlx5_devx_mkey_attr mkey_attr;
4133         struct mlx5_counter_stats_mem_mng *mem_mng;
4134         volatile struct flow_counter_stats *raw_data;
4135         int size = (sizeof(struct flow_counter_stats) *
4136                         MLX5_COUNTERS_PER_POOL +
4137                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
4138                         sizeof(struct mlx5_counter_stats_mem_mng);
4139         uint8_t *mem = rte_calloc(__func__, 1, size, sysconf(_SC_PAGESIZE));
4140         int i;
4141
4142         if (!mem) {
4143                 rte_errno = ENOMEM;
4144                 return NULL;
4145         }
4146         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
4147         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
4148         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
4149                                                  IBV_ACCESS_LOCAL_WRITE);
4150         if (!mem_mng->umem) {
4151                 rte_errno = errno;
4152                 rte_free(mem);
4153                 return NULL;
4154         }
4155         mkey_attr.addr = (uintptr_t)mem;
4156         mkey_attr.size = size;
4157         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
4158         mkey_attr.pd = sh->pdn;
4159         mkey_attr.log_entity_size = 0;
4160         mkey_attr.pg_access = 0;
4161         mkey_attr.klm_array = NULL;
4162         mkey_attr.klm_num = 0;
4163         if (priv->config.hca_attr.relaxed_ordering_write &&
4164                 priv->config.hca_attr.relaxed_ordering_read  &&
4165                 !haswell_broadwell_cpu)
4166                 mkey_attr.relaxed_ordering = 1;
4167         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
4168         if (!mem_mng->dm) {
4169                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
4170                 rte_errno = errno;
4171                 rte_free(mem);
4172                 return NULL;
4173         }
4174         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
4175         raw_data = (volatile struct flow_counter_stats *)mem;
4176         for (i = 0; i < raws_n; ++i) {
4177                 mem_mng->raws[i].mem_mng = mem_mng;
4178                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
4179         }
4180         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
4181         return mem_mng;
4182 }
4183
4184 /**
4185  * Resize a counter container.
4186  *
4187  * @param[in] dev
4188  *   Pointer to the Ethernet device structure.
4189  * @param[in] batch
4190  *   Whether the pool is for counter that was allocated by batch command.
4191  * @param[in] age
4192  *   Whether the pool is for Aging counter.
4193  *
4194  * @return
4195  *   0 on success, otherwise negative errno value and rte_errno is set.
4196  */
4197 static int
4198 flow_dv_container_resize(struct rte_eth_dev *dev,
4199                                 uint32_t batch, uint32_t age)
4200 {
4201         struct mlx5_priv *priv = dev->data->dev_private;
4202         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4203                                                                age);
4204         struct mlx5_counter_stats_mem_mng *mem_mng = NULL;
4205         void *old_pools = cont->pools;
4206         uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
4207         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
4208         void *pools = rte_calloc(__func__, 1, mem_size, 0);
4209
4210         if (!pools) {
4211                 rte_errno = ENOMEM;
4212                 return -ENOMEM;
4213         }
4214         if (old_pools)
4215                 memcpy(pools, old_pools, cont->n *
4216                                        sizeof(struct mlx5_flow_counter_pool *));
4217         /*
4218          * Fallback mode query the counter directly, no background query
4219          * resources are needed.
4220          */
4221         if (!priv->counter_fallback) {
4222                 int i;
4223
4224                 mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
4225                           MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
4226                 if (!mem_mng) {
4227                         rte_free(pools);
4228                         return -ENOMEM;
4229                 }
4230                 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
4231                         LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
4232                                          mem_mng->raws +
4233                                          MLX5_CNT_CONTAINER_RESIZE +
4234                                          i, next);
4235         }
4236         rte_spinlock_lock(&cont->resize_sl);
4237         cont->n = resize;
4238         cont->mem_mng = mem_mng;
4239         cont->pools = pools;
4240         rte_spinlock_unlock(&cont->resize_sl);
4241         if (old_pools)
4242                 rte_free(old_pools);
4243         return 0;
4244 }
4245
4246 /**
4247  * Query a devx flow counter.
4248  *
4249  * @param[in] dev
4250  *   Pointer to the Ethernet device structure.
4251  * @param[in] cnt
4252  *   Index to the flow counter.
4253  * @param[out] pkts
4254  *   The statistics value of packets.
4255  * @param[out] bytes
4256  *   The statistics value of bytes.
4257  *
4258  * @return
4259  *   0 on success, otherwise a negative errno value and rte_errno is set.
4260  */
4261 static inline int
4262 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
4263                      uint64_t *bytes)
4264 {
4265         struct mlx5_priv *priv = dev->data->dev_private;
4266         struct mlx5_flow_counter_pool *pool = NULL;
4267         struct mlx5_flow_counter *cnt;
4268         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4269         int offset;
4270
4271         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4272         MLX5_ASSERT(pool);
4273         if (counter < MLX5_CNT_BATCH_OFFSET) {
4274                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4275                 if (priv->counter_fallback)
4276                         return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
4277                                         0, pkts, bytes, 0, NULL, NULL, 0);
4278         }
4279
4280         rte_spinlock_lock(&pool->sl);
4281         /*
4282          * The single counters allocation may allocate smaller ID than the
4283          * current allocated in parallel to the host reading.
4284          * In this case the new counter values must be reported as 0.
4285          */
4286         if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) {
4287                 *pkts = 0;
4288                 *bytes = 0;
4289         } else {
4290                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
4291                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
4292                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
4293         }
4294         rte_spinlock_unlock(&pool->sl);
4295         return 0;
4296 }
4297
4298 /**
4299  * Create and initialize a new counter pool.
4300  *
4301  * @param[in] dev
4302  *   Pointer to the Ethernet device structure.
4303  * @param[out] dcs
4304  *   The devX counter handle.
4305  * @param[in] batch
4306  *   Whether the pool is for counter that was allocated by batch command.
4307  * @param[in] age
4308  *   Whether the pool is for counter that was allocated for aging.
4309  * @param[in/out] cont_cur
4310  *   Pointer to the container pointer, it will be update in pool resize.
4311  *
4312  * @return
4313  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
4314  */
4315 static struct mlx5_flow_counter_pool *
4316 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
4317                     uint32_t batch, uint32_t age)
4318 {
4319         struct mlx5_priv *priv = dev->data->dev_private;
4320         struct mlx5_flow_counter_pool *pool;
4321         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4322                                                                age);
4323         int16_t n_valid = rte_atomic16_read(&cont->n_valid);
4324         uint32_t size = sizeof(*pool);
4325
4326         if (cont->n == n_valid && flow_dv_container_resize(dev, batch, age))
4327                 return NULL;
4328         size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
4329         size += (batch ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
4330         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE);
4331         pool = rte_calloc(__func__, 1, size, 0);
4332         if (!pool) {
4333                 rte_errno = ENOMEM;
4334                 return NULL;
4335         }
4336         pool->min_dcs = dcs;
4337         if (!priv->counter_fallback)
4338                 pool->raw = cont->mem_mng->raws + n_valid %
4339                                                       MLX5_CNT_CONTAINER_RESIZE;
4340         pool->raw_hw = NULL;
4341         pool->type = 0;
4342         pool->type |= (batch ? 0 :  CNT_POOL_TYPE_EXT);
4343         pool->type |= (!age ? 0 :  CNT_POOL_TYPE_AGE);
4344         pool->query_gen = 0;
4345         rte_spinlock_init(&pool->sl);
4346         TAILQ_INIT(&pool->counters[0]);
4347         TAILQ_INIT(&pool->counters[1]);
4348         TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
4349         pool->index = n_valid;
4350         cont->pools[n_valid] = pool;
4351         if (!batch) {
4352                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
4353
4354                 if (base < cont->min_id)
4355                         cont->min_id = base;
4356                 if (base > cont->max_id)
4357                         cont->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
4358                 cont->last_pool_idx = pool->index;
4359         }
4360         /* Pool initialization must be updated before host thread access. */
4361         rte_cio_wmb();
4362         rte_atomic16_add(&cont->n_valid, 1);
4363         return pool;
4364 }
4365
4366 /**
4367  * Update the minimum dcs-id for aged or no-aged counter pool.
4368  *
4369  * @param[in] dev
4370  *   Pointer to the Ethernet device structure.
4371  * @param[in] pool
4372  *   Current counter pool.
4373  * @param[in] batch
4374  *   Whether the pool is for counter that was allocated by batch command.
4375  * @param[in] age
4376  *   Whether the counter is for aging.
4377  */
4378 static void
4379 flow_dv_counter_update_min_dcs(struct rte_eth_dev *dev,
4380                         struct mlx5_flow_counter_pool *pool,
4381                         uint32_t batch, uint32_t age)
4382 {
4383         struct mlx5_priv *priv = dev->data->dev_private;
4384         struct mlx5_flow_counter_pool *other;
4385         struct mlx5_pools_container *cont;
4386
4387         cont = MLX5_CNT_CONTAINER(priv->sh, batch, (age ^ 0x1));
4388         other = flow_dv_find_pool_by_id(cont, pool->min_dcs->id);
4389         if (!other)
4390                 return;
4391         if (pool->min_dcs->id < other->min_dcs->id) {
4392                 rte_atomic64_set(&other->a64_dcs,
4393                         rte_atomic64_read(&pool->a64_dcs));
4394         } else {
4395                 rte_atomic64_set(&pool->a64_dcs,
4396                         rte_atomic64_read(&other->a64_dcs));
4397         }
4398 }
4399 /**
4400  * Prepare a new counter and/or a new counter pool.
4401  *
4402  * @param[in] dev
4403  *   Pointer to the Ethernet device structure.
4404  * @param[out] cnt_free
4405  *   Where to put the pointer of a new counter.
4406  * @param[in] batch
4407  *   Whether the pool is for counter that was allocated by batch command.
4408  * @param[in] age
4409  *   Whether the pool is for counter that was allocated for aging.
4410  *
4411  * @return
4412  *   The counter pool pointer and @p cnt_free is set on success,
4413  *   NULL otherwise and rte_errno is set.
4414  */
4415 static struct mlx5_flow_counter_pool *
4416 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
4417                              struct mlx5_flow_counter **cnt_free,
4418                              uint32_t batch, uint32_t age)
4419 {
4420         struct mlx5_priv *priv = dev->data->dev_private;
4421         struct mlx5_pools_container *cont;
4422         struct mlx5_flow_counter_pool *pool;
4423         struct mlx5_counters tmp_tq;
4424         struct mlx5_devx_obj *dcs = NULL;
4425         struct mlx5_flow_counter *cnt;
4426         uint32_t i;
4427
4428         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4429         if (!batch) {
4430                 /* bulk_bitmap must be 0 for single counter allocation. */
4431                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
4432                 if (!dcs)
4433                         return NULL;
4434                 pool = flow_dv_find_pool_by_id(cont, dcs->id);
4435                 if (!pool) {
4436                         pool = flow_dv_pool_create(dev, dcs, batch, age);
4437                         if (!pool) {
4438                                 mlx5_devx_cmd_destroy(dcs);
4439                                 return NULL;
4440                         }
4441                 } else if (dcs->id < pool->min_dcs->id) {
4442                         rte_atomic64_set(&pool->a64_dcs,
4443                                          (int64_t)(uintptr_t)dcs);
4444                 }
4445                 flow_dv_counter_update_min_dcs(dev,
4446                                                 pool, batch, age);
4447                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
4448                 cnt = MLX5_POOL_GET_CNT(pool, i);
4449                 cnt->pool = pool;
4450                 MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs;
4451                 *cnt_free = cnt;
4452                 return pool;
4453         }
4454         /* bulk_bitmap is in 128 counters units. */
4455         if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
4456                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
4457         if (!dcs) {
4458                 rte_errno = ENODATA;
4459                 return NULL;
4460         }
4461         pool = flow_dv_pool_create(dev, dcs, batch, age);
4462         if (!pool) {
4463                 mlx5_devx_cmd_destroy(dcs);
4464                 return NULL;
4465         }
4466         TAILQ_INIT(&tmp_tq);
4467         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
4468                 cnt = MLX5_POOL_GET_CNT(pool, i);
4469                 cnt->pool = pool;
4470                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
4471         }
4472         rte_spinlock_lock(&cont->csl);
4473         TAILQ_CONCAT(&cont->counters, &tmp_tq, next);
4474         rte_spinlock_unlock(&cont->csl);
4475         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
4476         (*cnt_free)->pool = pool;
4477         return pool;
4478 }
4479
4480 /**
4481  * Search for existed shared counter.
4482  *
4483  * @param[in] dev
4484  *   Pointer to the Ethernet device structure.
4485  * @param[in] id
4486  *   The shared counter ID to search.
4487  * @param[out] ppool
4488  *   mlx5 flow counter pool in the container,
4489  *
4490  * @return
4491  *   NULL if not existed, otherwise pointer to the shared extend counter.
4492  */
4493 static struct mlx5_flow_counter_ext *
4494 flow_dv_counter_shared_search(struct rte_eth_dev *dev, uint32_t id,
4495                               struct mlx5_flow_counter_pool **ppool)
4496 {
4497         struct mlx5_priv *priv = dev->data->dev_private;
4498         union mlx5_l3t_data data;
4499         uint32_t cnt_idx;
4500
4501         if (mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) || !data.dword)
4502                 return NULL;
4503         cnt_idx = data.dword;
4504         /*
4505          * Shared counters don't have age info. The counter extend is after
4506          * the counter datat structure.
4507          */
4508         return (struct mlx5_flow_counter_ext *)
4509                ((flow_dv_counter_get_by_idx(dev, cnt_idx, ppool)) + 1);
4510 }
4511
4512 /**
4513  * Allocate a flow counter.
4514  *
4515  * @param[in] dev
4516  *   Pointer to the Ethernet device structure.
4517  * @param[in] shared
4518  *   Indicate if this counter is shared with other flows.
4519  * @param[in] id
4520  *   Counter identifier.
4521  * @param[in] group
4522  *   Counter flow group.
4523  * @param[in] age
4524  *   Whether the counter was allocated for aging.
4525  *
4526  * @return
4527  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
4528  */
4529 static uint32_t
4530 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
4531                       uint16_t group, uint32_t age)
4532 {
4533         struct mlx5_priv *priv = dev->data->dev_private;
4534         struct mlx5_flow_counter_pool *pool = NULL;
4535         struct mlx5_flow_counter *cnt_free = NULL;
4536         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4537         /*
4538          * Currently group 0 flow counter cannot be assigned to a flow if it is
4539          * not the first one in the batch counter allocation, so it is better
4540          * to allocate counters one by one for these flows in a separate
4541          * container.
4542          * A counter can be shared between different groups so need to take
4543          * shared counters from the single container.
4544          */
4545         uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0;
4546         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4547                                                                age);
4548         uint32_t cnt_idx;
4549
4550         if (!priv->config.devx) {
4551                 rte_errno = ENOTSUP;
4552                 return 0;
4553         }
4554         if (shared) {
4555                 cnt_ext = flow_dv_counter_shared_search(dev, id, &pool);
4556                 if (cnt_ext) {
4557                         if (cnt_ext->ref_cnt + 1 == 0) {
4558                                 rte_errno = E2BIG;
4559                                 return 0;
4560                         }
4561                         cnt_ext->ref_cnt++;
4562                         cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL +
4563                                   (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL)
4564                                   + 1;
4565                         return cnt_idx;
4566                 }
4567         }
4568         /* Get free counters from container. */
4569         rte_spinlock_lock(&cont->csl);
4570         cnt_free = TAILQ_FIRST(&cont->counters);
4571         if (cnt_free)
4572                 TAILQ_REMOVE(&cont->counters, cnt_free, next);
4573         rte_spinlock_unlock(&cont->csl);
4574         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free,
4575                                                        batch, age))
4576                 goto err;
4577         pool = cnt_free->pool;
4578         if (!batch)
4579                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free);
4580         /* Create a DV counter action only in the first time usage. */
4581         if (!cnt_free->action) {
4582                 uint16_t offset;
4583                 struct mlx5_devx_obj *dcs;
4584
4585                 if (batch) {
4586                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
4587                         dcs = pool->min_dcs;
4588                 } else {
4589                         offset = 0;
4590                         dcs = cnt_ext->dcs;
4591                 }
4592                 cnt_free->action = mlx5_glue->dv_create_flow_action_counter
4593                                         (dcs->obj, offset);
4594                 if (!cnt_free->action) {
4595                         rte_errno = errno;
4596                         goto err;
4597                 }
4598         }
4599         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
4600                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
4601         cnt_idx += batch * MLX5_CNT_BATCH_OFFSET;
4602         cnt_idx += age * MLX5_CNT_AGE_OFFSET;
4603         /* Update the counter reset values. */
4604         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
4605                                  &cnt_free->bytes))
4606                 goto err;
4607         if (cnt_ext) {
4608                 cnt_ext->shared = shared;
4609                 cnt_ext->ref_cnt = 1;
4610                 cnt_ext->id = id;
4611                 if (shared) {
4612                         union mlx5_l3t_data data;
4613
4614                         data.dword = cnt_idx;
4615                         if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
4616                                 return 0;
4617                 }
4618         }
4619         if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on)
4620                 /* Start the asynchronous batch query by the host thread. */
4621                 mlx5_set_query_alarm(priv->sh);
4622         return cnt_idx;
4623 err:
4624         if (cnt_free) {
4625                 cnt_free->pool = pool;
4626                 rte_spinlock_lock(&cont->csl);
4627                 TAILQ_INSERT_TAIL(&cont->counters, cnt_free, next);
4628                 rte_spinlock_unlock(&cont->csl);
4629         }
4630         return 0;
4631 }
4632
4633 /**
4634  * Get age param from counter index.
4635  *
4636  * @param[in] dev
4637  *   Pointer to the Ethernet device structure.
4638  * @param[in] counter
4639  *   Index to the counter handler.
4640  *
4641  * @return
4642  *   The aging parameter specified for the counter index.
4643  */
4644 static struct mlx5_age_param*
4645 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
4646                                 uint32_t counter)
4647 {
4648         struct mlx5_flow_counter *cnt;
4649         struct mlx5_flow_counter_pool *pool = NULL;
4650
4651         flow_dv_counter_get_by_idx(dev, counter, &pool);
4652         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
4653         cnt = MLX5_POOL_GET_CNT(pool, counter);
4654         return MLX5_CNT_TO_AGE(cnt);
4655 }
4656
4657 /**
4658  * Remove a flow counter from aged counter list.
4659  *
4660  * @param[in] dev
4661  *   Pointer to the Ethernet device structure.
4662  * @param[in] counter
4663  *   Index to the counter handler.
4664  * @param[in] cnt
4665  *   Pointer to the counter handler.
4666  */
4667 static void
4668 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
4669                                 uint32_t counter, struct mlx5_flow_counter *cnt)
4670 {
4671         struct mlx5_age_info *age_info;
4672         struct mlx5_age_param *age_param;
4673         struct mlx5_priv *priv = dev->data->dev_private;
4674
4675         age_info = GET_PORT_AGE_INFO(priv);
4676         age_param = flow_dv_counter_idx_get_age(dev, counter);
4677         if (rte_atomic16_cmpset((volatile uint16_t *)
4678                         &age_param->state,
4679                         AGE_CANDIDATE, AGE_FREE)
4680                         != AGE_CANDIDATE) {
4681                 /**
4682                  * We need the lock even it is age timeout,
4683                  * since counter may still in process.
4684                  */
4685                 rte_spinlock_lock(&age_info->aged_sl);
4686                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
4687                 rte_spinlock_unlock(&age_info->aged_sl);
4688         }
4689         rte_atomic16_set(&age_param->state, AGE_FREE);
4690 }
4691 /**
4692  * Release a flow counter.
4693  *
4694  * @param[in] dev
4695  *   Pointer to the Ethernet device structure.
4696  * @param[in] counter
4697  *   Index to the counter handler.
4698  */
4699 static void
4700 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
4701 {
4702         struct mlx5_priv *priv = dev->data->dev_private;
4703         struct mlx5_flow_counter_pool *pool = NULL;
4704         struct mlx5_flow_counter *cnt;
4705         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4706
4707         if (!counter)
4708                 return;
4709         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4710         MLX5_ASSERT(pool);
4711         if (counter < MLX5_CNT_BATCH_OFFSET) {
4712                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4713                 if (cnt_ext) {
4714                         if (--cnt_ext->ref_cnt)
4715                                 return;
4716                         if (cnt_ext->shared)
4717                                 mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
4718                                                      cnt_ext->id);
4719                 }
4720         }
4721         if (IS_AGE_POOL(pool))
4722                 flow_dv_counter_remove_from_age(dev, counter, cnt);
4723         cnt->pool = pool;
4724         /*
4725          * Put the counter back to list to be updated in none fallback mode.
4726          * Currently, we are using two list alternately, while one is in query,
4727          * add the freed counter to the other list based on the pool query_gen
4728          * value. After query finishes, add counter the list to the global
4729          * container counter list. The list changes while query starts. In
4730          * this case, lock will not be needed as query callback and release
4731          * function both operate with the different list.
4732          *
4733          */
4734         if (!priv->counter_fallback)
4735                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
4736         else
4737                 TAILQ_INSERT_TAIL(&((MLX5_CNT_CONTAINER
4738                                   (priv->sh, 0, 0))->counters),
4739                                   cnt, next);
4740 }
4741
4742 /**
4743  * Verify the @p attributes will be correctly understood by the NIC and store
4744  * them in the @p flow if everything is correct.
4745  *
4746  * @param[in] dev
4747  *   Pointer to dev struct.
4748  * @param[in] attributes
4749  *   Pointer to flow attributes
4750  * @param[in] external
4751  *   This flow rule is created by request external to PMD.
4752  * @param[out] error
4753  *   Pointer to error structure.
4754  *
4755  * @return
4756  *   - 0 on success and non root table.
4757  *   - 1 on success and root table.
4758  *   - a negative errno value otherwise and rte_errno is set.
4759  */
4760 static int
4761 flow_dv_validate_attributes(struct rte_eth_dev *dev,
4762                             const struct rte_flow_attr *attributes,
4763                             bool external __rte_unused,
4764                             struct rte_flow_error *error)
4765 {
4766         struct mlx5_priv *priv = dev->data->dev_private;
4767         uint32_t priority_max = priv->config.flow_prio - 1;
4768         int ret = 0;
4769
4770 #ifndef HAVE_MLX5DV_DR
4771         if (attributes->group)
4772                 return rte_flow_error_set(error, ENOTSUP,
4773                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
4774                                           NULL,
4775                                           "groups are not supported");
4776 #else
4777         uint32_t table = 0;
4778
4779         ret = mlx5_flow_group_to_table(attributes, external,
4780                                        attributes->group, !!priv->fdb_def_rule,
4781                                        &table, error);
4782         if (ret)
4783                 return ret;
4784         if (!table)
4785                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4786 #endif
4787         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
4788             attributes->priority >= priority_max)
4789                 return rte_flow_error_set(error, ENOTSUP,
4790                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
4791                                           NULL,
4792                                           "priority out of range");
4793         if (attributes->transfer) {
4794                 if (!priv->config.dv_esw_en)
4795                         return rte_flow_error_set
4796                                 (error, ENOTSUP,
4797                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4798                                  "E-Switch dr is not supported");
4799                 if (!(priv->representor || priv->master))
4800                         return rte_flow_error_set
4801                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4802                                  NULL, "E-Switch configuration can only be"
4803                                  " done by a master or a representor device");
4804                 if (attributes->egress)
4805                         return rte_flow_error_set
4806                                 (error, ENOTSUP,
4807                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
4808                                  "egress is not supported");
4809         }
4810         if (!(attributes->egress ^ attributes->ingress))
4811                 return rte_flow_error_set(error, ENOTSUP,
4812                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
4813                                           "must specify exactly one of "
4814                                           "ingress or egress");
4815         return ret;
4816 }
4817
4818 /**
4819  * Internal validation function. For validating both actions and items.
4820  *
4821  * @param[in] dev
4822  *   Pointer to the rte_eth_dev structure.
4823  * @param[in] attr
4824  *   Pointer to the flow attributes.
4825  * @param[in] items
4826  *   Pointer to the list of items.
4827  * @param[in] actions
4828  *   Pointer to the list of actions.
4829  * @param[in] external
4830  *   This flow rule is created by request external to PMD.
4831  * @param[in] hairpin
4832  *   Number of hairpin TX actions, 0 means classic flow.
4833  * @param[out] error
4834  *   Pointer to the error structure.
4835  *
4836  * @return
4837  *   0 on success, a negative errno value otherwise and rte_errno is set.
4838  */
4839 static int
4840 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
4841                  const struct rte_flow_item items[],
4842                  const struct rte_flow_action actions[],
4843                  bool external, int hairpin, struct rte_flow_error *error)
4844 {
4845         int ret;
4846         uint64_t action_flags = 0;
4847         uint64_t item_flags = 0;
4848         uint64_t last_item = 0;
4849         uint8_t next_protocol = 0xff;
4850         uint16_t ether_type = 0;
4851         int actions_n = 0;
4852         uint8_t item_ipv6_proto = 0;
4853         const struct rte_flow_item *gre_item = NULL;
4854         const struct rte_flow_action_raw_decap *decap;
4855         const struct rte_flow_action_raw_encap *encap;
4856         const struct rte_flow_action_rss *rss;
4857         const struct rte_flow_item_tcp nic_tcp_mask = {
4858                 .hdr = {
4859                         .tcp_flags = 0xFF,
4860                         .src_port = RTE_BE16(UINT16_MAX),
4861                         .dst_port = RTE_BE16(UINT16_MAX),
4862                 }
4863         };
4864         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
4865                 .hdr = {
4866                         .src_addr = RTE_BE32(0xffffffff),
4867                         .dst_addr = RTE_BE32(0xffffffff),
4868                         .type_of_service = 0xff,
4869                         .next_proto_id = 0xff,
4870                         .time_to_live = 0xff,
4871                 },
4872         };
4873         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
4874                 .hdr = {
4875                         .src_addr =
4876                         "\xff\xff\xff\xff\xff\xff\xff\xff"
4877                         "\xff\xff\xff\xff\xff\xff\xff\xff",
4878                         .dst_addr =
4879                         "\xff\xff\xff\xff\xff\xff\xff\xff"
4880                         "\xff\xff\xff\xff\xff\xff\xff\xff",
4881                         .vtc_flow = RTE_BE32(0xffffffff),
4882                         .proto = 0xff,
4883                         .hop_limits = 0xff,
4884                 },
4885         };
4886         struct mlx5_priv *priv = dev->data->dev_private;
4887         struct mlx5_dev_config *dev_conf = &priv->config;
4888         uint16_t queue_index = 0xFFFF;
4889         const struct rte_flow_item_vlan *vlan_m = NULL;
4890         int16_t rw_act_num = 0;
4891         uint64_t is_root;
4892
4893         if (items == NULL)
4894                 return -1;
4895         ret = flow_dv_validate_attributes(dev, attr, external, error);
4896         if (ret < 0)
4897                 return ret;
4898         is_root = (uint64_t)ret;
4899         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4900                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
4901                 int type = items->type;
4902
4903                 switch (type) {
4904                 case RTE_FLOW_ITEM_TYPE_VOID:
4905                         break;
4906                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4907                         ret = flow_dv_validate_item_port_id
4908                                         (dev, items, attr, item_flags, error);
4909                         if (ret < 0)
4910                                 return ret;
4911                         last_item = MLX5_FLOW_ITEM_PORT_ID;
4912                         break;
4913                 case RTE_FLOW_ITEM_TYPE_ETH:
4914                         ret = mlx5_flow_validate_item_eth(items, item_flags,
4915                                                           error);
4916                         if (ret < 0)
4917                                 return ret;
4918                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
4919                                              MLX5_FLOW_LAYER_OUTER_L2;
4920                         if (items->mask != NULL && items->spec != NULL) {
4921                                 ether_type =
4922                                         ((const struct rte_flow_item_eth *)
4923                                          items->spec)->type;
4924                                 ether_type &=
4925                                         ((const struct rte_flow_item_eth *)
4926                                          items->mask)->type;
4927                                 ether_type = rte_be_to_cpu_16(ether_type);
4928                         } else {
4929                                 ether_type = 0;
4930                         }
4931                         break;
4932                 case RTE_FLOW_ITEM_TYPE_VLAN:
4933                         ret = flow_dv_validate_item_vlan(items, item_flags,
4934                                                          dev, error);
4935                         if (ret < 0)
4936                                 return ret;
4937                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
4938                                              MLX5_FLOW_LAYER_OUTER_VLAN;
4939                         if (items->mask != NULL && items->spec != NULL) {
4940                                 ether_type =
4941                                         ((const struct rte_flow_item_vlan *)
4942                                          items->spec)->inner_type;
4943                                 ether_type &=
4944                                         ((const struct rte_flow_item_vlan *)
4945                                          items->mask)->inner_type;
4946                                 ether_type = rte_be_to_cpu_16(ether_type);
4947                         } else {
4948                                 ether_type = 0;
4949                         }
4950                         /* Store outer VLAN mask for of_push_vlan action. */
4951                         if (!tunnel)
4952                                 vlan_m = items->mask;
4953                         break;
4954                 case RTE_FLOW_ITEM_TYPE_IPV4:
4955                         mlx5_flow_tunnel_ip_check(items, next_protocol,
4956                                                   &item_flags, &tunnel);
4957                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
4958                                                            last_item,
4959                                                            ether_type,
4960                                                            &nic_ipv4_mask,
4961                                                            error);
4962                         if (ret < 0)
4963                                 return ret;
4964                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
4965                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
4966                         if (items->mask != NULL &&
4967                             ((const struct rte_flow_item_ipv4 *)
4968                              items->mask)->hdr.next_proto_id) {
4969                                 next_protocol =
4970                                         ((const struct rte_flow_item_ipv4 *)
4971                                          (items->spec))->hdr.next_proto_id;
4972                                 next_protocol &=
4973                                         ((const struct rte_flow_item_ipv4 *)
4974                                          (items->mask))->hdr.next_proto_id;
4975                         } else {
4976                                 /* Reset for inner layer. */
4977                                 next_protocol = 0xff;
4978                         }
4979                         break;
4980                 case RTE_FLOW_ITEM_TYPE_IPV6:
4981                         mlx5_flow_tunnel_ip_check(items, next_protocol,
4982                                                   &item_flags, &tunnel);
4983                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
4984                                                            last_item,
4985                                                            ether_type,
4986                                                            &nic_ipv6_mask,
4987                                                            error);
4988                         if (ret < 0)
4989                                 return ret;
4990                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
4991                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
4992                         if (items->mask != NULL &&
4993                             ((const struct rte_flow_item_ipv6 *)
4994                              items->mask)->hdr.proto) {
4995                                 item_ipv6_proto =
4996                                         ((const struct rte_flow_item_ipv6 *)
4997                                          items->spec)->hdr.proto;
4998                                 next_protocol =
4999                                         ((const struct rte_flow_item_ipv6 *)
5000                                          items->spec)->hdr.proto;
5001                                 next_protocol &=
5002                                         ((const struct rte_flow_item_ipv6 *)
5003                                          items->mask)->hdr.proto;
5004                         } else {
5005                                 /* Reset for inner layer. */
5006                                 next_protocol = 0xff;
5007                         }
5008                         break;
5009                 case RTE_FLOW_ITEM_TYPE_TCP:
5010                         ret = mlx5_flow_validate_item_tcp
5011                                                 (items, item_flags,
5012                                                  next_protocol,
5013                                                  &nic_tcp_mask,
5014                                                  error);
5015                         if (ret < 0)
5016                                 return ret;
5017                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5018                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5019                         break;
5020                 case RTE_FLOW_ITEM_TYPE_UDP:
5021                         ret = mlx5_flow_validate_item_udp(items, item_flags,
5022                                                           next_protocol,
5023                                                           error);
5024                         if (ret < 0)
5025                                 return ret;
5026                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5027                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5028                         break;
5029                 case RTE_FLOW_ITEM_TYPE_GRE:
5030                         ret = mlx5_flow_validate_item_gre(items, item_flags,
5031                                                           next_protocol, error);
5032                         if (ret < 0)
5033                                 return ret;
5034                         gre_item = items;
5035                         last_item = MLX5_FLOW_LAYER_GRE;
5036                         break;
5037                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5038                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
5039                                                             next_protocol,
5040                                                             error);
5041                         if (ret < 0)
5042                                 return ret;
5043                         last_item = MLX5_FLOW_LAYER_NVGRE;
5044                         break;
5045                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5046                         ret = mlx5_flow_validate_item_gre_key
5047                                 (items, item_flags, gre_item, error);
5048                         if (ret < 0)
5049                                 return ret;
5050                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5051                         break;
5052                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5053                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
5054                                                             error);
5055                         if (ret < 0)
5056                                 return ret;
5057                         last_item = MLX5_FLOW_LAYER_VXLAN;
5058                         break;
5059                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5060                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
5061                                                                 item_flags, dev,
5062                                                                 error);
5063                         if (ret < 0)
5064                                 return ret;
5065                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5066                         break;
5067                 case RTE_FLOW_ITEM_TYPE_GENEVE:
5068                         ret = mlx5_flow_validate_item_geneve(items,
5069                                                              item_flags, dev,
5070                                                              error);
5071                         if (ret < 0)
5072                                 return ret;
5073                         last_item = MLX5_FLOW_LAYER_GENEVE;
5074                         break;
5075                 case RTE_FLOW_ITEM_TYPE_MPLS:
5076                         ret = mlx5_flow_validate_item_mpls(dev, items,
5077                                                            item_flags,
5078                                                            last_item, error);
5079                         if (ret < 0)
5080                                 return ret;
5081                         last_item = MLX5_FLOW_LAYER_MPLS;
5082                         break;
5083
5084                 case RTE_FLOW_ITEM_TYPE_MARK:
5085                         ret = flow_dv_validate_item_mark(dev, items, attr,
5086                                                          error);
5087                         if (ret < 0)
5088                                 return ret;
5089                         last_item = MLX5_FLOW_ITEM_MARK;
5090                         break;
5091                 case RTE_FLOW_ITEM_TYPE_META:
5092                         ret = flow_dv_validate_item_meta(dev, items, attr,
5093                                                          error);
5094                         if (ret < 0)
5095                                 return ret;
5096                         last_item = MLX5_FLOW_ITEM_METADATA;
5097                         break;
5098                 case RTE_FLOW_ITEM_TYPE_ICMP:
5099                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
5100                                                            next_protocol,
5101                                                            error);
5102                         if (ret < 0)
5103                                 return ret;
5104                         last_item = MLX5_FLOW_LAYER_ICMP;
5105                         break;
5106                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5107                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
5108                                                             next_protocol,
5109                                                             error);
5110                         if (ret < 0)
5111                                 return ret;
5112                         item_ipv6_proto = IPPROTO_ICMPV6;
5113                         last_item = MLX5_FLOW_LAYER_ICMP6;
5114                         break;
5115                 case RTE_FLOW_ITEM_TYPE_TAG:
5116                         ret = flow_dv_validate_item_tag(dev, items,
5117                                                         attr, error);
5118                         if (ret < 0)
5119                                 return ret;
5120                         last_item = MLX5_FLOW_ITEM_TAG;
5121                         break;
5122                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
5123                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
5124                         break;
5125                 case RTE_FLOW_ITEM_TYPE_GTP:
5126                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
5127                                                         error);
5128                         if (ret < 0)
5129                                 return ret;
5130                         last_item = MLX5_FLOW_LAYER_GTP;
5131                         break;
5132                 default:
5133                         return rte_flow_error_set(error, ENOTSUP,
5134                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5135                                                   NULL, "item not supported");
5136                 }
5137                 item_flags |= last_item;
5138         }
5139         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5140                 int type = actions->type;
5141                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5142                         return rte_flow_error_set(error, ENOTSUP,
5143                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5144                                                   actions, "too many actions");
5145                 switch (type) {
5146                 case RTE_FLOW_ACTION_TYPE_VOID:
5147                         break;
5148                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5149                         ret = flow_dv_validate_action_port_id(dev,
5150                                                               action_flags,
5151                                                               actions,
5152                                                               attr,
5153                                                               error);
5154                         if (ret)
5155                                 return ret;
5156                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5157                         ++actions_n;
5158                         break;
5159                 case RTE_FLOW_ACTION_TYPE_FLAG:
5160                         ret = flow_dv_validate_action_flag(dev, action_flags,
5161                                                            attr, error);
5162                         if (ret < 0)
5163                                 return ret;
5164                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5165                                 /* Count all modify-header actions as one. */
5166                                 if (!(action_flags &
5167                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5168                                         ++actions_n;
5169                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
5170                                                 MLX5_FLOW_ACTION_MARK_EXT;
5171                         } else {
5172                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
5173                                 ++actions_n;
5174                         }
5175                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5176                         break;
5177                 case RTE_FLOW_ACTION_TYPE_MARK:
5178                         ret = flow_dv_validate_action_mark(dev, actions,
5179                                                            action_flags,
5180                                                            attr, error);
5181                         if (ret < 0)
5182                                 return ret;
5183                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5184                                 /* Count all modify-header actions as one. */
5185                                 if (!(action_flags &
5186                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5187                                         ++actions_n;
5188                                 action_flags |= MLX5_FLOW_ACTION_MARK |
5189                                                 MLX5_FLOW_ACTION_MARK_EXT;
5190                         } else {
5191                                 action_flags |= MLX5_FLOW_ACTION_MARK;
5192                                 ++actions_n;
5193                         }
5194                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5195                         break;
5196                 case RTE_FLOW_ACTION_TYPE_SET_META:
5197                         ret = flow_dv_validate_action_set_meta(dev, actions,
5198                                                                action_flags,
5199                                                                attr, error);
5200                         if (ret < 0)
5201                                 return ret;
5202                         /* Count all modify-header actions as one action. */
5203                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5204                                 ++actions_n;
5205                         action_flags |= MLX5_FLOW_ACTION_SET_META;
5206                         rw_act_num += MLX5_ACT_NUM_SET_META;
5207                         break;
5208                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5209                         ret = flow_dv_validate_action_set_tag(dev, actions,
5210                                                               action_flags,
5211                                                               attr, error);
5212                         if (ret < 0)
5213                                 return ret;
5214                         /* Count all modify-header actions as one action. */
5215                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5216                                 ++actions_n;
5217                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
5218                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5219                         break;
5220                 case RTE_FLOW_ACTION_TYPE_DROP:
5221                         ret = mlx5_flow_validate_action_drop(action_flags,
5222                                                              attr, error);
5223                         if (ret < 0)
5224                                 return ret;
5225                         action_flags |= MLX5_FLOW_ACTION_DROP;
5226                         ++actions_n;
5227                         break;
5228                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5229                         ret = mlx5_flow_validate_action_queue(actions,
5230                                                               action_flags, dev,
5231                                                               attr, error);
5232                         if (ret < 0)
5233                                 return ret;
5234                         queue_index = ((const struct rte_flow_action_queue *)
5235                                                         (actions->conf))->index;
5236                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
5237                         ++actions_n;
5238                         break;
5239                 case RTE_FLOW_ACTION_TYPE_RSS:
5240                         rss = actions->conf;
5241                         ret = mlx5_flow_validate_action_rss(actions,
5242                                                             action_flags, dev,
5243                                                             attr, item_flags,
5244                                                             error);
5245                         if (ret < 0)
5246                                 return ret;
5247                         if (rss != NULL && rss->queue_num)
5248                                 queue_index = rss->queue[0];
5249                         action_flags |= MLX5_FLOW_ACTION_RSS;
5250                         ++actions_n;
5251                         break;
5252                 case RTE_FLOW_ACTION_TYPE_COUNT:
5253                         ret = flow_dv_validate_action_count(dev, error);
5254                         if (ret < 0)
5255                                 return ret;
5256                         action_flags |= MLX5_FLOW_ACTION_COUNT;
5257                         ++actions_n;
5258                         break;
5259                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5260                         if (flow_dv_validate_action_pop_vlan(dev,
5261                                                              action_flags,
5262                                                              actions,
5263                                                              item_flags, attr,
5264                                                              error))
5265                                 return -rte_errno;
5266                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5267                         ++actions_n;
5268                         break;
5269                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5270                         ret = flow_dv_validate_action_push_vlan(dev,
5271                                                                 action_flags,
5272                                                                 vlan_m,
5273                                                                 actions, attr,
5274                                                                 error);
5275                         if (ret < 0)
5276                                 return ret;
5277                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5278                         ++actions_n;
5279                         break;
5280                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5281                         ret = flow_dv_validate_action_set_vlan_pcp
5282                                                 (action_flags, actions, error);
5283                         if (ret < 0)
5284                                 return ret;
5285                         /* Count PCP with push_vlan command. */
5286                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
5287                         break;
5288                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5289                         ret = flow_dv_validate_action_set_vlan_vid
5290                                                 (item_flags, action_flags,
5291                                                  actions, error);
5292                         if (ret < 0)
5293                                 return ret;
5294                         /* Count VID with push_vlan command. */
5295                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
5296                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
5297                         break;
5298                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5299                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5300                         ret = flow_dv_validate_action_l2_encap(dev,
5301                                                                action_flags,
5302                                                                actions, attr,
5303                                                                error);
5304                         if (ret < 0)
5305                                 return ret;
5306                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
5307                         ++actions_n;
5308                         break;
5309                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5310                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5311                         ret = flow_dv_validate_action_decap(dev, action_flags,
5312                                                             attr, error);
5313                         if (ret < 0)
5314                                 return ret;
5315                         action_flags |= MLX5_FLOW_ACTION_DECAP;
5316                         ++actions_n;
5317                         break;
5318                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5319                         ret = flow_dv_validate_action_raw_encap_decap
5320                                 (dev, NULL, actions->conf, attr, &action_flags,
5321                                  &actions_n, error);
5322                         if (ret < 0)
5323                                 return ret;
5324                         break;
5325                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5326                         decap = actions->conf;
5327                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5328                                 ;
5329                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5330                                 encap = NULL;
5331                                 actions--;
5332                         } else {
5333                                 encap = actions->conf;
5334                         }
5335                         ret = flow_dv_validate_action_raw_encap_decap
5336                                            (dev,
5337                                             decap ? decap : &empty_decap, encap,
5338                                             attr, &action_flags, &actions_n,
5339                                             error);
5340                         if (ret < 0)
5341                                 return ret;
5342                         break;
5343                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5344                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5345                         ret = flow_dv_validate_action_modify_mac(action_flags,
5346                                                                  actions,
5347                                                                  item_flags,
5348                                                                  error);
5349                         if (ret < 0)
5350                                 return ret;
5351                         /* Count all modify-header actions as one action. */
5352                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5353                                 ++actions_n;
5354                         action_flags |= actions->type ==
5355                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5356                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
5357                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
5358                         /*
5359                          * Even if the source and destination MAC addresses have
5360                          * overlap in the header with 4B alignment, the convert
5361                          * function will handle them separately and 4 SW actions
5362                          * will be created. And 2 actions will be added each
5363                          * time no matter how many bytes of address will be set.
5364                          */
5365                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
5366                         break;
5367                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5368                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5369                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
5370                                                                   actions,
5371                                                                   item_flags,
5372                                                                   error);
5373                         if (ret < 0)
5374                                 return ret;
5375                         /* Count all modify-header actions as one action. */
5376                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5377                                 ++actions_n;
5378                         action_flags |= actions->type ==
5379                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5380                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
5381                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
5382                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
5383                         break;
5384                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5385                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5386                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
5387                                                                   actions,
5388                                                                   item_flags,
5389                                                                   error);
5390                         if (ret < 0)
5391                                 return ret;
5392                         if (item_ipv6_proto == IPPROTO_ICMPV6)
5393                                 return rte_flow_error_set(error, ENOTSUP,
5394                                         RTE_FLOW_ERROR_TYPE_ACTION,
5395                                         actions,
5396                                         "Can't change header "
5397                                         "with ICMPv6 proto");
5398                         /* Count all modify-header actions as one action. */
5399                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5400                                 ++actions_n;
5401                         action_flags |= actions->type ==
5402                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5403                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
5404                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
5405                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
5406                         break;
5407                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5408                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5409                         ret = flow_dv_validate_action_modify_tp(action_flags,
5410                                                                 actions,
5411                                                                 item_flags,
5412                                                                 error);
5413                         if (ret < 0)
5414                                 return ret;
5415                         /* Count all modify-header actions as one action. */
5416                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5417                                 ++actions_n;
5418                         action_flags |= actions->type ==
5419                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5420                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
5421                                                 MLX5_FLOW_ACTION_SET_TP_DST;
5422                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
5423                         break;
5424                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5425                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5426                         ret = flow_dv_validate_action_modify_ttl(action_flags,
5427                                                                  actions,
5428                                                                  item_flags,
5429                                                                  error);
5430                         if (ret < 0)
5431                                 return ret;
5432                         /* Count all modify-header actions as one action. */
5433                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5434                                 ++actions_n;
5435                         action_flags |= actions->type ==
5436                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
5437                                                 MLX5_FLOW_ACTION_SET_TTL :
5438                                                 MLX5_FLOW_ACTION_DEC_TTL;
5439                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
5440                         break;
5441                 case RTE_FLOW_ACTION_TYPE_JUMP:
5442                         ret = flow_dv_validate_action_jump(actions,
5443                                                            action_flags,
5444                                                            attr, external,
5445                                                            error);
5446                         if (ret)
5447                                 return ret;
5448                         ++actions_n;
5449                         action_flags |= MLX5_FLOW_ACTION_JUMP;
5450                         break;
5451                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5452                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5453                         ret = flow_dv_validate_action_modify_tcp_seq
5454                                                                 (action_flags,
5455                                                                  actions,
5456                                                                  item_flags,
5457                                                                  error);
5458                         if (ret < 0)
5459                                 return ret;
5460                         /* Count all modify-header actions as one action. */
5461                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5462                                 ++actions_n;
5463                         action_flags |= actions->type ==
5464                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5465                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
5466                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5467                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
5468                         break;
5469                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5470                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5471                         ret = flow_dv_validate_action_modify_tcp_ack
5472                                                                 (action_flags,
5473                                                                  actions,
5474                                                                  item_flags,
5475                                                                  error);
5476                         if (ret < 0)
5477                                 return ret;
5478                         /* Count all modify-header actions as one action. */
5479                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5480                                 ++actions_n;
5481                         action_flags |= actions->type ==
5482                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
5483                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
5484                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
5485                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
5486                         break;
5487                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
5488                         break;
5489                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
5490                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
5491                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5492                         break;
5493                 case RTE_FLOW_ACTION_TYPE_METER:
5494                         ret = mlx5_flow_validate_action_meter(dev,
5495                                                               action_flags,
5496                                                               actions, attr,
5497                                                               error);
5498                         if (ret < 0)
5499                                 return ret;
5500                         action_flags |= MLX5_FLOW_ACTION_METER;
5501                         ++actions_n;
5502                         /* Meter action will add one more TAG action. */
5503                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5504                         break;
5505                 case RTE_FLOW_ACTION_TYPE_AGE:
5506                         ret = flow_dv_validate_action_age(action_flags,
5507                                                           actions, dev,
5508                                                           error);
5509                         if (ret < 0)
5510                                 return ret;
5511                         action_flags |= MLX5_FLOW_ACTION_AGE;
5512                         ++actions_n;
5513                         break;
5514                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5515                         ret = flow_dv_validate_action_modify_ipv4_dscp
5516                                                          (action_flags,
5517                                                           actions,
5518                                                           item_flags,
5519                                                           error);
5520                         if (ret < 0)
5521                                 return ret;
5522                         /* Count all modify-header actions as one action. */
5523                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5524                                 ++actions_n;
5525                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
5526                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5527                         break;
5528                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5529                         ret = flow_dv_validate_action_modify_ipv6_dscp
5530                                                                 (action_flags,
5531                                                                  actions,
5532                                                                  item_flags,
5533                                                                  error);
5534                         if (ret < 0)
5535                                 return ret;
5536                         /* Count all modify-header actions as one action. */
5537                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5538                                 ++actions_n;
5539                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
5540                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5541                         break;
5542                 default:
5543                         return rte_flow_error_set(error, ENOTSUP,
5544                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5545                                                   actions,
5546                                                   "action not supported");
5547                 }
5548         }
5549         /*
5550          * Validate the drop action mutual exclusion with other actions.
5551          * Drop action is mutually-exclusive with any other action, except for
5552          * Count action.
5553          */
5554         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
5555             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
5556                 return rte_flow_error_set(error, EINVAL,
5557                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5558                                           "Drop action is mutually-exclusive "
5559                                           "with any other action, except for "
5560                                           "Count action");
5561         /* Eswitch has few restrictions on using items and actions */
5562         if (attr->transfer) {
5563                 if (!mlx5_flow_ext_mreg_supported(dev) &&
5564                     action_flags & MLX5_FLOW_ACTION_FLAG)
5565                         return rte_flow_error_set(error, ENOTSUP,
5566                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5567                                                   NULL,
5568                                                   "unsupported action FLAG");
5569                 if (!mlx5_flow_ext_mreg_supported(dev) &&
5570                     action_flags & MLX5_FLOW_ACTION_MARK)
5571                         return rte_flow_error_set(error, ENOTSUP,
5572                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5573                                                   NULL,
5574                                                   "unsupported action MARK");
5575                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
5576                         return rte_flow_error_set(error, ENOTSUP,
5577                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5578                                                   NULL,
5579                                                   "unsupported action QUEUE");
5580                 if (action_flags & MLX5_FLOW_ACTION_RSS)
5581                         return rte_flow_error_set(error, ENOTSUP,
5582                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5583                                                   NULL,
5584                                                   "unsupported action RSS");
5585                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5586                         return rte_flow_error_set(error, EINVAL,
5587                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5588                                                   actions,
5589                                                   "no fate action is found");
5590         } else {
5591                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
5592                         return rte_flow_error_set(error, EINVAL,
5593                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5594                                                   actions,
5595                                                   "no fate action is found");
5596         }
5597         /* Continue validation for Xcap actions.*/
5598         if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) && (queue_index == 0xFFFF ||
5599             mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
5600                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
5601                     MLX5_FLOW_XCAP_ACTIONS)
5602                         return rte_flow_error_set(error, ENOTSUP,
5603                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5604                                                   NULL, "encap and decap "
5605                                                   "combination aren't supported");
5606                 if (!attr->transfer && attr->ingress && (action_flags &
5607                                                         MLX5_FLOW_ACTION_ENCAP))
5608                         return rte_flow_error_set(error, ENOTSUP,
5609                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5610                                                   NULL, "encap is not supported"
5611                                                   " for ingress traffic");
5612         }
5613         /* Hairpin flow will add one more TAG action. */
5614         if (hairpin > 0)
5615                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5616         /* extra metadata enabled: one more TAG action will be add. */
5617         if (dev_conf->dv_flow_en &&
5618             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
5619             mlx5_flow_ext_mreg_supported(dev))
5620                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5621         if ((uint32_t)rw_act_num >
5622                         flow_dv_modify_hdr_action_max(dev, is_root)) {
5623                 return rte_flow_error_set(error, ENOTSUP,
5624                                           RTE_FLOW_ERROR_TYPE_ACTION,
5625                                           NULL, "too many header modify"
5626                                           " actions to support");
5627         }
5628         return 0;
5629 }
5630
5631 /**
5632  * Internal preparation function. Allocates the DV flow size,
5633  * this size is constant.
5634  *
5635  * @param[in] dev
5636  *   Pointer to the rte_eth_dev structure.
5637  * @param[in] attr
5638  *   Pointer to the flow attributes.
5639  * @param[in] items
5640  *   Pointer to the list of items.
5641  * @param[in] actions
5642  *   Pointer to the list of actions.
5643  * @param[out] error
5644  *   Pointer to the error structure.
5645  *
5646  * @return
5647  *   Pointer to mlx5_flow object on success,
5648  *   otherwise NULL and rte_errno is set.
5649  */
5650 static struct mlx5_flow *
5651 flow_dv_prepare(struct rte_eth_dev *dev,
5652                 const struct rte_flow_attr *attr __rte_unused,
5653                 const struct rte_flow_item items[] __rte_unused,
5654                 const struct rte_flow_action actions[] __rte_unused,
5655                 struct rte_flow_error *error)
5656 {
5657         uint32_t handle_idx = 0;
5658         struct mlx5_flow *dev_flow;
5659         struct mlx5_flow_handle *dev_handle;
5660         struct mlx5_priv *priv = dev->data->dev_private;
5661
5662         /* In case of corrupting the memory. */
5663         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
5664                 rte_flow_error_set(error, ENOSPC,
5665                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5666                                    "not free temporary device flow");
5667                 return NULL;
5668         }
5669         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
5670                                    &handle_idx);
5671         if (!dev_handle) {
5672                 rte_flow_error_set(error, ENOMEM,
5673                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5674                                    "not enough memory to create flow handle");
5675                 return NULL;
5676         }
5677         /* No multi-thread supporting. */
5678         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
5679         dev_flow->handle = dev_handle;
5680         dev_flow->handle_idx = handle_idx;
5681         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
5682         /*
5683          * The matching value needs to be cleared to 0 before using. In the
5684          * past, it will be automatically cleared when using rte_*alloc
5685          * API. The time consumption will be almost the same as before.
5686          */
5687         memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
5688         dev_flow->ingress = attr->ingress;
5689         dev_flow->dv.transfer = attr->transfer;
5690         return dev_flow;
5691 }
5692
5693 #ifdef RTE_LIBRTE_MLX5_DEBUG
5694 /**
5695  * Sanity check for match mask and value. Similar to check_valid_spec() in
5696  * kernel driver. If unmasked bit is present in value, it returns failure.
5697  *
5698  * @param match_mask
5699  *   pointer to match mask buffer.
5700  * @param match_value
5701  *   pointer to match value buffer.
5702  *
5703  * @return
5704  *   0 if valid, -EINVAL otherwise.
5705  */
5706 static int
5707 flow_dv_check_valid_spec(void *match_mask, void *match_value)
5708 {
5709         uint8_t *m = match_mask;
5710         uint8_t *v = match_value;
5711         unsigned int i;
5712
5713         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
5714                 if (v[i] & ~m[i]) {
5715                         DRV_LOG(ERR,
5716                                 "match_value differs from match_criteria"
5717                                 " %p[%u] != %p[%u]",
5718                                 match_value, i, match_mask, i);
5719                         return -EINVAL;
5720                 }
5721         }
5722         return 0;
5723 }
5724 #endif
5725
5726 /**
5727  * Add match of ip_version.
5728  *
5729  * @param[in] group
5730  *   Flow group.
5731  * @param[in] headers_v
5732  *   Values header pointer.
5733  * @param[in] headers_m
5734  *   Masks header pointer.
5735  * @param[in] ip_version
5736  *   The IP version to set.
5737  */
5738 static inline void
5739 flow_dv_set_match_ip_version(uint32_t group,
5740                              void *headers_v,
5741                              void *headers_m,
5742                              uint8_t ip_version)
5743 {
5744         if (group == 0)
5745                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
5746         else
5747                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
5748                          ip_version);
5749         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
5750         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
5751         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
5752 }
5753
5754 /**
5755  * Add Ethernet item to matcher and to the value.
5756  *
5757  * @param[in, out] matcher
5758  *   Flow matcher.
5759  * @param[in, out] key
5760  *   Flow matcher value.
5761  * @param[in] item
5762  *   Flow pattern to translate.
5763  * @param[in] inner
5764  *   Item is inner pattern.
5765  */
5766 static void
5767 flow_dv_translate_item_eth(void *matcher, void *key,
5768                            const struct rte_flow_item *item, int inner,
5769                            uint32_t group)
5770 {
5771         const struct rte_flow_item_eth *eth_m = item->mask;
5772         const struct rte_flow_item_eth *eth_v = item->spec;
5773         const struct rte_flow_item_eth nic_mask = {
5774                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
5775                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
5776                 .type = RTE_BE16(0xffff),
5777         };
5778         void *headers_m;
5779         void *headers_v;
5780         char *l24_v;
5781         unsigned int i;
5782
5783         if (!eth_v)
5784                 return;
5785         if (!eth_m)
5786                 eth_m = &nic_mask;
5787         if (inner) {
5788                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5789                                          inner_headers);
5790                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5791         } else {
5792                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5793                                          outer_headers);
5794                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5795         }
5796         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
5797                &eth_m->dst, sizeof(eth_m->dst));
5798         /* The value must be in the range of the mask. */
5799         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
5800         for (i = 0; i < sizeof(eth_m->dst); ++i)
5801                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
5802         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
5803                &eth_m->src, sizeof(eth_m->src));
5804         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
5805         /* The value must be in the range of the mask. */
5806         for (i = 0; i < sizeof(eth_m->dst); ++i)
5807                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
5808         if (eth_v->type) {
5809                 /* When ethertype is present set mask for tagged VLAN. */
5810                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
5811                 /* Set value for tagged VLAN if ethertype is 802.1Q. */
5812                 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
5813                     eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
5814                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag,
5815                                  1);
5816                         /* Return here to avoid setting match on ethertype. */
5817                         return;
5818                 }
5819         }
5820         /*
5821          * HW supports match on one Ethertype, the Ethertype following the last
5822          * VLAN tag of the packet (see PRM).
5823          * Set match on ethertype only if ETH header is not followed by VLAN.
5824          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
5825          * ethertype, and use ip_version field instead.
5826          */
5827         if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
5828             eth_m->type == 0xFFFF) {
5829                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
5830         } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
5831                    eth_m->type == 0xFFFF) {
5832                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
5833         } else {
5834                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
5835                          rte_be_to_cpu_16(eth_m->type));
5836                 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
5837                                      ethertype);
5838                 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
5839         }
5840 }
5841
5842 /**
5843  * Add VLAN item to matcher and to the value.
5844  *
5845  * @param[in, out] dev_flow
5846  *   Flow descriptor.
5847  * @param[in, out] matcher
5848  *   Flow matcher.
5849  * @param[in, out] key
5850  *   Flow matcher value.
5851  * @param[in] item
5852  *   Flow pattern to translate.
5853  * @param[in] inner
5854  *   Item is inner pattern.
5855  */
5856 static void
5857 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
5858                             void *matcher, void *key,
5859                             const struct rte_flow_item *item,
5860                             int inner, uint32_t group)
5861 {
5862         const struct rte_flow_item_vlan *vlan_m = item->mask;
5863         const struct rte_flow_item_vlan *vlan_v = item->spec;
5864         void *headers_m;
5865         void *headers_v;
5866         uint16_t tci_m;
5867         uint16_t tci_v;
5868
5869         if (inner) {
5870                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5871                                          inner_headers);
5872                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5873         } else {
5874                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5875                                          outer_headers);
5876                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5877                 /*
5878                  * This is workaround, masks are not supported,
5879                  * and pre-validated.
5880                  */
5881                 if (vlan_v)
5882                         dev_flow->handle->vf_vlan.tag =
5883                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
5884         }
5885         /*
5886          * When VLAN item exists in flow, mark packet as tagged,
5887          * even if TCI is not specified.
5888          */
5889         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
5890         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
5891         if (!vlan_v)
5892                 return;
5893         if (!vlan_m)
5894                 vlan_m = &rte_flow_item_vlan_mask;
5895         tci_m = rte_be_to_cpu_16(vlan_m->tci);
5896         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
5897         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
5898         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
5899         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
5900         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
5901         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
5902         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
5903         /*
5904          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
5905          * ethertype, and use ip_version field instead.
5906          */
5907         if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
5908             vlan_m->inner_type == 0xFFFF) {
5909                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
5910         } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
5911                    vlan_m->inner_type == 0xFFFF) {
5912                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
5913         } else {
5914                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
5915                          rte_be_to_cpu_16(vlan_m->inner_type));
5916                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
5917                          rte_be_to_cpu_16(vlan_m->inner_type &
5918                                           vlan_v->inner_type));
5919         }
5920 }
5921
5922 /**
5923  * Add IPV4 item to matcher and to the value.
5924  *
5925  * @param[in, out] matcher
5926  *   Flow matcher.
5927  * @param[in, out] key
5928  *   Flow matcher value.
5929  * @param[in] item
5930  *   Flow pattern to translate.
5931  * @param[in] item_flags
5932  *   Bit-fields that holds the items detected until now.
5933  * @param[in] inner
5934  *   Item is inner pattern.
5935  * @param[in] group
5936  *   The group to insert the rule.
5937  */
5938 static void
5939 flow_dv_translate_item_ipv4(void *matcher, void *key,
5940                             const struct rte_flow_item *item,
5941                             const uint64_t item_flags,
5942                             int inner, uint32_t group)
5943 {
5944         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
5945         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
5946         const struct rte_flow_item_ipv4 nic_mask = {
5947                 .hdr = {
5948                         .src_addr = RTE_BE32(0xffffffff),
5949                         .dst_addr = RTE_BE32(0xffffffff),
5950                         .type_of_service = 0xff,
5951                         .next_proto_id = 0xff,
5952                         .time_to_live = 0xff,
5953                 },
5954         };
5955         void *headers_m;
5956         void *headers_v;
5957         char *l24_m;
5958         char *l24_v;
5959         uint8_t tos;
5960
5961         if (inner) {
5962                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5963                                          inner_headers);
5964                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5965         } else {
5966                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5967                                          outer_headers);
5968                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5969         }
5970         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
5971         /*
5972          * On outer header (which must contains L2), or inner header with L2,
5973          * set cvlan_tag mask bit to mark this packet as untagged.
5974          * This should be done even if item->spec is empty.
5975          */
5976         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
5977                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
5978         if (!ipv4_v)
5979                 return;
5980         if (!ipv4_m)
5981                 ipv4_m = &nic_mask;
5982         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
5983                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
5984         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
5985                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
5986         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
5987         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
5988         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
5989                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
5990         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
5991                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
5992         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
5993         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
5994         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
5995         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
5996                  ipv4_m->hdr.type_of_service);
5997         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
5998         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
5999                  ipv4_m->hdr.type_of_service >> 2);
6000         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
6001         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6002                  ipv4_m->hdr.next_proto_id);
6003         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6004                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
6005         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6006                  ipv4_m->hdr.time_to_live);
6007         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6008                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
6009 }
6010
6011 /**
6012  * Add IPV6 item to matcher and to the value.
6013  *
6014  * @param[in, out] matcher
6015  *   Flow matcher.
6016  * @param[in, out] key
6017  *   Flow matcher value.
6018  * @param[in] item
6019  *   Flow pattern to translate.
6020  * @param[in] item_flags
6021  *   Bit-fields that holds the items detected until now.
6022  * @param[in] inner
6023  *   Item is inner pattern.
6024  * @param[in] group
6025  *   The group to insert the rule.
6026  */
6027 static void
6028 flow_dv_translate_item_ipv6(void *matcher, void *key,
6029                             const struct rte_flow_item *item,
6030                             const uint64_t item_flags,
6031                             int inner, uint32_t group)
6032 {
6033         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
6034         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
6035         const struct rte_flow_item_ipv6 nic_mask = {
6036                 .hdr = {
6037                         .src_addr =
6038                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6039                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6040                         .dst_addr =
6041                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6042                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6043                         .vtc_flow = RTE_BE32(0xffffffff),
6044                         .proto = 0xff,
6045                         .hop_limits = 0xff,
6046                 },
6047         };
6048         void *headers_m;
6049         void *headers_v;
6050         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6051         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6052         char *l24_m;
6053         char *l24_v;
6054         uint32_t vtc_m;
6055         uint32_t vtc_v;
6056         int i;
6057         int size;
6058
6059         if (inner) {
6060                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6061                                          inner_headers);
6062                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6063         } else {
6064                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6065                                          outer_headers);
6066                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6067         }
6068         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6069         /*
6070          * On outer header (which must contains L2), or inner header with L2,
6071          * set cvlan_tag mask bit to mark this packet as untagged.
6072          * This should be done even if item->spec is empty.
6073          */
6074         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6075                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6076         if (!ipv6_v)
6077                 return;
6078         if (!ipv6_m)
6079                 ipv6_m = &nic_mask;
6080         size = sizeof(ipv6_m->hdr.dst_addr);
6081         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6082                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6083         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6084                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6085         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
6086         for (i = 0; i < size; ++i)
6087                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
6088         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6089                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6090         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6091                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6092         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
6093         for (i = 0; i < size; ++i)
6094                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
6095         /* TOS. */
6096         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
6097         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
6098         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
6099         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
6100         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
6101         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
6102         /* Label. */
6103         if (inner) {
6104                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
6105                          vtc_m);
6106                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
6107                          vtc_v);
6108         } else {
6109                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
6110                          vtc_m);
6111                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
6112                          vtc_v);
6113         }
6114         /* Protocol. */
6115         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6116                  ipv6_m->hdr.proto);
6117         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6118                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
6119         /* Hop limit. */
6120         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6121                  ipv6_m->hdr.hop_limits);
6122         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6123                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
6124 }
6125
6126 /**
6127  * Add TCP item to matcher and to the value.
6128  *
6129  * @param[in, out] matcher
6130  *   Flow matcher.
6131  * @param[in, out] key
6132  *   Flow matcher value.
6133  * @param[in] item
6134  *   Flow pattern to translate.
6135  * @param[in] inner
6136  *   Item is inner pattern.
6137  */
6138 static void
6139 flow_dv_translate_item_tcp(void *matcher, void *key,
6140                            const struct rte_flow_item *item,
6141                            int inner)
6142 {
6143         const struct rte_flow_item_tcp *tcp_m = item->mask;
6144         const struct rte_flow_item_tcp *tcp_v = item->spec;
6145         void *headers_m;
6146         void *headers_v;
6147
6148         if (inner) {
6149                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6150                                          inner_headers);
6151                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6152         } else {
6153                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6154                                          outer_headers);
6155                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6156         }
6157         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6158         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
6159         if (!tcp_v)
6160                 return;
6161         if (!tcp_m)
6162                 tcp_m = &rte_flow_item_tcp_mask;
6163         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
6164                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
6165         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
6166                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
6167         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
6168                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
6169         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
6170                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
6171         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
6172                  tcp_m->hdr.tcp_flags);
6173         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
6174                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
6175 }
6176
6177 /**
6178  * Add UDP item to matcher and to the value.
6179  *
6180  * @param[in, out] matcher
6181  *   Flow matcher.
6182  * @param[in, out] key
6183  *   Flow matcher value.
6184  * @param[in] item
6185  *   Flow pattern to translate.
6186  * @param[in] inner
6187  *   Item is inner pattern.
6188  */
6189 static void
6190 flow_dv_translate_item_udp(void *matcher, void *key,
6191                            const struct rte_flow_item *item,
6192                            int inner)
6193 {
6194         const struct rte_flow_item_udp *udp_m = item->mask;
6195         const struct rte_flow_item_udp *udp_v = item->spec;
6196         void *headers_m;
6197         void *headers_v;
6198
6199         if (inner) {
6200                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6201                                          inner_headers);
6202                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6203         } else {
6204                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6205                                          outer_headers);
6206                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6207         }
6208         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6209         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
6210         if (!udp_v)
6211                 return;
6212         if (!udp_m)
6213                 udp_m = &rte_flow_item_udp_mask;
6214         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
6215                  rte_be_to_cpu_16(udp_m->hdr.src_port));
6216         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
6217                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
6218         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
6219                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
6220         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6221                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
6222 }
6223
6224 /**
6225  * Add GRE optional Key item to matcher and to the value.
6226  *
6227  * @param[in, out] matcher
6228  *   Flow matcher.
6229  * @param[in, out] key
6230  *   Flow matcher value.
6231  * @param[in] item
6232  *   Flow pattern to translate.
6233  * @param[in] inner
6234  *   Item is inner pattern.
6235  */
6236 static void
6237 flow_dv_translate_item_gre_key(void *matcher, void *key,
6238                                    const struct rte_flow_item *item)
6239 {
6240         const rte_be32_t *key_m = item->mask;
6241         const rte_be32_t *key_v = item->spec;
6242         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6243         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6244         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
6245
6246         /* GRE K bit must be on and should already be validated */
6247         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
6248         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
6249         if (!key_v)
6250                 return;
6251         if (!key_m)
6252                 key_m = &gre_key_default_mask;
6253         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
6254                  rte_be_to_cpu_32(*key_m) >> 8);
6255         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
6256                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
6257         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
6258                  rte_be_to_cpu_32(*key_m) & 0xFF);
6259         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
6260                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
6261 }
6262
6263 /**
6264  * Add GRE item to matcher and to the value.
6265  *
6266  * @param[in, out] matcher
6267  *   Flow matcher.
6268  * @param[in, out] key
6269  *   Flow matcher value.
6270  * @param[in] item
6271  *   Flow pattern to translate.
6272  * @param[in] inner
6273  *   Item is inner pattern.
6274  */
6275 static void
6276 flow_dv_translate_item_gre(void *matcher, void *key,
6277                            const struct rte_flow_item *item,
6278                            int inner)
6279 {
6280         const struct rte_flow_item_gre *gre_m = item->mask;
6281         const struct rte_flow_item_gre *gre_v = item->spec;
6282         void *headers_m;
6283         void *headers_v;
6284         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6285         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6286         struct {
6287                 union {
6288                         __extension__
6289                         struct {
6290                                 uint16_t version:3;
6291                                 uint16_t rsvd0:9;
6292                                 uint16_t s_present:1;
6293                                 uint16_t k_present:1;
6294                                 uint16_t rsvd_bit1:1;
6295                                 uint16_t c_present:1;
6296                         };
6297                         uint16_t value;
6298                 };
6299         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
6300
6301         if (inner) {
6302                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6303                                          inner_headers);
6304                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6305         } else {
6306                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6307                                          outer_headers);
6308                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6309         }
6310         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6311         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
6312         if (!gre_v)
6313                 return;
6314         if (!gre_m)
6315                 gre_m = &rte_flow_item_gre_mask;
6316         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
6317                  rte_be_to_cpu_16(gre_m->protocol));
6318         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6319                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
6320         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
6321         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
6322         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
6323                  gre_crks_rsvd0_ver_m.c_present);
6324         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
6325                  gre_crks_rsvd0_ver_v.c_present &
6326                  gre_crks_rsvd0_ver_m.c_present);
6327         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
6328                  gre_crks_rsvd0_ver_m.k_present);
6329         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
6330                  gre_crks_rsvd0_ver_v.k_present &
6331                  gre_crks_rsvd0_ver_m.k_present);
6332         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
6333                  gre_crks_rsvd0_ver_m.s_present);
6334         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
6335                  gre_crks_rsvd0_ver_v.s_present &
6336                  gre_crks_rsvd0_ver_m.s_present);
6337 }
6338
6339 /**
6340  * Add NVGRE item to matcher and to the value.
6341  *
6342  * @param[in, out] matcher
6343  *   Flow matcher.
6344  * @param[in, out] key
6345  *   Flow matcher value.
6346  * @param[in] item
6347  *   Flow pattern to translate.
6348  * @param[in] inner
6349  *   Item is inner pattern.
6350  */
6351 static void
6352 flow_dv_translate_item_nvgre(void *matcher, void *key,
6353                              const struct rte_flow_item *item,
6354                              int inner)
6355 {
6356         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
6357         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
6358         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6359         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6360         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
6361         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
6362         char *gre_key_m;
6363         char *gre_key_v;
6364         int size;
6365         int i;
6366
6367         /* For NVGRE, GRE header fields must be set with defined values. */
6368         const struct rte_flow_item_gre gre_spec = {
6369                 .c_rsvd0_ver = RTE_BE16(0x2000),
6370                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
6371         };
6372         const struct rte_flow_item_gre gre_mask = {
6373                 .c_rsvd0_ver = RTE_BE16(0xB000),
6374                 .protocol = RTE_BE16(UINT16_MAX),
6375         };
6376         const struct rte_flow_item gre_item = {
6377                 .spec = &gre_spec,
6378                 .mask = &gre_mask,
6379                 .last = NULL,
6380         };
6381         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
6382         if (!nvgre_v)
6383                 return;
6384         if (!nvgre_m)
6385                 nvgre_m = &rte_flow_item_nvgre_mask;
6386         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
6387         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
6388         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
6389         memcpy(gre_key_m, tni_flow_id_m, size);
6390         for (i = 0; i < size; ++i)
6391                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
6392 }
6393
6394 /**
6395  * Add VXLAN item to matcher and to the value.
6396  *
6397  * @param[in, out] matcher
6398  *   Flow matcher.
6399  * @param[in, out] key
6400  *   Flow matcher value.
6401  * @param[in] item
6402  *   Flow pattern to translate.
6403  * @param[in] inner
6404  *   Item is inner pattern.
6405  */
6406 static void
6407 flow_dv_translate_item_vxlan(void *matcher, void *key,
6408                              const struct rte_flow_item *item,
6409                              int inner)
6410 {
6411         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
6412         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
6413         void *headers_m;
6414         void *headers_v;
6415         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6416         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6417         char *vni_m;
6418         char *vni_v;
6419         uint16_t dport;
6420         int size;
6421         int i;
6422
6423         if (inner) {
6424                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6425                                          inner_headers);
6426                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6427         } else {
6428                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6429                                          outer_headers);
6430                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6431         }
6432         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6433                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6434         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6435                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6436                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6437         }
6438         if (!vxlan_v)
6439                 return;
6440         if (!vxlan_m)
6441                 vxlan_m = &rte_flow_item_vxlan_mask;
6442         size = sizeof(vxlan_m->vni);
6443         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
6444         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
6445         memcpy(vni_m, vxlan_m->vni, size);
6446         for (i = 0; i < size; ++i)
6447                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6448 }
6449
6450 /**
6451  * Add VXLAN-GPE item to matcher and to the value.
6452  *
6453  * @param[in, out] matcher
6454  *   Flow matcher.
6455  * @param[in, out] key
6456  *   Flow matcher value.
6457  * @param[in] item
6458  *   Flow pattern to translate.
6459  * @param[in] inner
6460  *   Item is inner pattern.
6461  */
6462
6463 static void
6464 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
6465                                  const struct rte_flow_item *item, int inner)
6466 {
6467         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
6468         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
6469         void *headers_m;
6470         void *headers_v;
6471         void *misc_m =
6472                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
6473         void *misc_v =
6474                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
6475         char *vni_m;
6476         char *vni_v;
6477         uint16_t dport;
6478         int size;
6479         int i;
6480         uint8_t flags_m = 0xff;
6481         uint8_t flags_v = 0xc;
6482
6483         if (inner) {
6484                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6485                                          inner_headers);
6486                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6487         } else {
6488                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6489                                          outer_headers);
6490                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6491         }
6492         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6493                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6494         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6495                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6496                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6497         }
6498         if (!vxlan_v)
6499                 return;
6500         if (!vxlan_m)
6501                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
6502         size = sizeof(vxlan_m->vni);
6503         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
6504         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
6505         memcpy(vni_m, vxlan_m->vni, size);
6506         for (i = 0; i < size; ++i)
6507                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6508         if (vxlan_m->flags) {
6509                 flags_m = vxlan_m->flags;
6510                 flags_v = vxlan_v->flags;
6511         }
6512         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
6513         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
6514         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
6515                  vxlan_m->protocol);
6516         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
6517                  vxlan_v->protocol);
6518 }
6519
6520 /**
6521  * Add Geneve item to matcher and to the value.
6522  *
6523  * @param[in, out] matcher
6524  *   Flow matcher.
6525  * @param[in, out] key
6526  *   Flow matcher value.
6527  * @param[in] item
6528  *   Flow pattern to translate.
6529  * @param[in] inner
6530  *   Item is inner pattern.
6531  */
6532
6533 static void
6534 flow_dv_translate_item_geneve(void *matcher, void *key,
6535                               const struct rte_flow_item *item, int inner)
6536 {
6537         const struct rte_flow_item_geneve *geneve_m = item->mask;
6538         const struct rte_flow_item_geneve *geneve_v = item->spec;
6539         void *headers_m;
6540         void *headers_v;
6541         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6542         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6543         uint16_t dport;
6544         uint16_t gbhdr_m;
6545         uint16_t gbhdr_v;
6546         char *vni_m;
6547         char *vni_v;
6548         size_t size, i;
6549
6550         if (inner) {
6551                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6552                                          inner_headers);
6553                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6554         } else {
6555                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6556                                          outer_headers);
6557                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6558         }
6559         dport = MLX5_UDP_PORT_GENEVE;
6560         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6561                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6562                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6563         }
6564         if (!geneve_v)
6565                 return;
6566         if (!geneve_m)
6567                 geneve_m = &rte_flow_item_geneve_mask;
6568         size = sizeof(geneve_m->vni);
6569         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
6570         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
6571         memcpy(vni_m, geneve_m->vni, size);
6572         for (i = 0; i < size; ++i)
6573                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
6574         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
6575                  rte_be_to_cpu_16(geneve_m->protocol));
6576         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
6577                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
6578         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
6579         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
6580         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
6581                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
6582         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
6583                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
6584         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
6585                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
6586         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
6587                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
6588                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
6589 }
6590
6591 /**
6592  * Add MPLS item to matcher and to the value.
6593  *
6594  * @param[in, out] matcher
6595  *   Flow matcher.
6596  * @param[in, out] key
6597  *   Flow matcher value.
6598  * @param[in] item
6599  *   Flow pattern to translate.
6600  * @param[in] prev_layer
6601  *   The protocol layer indicated in previous item.
6602  * @param[in] inner
6603  *   Item is inner pattern.
6604  */
6605 static void
6606 flow_dv_translate_item_mpls(void *matcher, void *key,
6607                             const struct rte_flow_item *item,
6608                             uint64_t prev_layer,
6609                             int inner)
6610 {
6611         const uint32_t *in_mpls_m = item->mask;
6612         const uint32_t *in_mpls_v = item->spec;
6613         uint32_t *out_mpls_m = 0;
6614         uint32_t *out_mpls_v = 0;
6615         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6616         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6617         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
6618                                      misc_parameters_2);
6619         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
6620         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
6621         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6622
6623         switch (prev_layer) {
6624         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
6625                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
6626                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6627                          MLX5_UDP_PORT_MPLS);
6628                 break;
6629         case MLX5_FLOW_LAYER_GRE:
6630                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
6631                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6632                          RTE_ETHER_TYPE_MPLS);
6633                 break;
6634         default:
6635                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6636                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6637                          IPPROTO_MPLS);
6638                 break;
6639         }
6640         if (!in_mpls_v)
6641                 return;
6642         if (!in_mpls_m)
6643                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
6644         switch (prev_layer) {
6645         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
6646                 out_mpls_m =
6647                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
6648                                                  outer_first_mpls_over_udp);
6649                 out_mpls_v =
6650                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
6651                                                  outer_first_mpls_over_udp);
6652                 break;
6653         case MLX5_FLOW_LAYER_GRE:
6654                 out_mpls_m =
6655                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
6656                                                  outer_first_mpls_over_gre);
6657                 out_mpls_v =
6658                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
6659                                                  outer_first_mpls_over_gre);
6660                 break;
6661         default:
6662                 /* Inner MPLS not over GRE is not supported. */
6663                 if (!inner) {
6664                         out_mpls_m =
6665                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
6666                                                          misc2_m,
6667                                                          outer_first_mpls);
6668                         out_mpls_v =
6669                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
6670                                                          misc2_v,
6671                                                          outer_first_mpls);
6672                 }
6673                 break;
6674         }
6675         if (out_mpls_m && out_mpls_v) {
6676                 *out_mpls_m = *in_mpls_m;
6677                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
6678         }
6679 }
6680
6681 /**
6682  * Add metadata register item to matcher
6683  *
6684  * @param[in, out] matcher
6685  *   Flow matcher.
6686  * @param[in, out] key
6687  *   Flow matcher value.
6688  * @param[in] reg_type
6689  *   Type of device metadata register
6690  * @param[in] value
6691  *   Register value
6692  * @param[in] mask
6693  *   Register mask
6694  */
6695 static void
6696 flow_dv_match_meta_reg(void *matcher, void *key,
6697                        enum modify_reg reg_type,
6698                        uint32_t data, uint32_t mask)
6699 {
6700         void *misc2_m =
6701                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
6702         void *misc2_v =
6703                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
6704         uint32_t temp;
6705
6706         data &= mask;
6707         switch (reg_type) {
6708         case REG_A:
6709                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
6710                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
6711                 break;
6712         case REG_B:
6713                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
6714                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
6715                 break;
6716         case REG_C_0:
6717                 /*
6718                  * The metadata register C0 field might be divided into
6719                  * source vport index and META item value, we should set
6720                  * this field according to specified mask, not as whole one.
6721                  */
6722                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
6723                 temp |= mask;
6724                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
6725                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
6726                 temp &= ~mask;
6727                 temp |= data;
6728                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
6729                 break;
6730         case REG_C_1:
6731                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
6732                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
6733                 break;
6734         case REG_C_2:
6735                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
6736                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
6737                 break;
6738         case REG_C_3:
6739                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
6740                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
6741                 break;
6742         case REG_C_4:
6743                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
6744                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
6745                 break;
6746         case REG_C_5:
6747                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
6748                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
6749                 break;
6750         case REG_C_6:
6751                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
6752                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
6753                 break;
6754         case REG_C_7:
6755                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
6756                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
6757                 break;
6758         default:
6759                 MLX5_ASSERT(false);
6760                 break;
6761         }
6762 }
6763
6764 /**
6765  * Add MARK item to matcher
6766  *
6767  * @param[in] dev
6768  *   The device to configure through.
6769  * @param[in, out] matcher
6770  *   Flow matcher.
6771  * @param[in, out] key
6772  *   Flow matcher value.
6773  * @param[in] item
6774  *   Flow pattern to translate.
6775  */
6776 static void
6777 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
6778                             void *matcher, void *key,
6779                             const struct rte_flow_item *item)
6780 {
6781         struct mlx5_priv *priv = dev->data->dev_private;
6782         const struct rte_flow_item_mark *mark;
6783         uint32_t value;
6784         uint32_t mask;
6785
6786         mark = item->mask ? (const void *)item->mask :
6787                             &rte_flow_item_mark_mask;
6788         mask = mark->id & priv->sh->dv_mark_mask;
6789         mark = (const void *)item->spec;
6790         MLX5_ASSERT(mark);
6791         value = mark->id & priv->sh->dv_mark_mask & mask;
6792         if (mask) {
6793                 enum modify_reg reg;
6794
6795                 /* Get the metadata register index for the mark. */
6796                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
6797                 MLX5_ASSERT(reg > 0);
6798                 if (reg == REG_C_0) {
6799                         struct mlx5_priv *priv = dev->data->dev_private;
6800                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6801                         uint32_t shl_c0 = rte_bsf32(msk_c0);
6802
6803                         mask &= msk_c0;
6804                         mask <<= shl_c0;
6805                         value <<= shl_c0;
6806                 }
6807                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
6808         }
6809 }
6810
6811 /**
6812  * Add META item to matcher
6813  *
6814  * @param[in] dev
6815  *   The devich to configure through.
6816  * @param[in, out] matcher
6817  *   Flow matcher.
6818  * @param[in, out] key
6819  *   Flow matcher value.
6820  * @param[in] attr
6821  *   Attributes of flow that includes this item.
6822  * @param[in] item
6823  *   Flow pattern to translate.
6824  */
6825 static void
6826 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
6827                             void *matcher, void *key,
6828                             const struct rte_flow_attr *attr,
6829                             const struct rte_flow_item *item)
6830 {
6831         const struct rte_flow_item_meta *meta_m;
6832         const struct rte_flow_item_meta *meta_v;
6833
6834         meta_m = (const void *)item->mask;
6835         if (!meta_m)
6836                 meta_m = &rte_flow_item_meta_mask;
6837         meta_v = (const void *)item->spec;
6838         if (meta_v) {
6839                 int reg;
6840                 uint32_t value = meta_v->data;
6841                 uint32_t mask = meta_m->data;
6842
6843                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
6844                 if (reg < 0)
6845                         return;
6846                 /*
6847                  * In datapath code there is no endianness
6848                  * coversions for perfromance reasons, all
6849                  * pattern conversions are done in rte_flow.
6850                  */
6851                 value = rte_cpu_to_be_32(value);
6852                 mask = rte_cpu_to_be_32(mask);
6853                 if (reg == REG_C_0) {
6854                         struct mlx5_priv *priv = dev->data->dev_private;
6855                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6856                         uint32_t shl_c0 = rte_bsf32(msk_c0);
6857 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6858                         uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
6859
6860                         value >>= shr_c0;
6861                         mask >>= shr_c0;
6862 #endif
6863                         value <<= shl_c0;
6864                         mask <<= shl_c0;
6865                         MLX5_ASSERT(msk_c0);
6866                         MLX5_ASSERT(!(~msk_c0 & mask));
6867                 }
6868                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
6869         }
6870 }
6871
6872 /**
6873  * Add vport metadata Reg C0 item to matcher
6874  *
6875  * @param[in, out] matcher
6876  *   Flow matcher.
6877  * @param[in, out] key
6878  *   Flow matcher value.
6879  * @param[in] reg
6880  *   Flow pattern to translate.
6881  */
6882 static void
6883 flow_dv_translate_item_meta_vport(void *matcher, void *key,
6884                                   uint32_t value, uint32_t mask)
6885 {
6886         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
6887 }
6888
6889 /**
6890  * Add tag item to matcher
6891  *
6892  * @param[in] dev
6893  *   The devich to configure through.
6894  * @param[in, out] matcher
6895  *   Flow matcher.
6896  * @param[in, out] key
6897  *   Flow matcher value.
6898  * @param[in] item
6899  *   Flow pattern to translate.
6900  */
6901 static void
6902 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
6903                                 void *matcher, void *key,
6904                                 const struct rte_flow_item *item)
6905 {
6906         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
6907         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
6908         uint32_t mask, value;
6909
6910         MLX5_ASSERT(tag_v);
6911         value = tag_v->data;
6912         mask = tag_m ? tag_m->data : UINT32_MAX;
6913         if (tag_v->id == REG_C_0) {
6914                 struct mlx5_priv *priv = dev->data->dev_private;
6915                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6916                 uint32_t shl_c0 = rte_bsf32(msk_c0);
6917
6918                 mask &= msk_c0;
6919                 mask <<= shl_c0;
6920                 value <<= shl_c0;
6921         }
6922         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
6923 }
6924
6925 /**
6926  * Add TAG item to matcher
6927  *
6928  * @param[in] dev
6929  *   The devich to configure through.
6930  * @param[in, out] matcher
6931  *   Flow matcher.
6932  * @param[in, out] key
6933  *   Flow matcher value.
6934  * @param[in] item
6935  *   Flow pattern to translate.
6936  */
6937 static void
6938 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
6939                            void *matcher, void *key,
6940                            const struct rte_flow_item *item)
6941 {
6942         const struct rte_flow_item_tag *tag_v = item->spec;
6943         const struct rte_flow_item_tag *tag_m = item->mask;
6944         enum modify_reg reg;
6945
6946         MLX5_ASSERT(tag_v);
6947         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
6948         /* Get the metadata register index for the tag. */
6949         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
6950         MLX5_ASSERT(reg > 0);
6951         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
6952 }
6953
6954 /**
6955  * Add source vport match to the specified matcher.
6956  *
6957  * @param[in, out] matcher
6958  *   Flow matcher.
6959  * @param[in, out] key
6960  *   Flow matcher value.
6961  * @param[in] port
6962  *   Source vport value to match
6963  * @param[in] mask
6964  *   Mask
6965  */
6966 static void
6967 flow_dv_translate_item_source_vport(void *matcher, void *key,
6968                                     int16_t port, uint16_t mask)
6969 {
6970         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6971         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6972
6973         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
6974         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
6975 }
6976
6977 /**
6978  * Translate port-id item to eswitch match on  port-id.
6979  *
6980  * @param[in] dev
6981  *   The devich to configure through.
6982  * @param[in, out] matcher
6983  *   Flow matcher.
6984  * @param[in, out] key
6985  *   Flow matcher value.
6986  * @param[in] item
6987  *   Flow pattern to translate.
6988  *
6989  * @return
6990  *   0 on success, a negative errno value otherwise.
6991  */
6992 static int
6993 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
6994                                void *key, const struct rte_flow_item *item)
6995 {
6996         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
6997         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
6998         struct mlx5_priv *priv;
6999         uint16_t mask, id;
7000
7001         mask = pid_m ? pid_m->id : 0xffff;
7002         id = pid_v ? pid_v->id : dev->data->port_id;
7003         priv = mlx5_port_to_eswitch_info(id, item == NULL);
7004         if (!priv)
7005                 return -rte_errno;
7006         /* Translate to vport field or to metadata, depending on mode. */
7007         if (priv->vport_meta_mask)
7008                 flow_dv_translate_item_meta_vport(matcher, key,
7009                                                   priv->vport_meta_tag,
7010                                                   priv->vport_meta_mask);
7011         else
7012                 flow_dv_translate_item_source_vport(matcher, key,
7013                                                     priv->vport_id, mask);
7014         return 0;
7015 }
7016
7017 /**
7018  * Add ICMP6 item to matcher and to the value.
7019  *
7020  * @param[in, out] matcher
7021  *   Flow matcher.
7022  * @param[in, out] key
7023  *   Flow matcher value.
7024  * @param[in] item
7025  *   Flow pattern to translate.
7026  * @param[in] inner
7027  *   Item is inner pattern.
7028  */
7029 static void
7030 flow_dv_translate_item_icmp6(void *matcher, void *key,
7031                               const struct rte_flow_item *item,
7032                               int inner)
7033 {
7034         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
7035         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
7036         void *headers_m;
7037         void *headers_v;
7038         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7039                                      misc_parameters_3);
7040         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7041         if (inner) {
7042                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7043                                          inner_headers);
7044                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7045         } else {
7046                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7047                                          outer_headers);
7048                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7049         }
7050         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7051         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
7052         if (!icmp6_v)
7053                 return;
7054         if (!icmp6_m)
7055                 icmp6_m = &rte_flow_item_icmp6_mask;
7056         /*
7057          * Force flow only to match the non-fragmented IPv6 ICMPv6 packets.
7058          * If only the protocol is specified, no need to match the frag.
7059          */
7060         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
7061         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
7062         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
7063         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
7064                  icmp6_v->type & icmp6_m->type);
7065         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
7066         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
7067                  icmp6_v->code & icmp6_m->code);
7068 }
7069
7070 /**
7071  * Add ICMP item to matcher and to the value.
7072  *
7073  * @param[in, out] matcher
7074  *   Flow matcher.
7075  * @param[in, out] key
7076  *   Flow matcher value.
7077  * @param[in] item
7078  *   Flow pattern to translate.
7079  * @param[in] inner
7080  *   Item is inner pattern.
7081  */
7082 static void
7083 flow_dv_translate_item_icmp(void *matcher, void *key,
7084                             const struct rte_flow_item *item,
7085                             int inner)
7086 {
7087         const struct rte_flow_item_icmp *icmp_m = item->mask;
7088         const struct rte_flow_item_icmp *icmp_v = item->spec;
7089         void *headers_m;
7090         void *headers_v;
7091         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7092                                      misc_parameters_3);
7093         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7094         if (inner) {
7095                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7096                                          inner_headers);
7097                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7098         } else {
7099                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7100                                          outer_headers);
7101                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7102         }
7103         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7104         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
7105         if (!icmp_v)
7106                 return;
7107         if (!icmp_m)
7108                 icmp_m = &rte_flow_item_icmp_mask;
7109         /*
7110          * Force flow only to match the non-fragmented IPv4 ICMP packets.
7111          * If only the protocol is specified, no need to match the frag.
7112          */
7113         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
7114         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
7115         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
7116                  icmp_m->hdr.icmp_type);
7117         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
7118                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
7119         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
7120                  icmp_m->hdr.icmp_code);
7121         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
7122                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
7123 }
7124
7125 /**
7126  * Add GTP item to matcher and to the value.
7127  *
7128  * @param[in, out] matcher
7129  *   Flow matcher.
7130  * @param[in, out] key
7131  *   Flow matcher value.
7132  * @param[in] item
7133  *   Flow pattern to translate.
7134  * @param[in] inner
7135  *   Item is inner pattern.
7136  */
7137 static void
7138 flow_dv_translate_item_gtp(void *matcher, void *key,
7139                            const struct rte_flow_item *item, int inner)
7140 {
7141         const struct rte_flow_item_gtp *gtp_m = item->mask;
7142         const struct rte_flow_item_gtp *gtp_v = item->spec;
7143         void *headers_m;
7144         void *headers_v;
7145         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7146                                      misc_parameters_3);
7147         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7148         uint16_t dport = RTE_GTPU_UDP_PORT;
7149
7150         if (inner) {
7151                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7152                                          inner_headers);
7153                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7154         } else {
7155                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7156                                          outer_headers);
7157                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7158         }
7159         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7160                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7161                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7162         }
7163         if (!gtp_v)
7164                 return;
7165         if (!gtp_m)
7166                 gtp_m = &rte_flow_item_gtp_mask;
7167         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
7168                  gtp_m->v_pt_rsv_flags);
7169         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
7170                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
7171         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
7172         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
7173                  gtp_v->msg_type & gtp_m->msg_type);
7174         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
7175                  rte_be_to_cpu_32(gtp_m->teid));
7176         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
7177                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
7178 }
7179
7180 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
7181
7182 #define HEADER_IS_ZERO(match_criteria, headers)                              \
7183         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
7184                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
7185
7186 /**
7187  * Calculate flow matcher enable bitmap.
7188  *
7189  * @param match_criteria
7190  *   Pointer to flow matcher criteria.
7191  *
7192  * @return
7193  *   Bitmap of enabled fields.
7194  */
7195 static uint8_t
7196 flow_dv_matcher_enable(uint32_t *match_criteria)
7197 {
7198         uint8_t match_criteria_enable;
7199
7200         match_criteria_enable =
7201                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
7202                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
7203         match_criteria_enable |=
7204                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
7205                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
7206         match_criteria_enable |=
7207                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
7208                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
7209         match_criteria_enable |=
7210                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
7211                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
7212         match_criteria_enable |=
7213                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
7214                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
7215         return match_criteria_enable;
7216 }
7217
7218
7219 /**
7220  * Get a flow table.
7221  *
7222  * @param[in, out] dev
7223  *   Pointer to rte_eth_dev structure.
7224  * @param[in] table_id
7225  *   Table id to use.
7226  * @param[in] egress
7227  *   Direction of the table.
7228  * @param[in] transfer
7229  *   E-Switch or NIC flow.
7230  * @param[out] error
7231  *   pointer to error structure.
7232  *
7233  * @return
7234  *   Returns tables resource based on the index, NULL in case of failed.
7235  */
7236 static struct mlx5_flow_tbl_resource *
7237 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
7238                          uint32_t table_id, uint8_t egress,
7239                          uint8_t transfer,
7240                          struct rte_flow_error *error)
7241 {
7242         struct mlx5_priv *priv = dev->data->dev_private;
7243         struct mlx5_dev_ctx_shared *sh = priv->sh;
7244         struct mlx5_flow_tbl_resource *tbl;
7245         union mlx5_flow_tbl_key table_key = {
7246                 {
7247                         .table_id = table_id,
7248                         .reserved = 0,
7249                         .domain = !!transfer,
7250                         .direction = !!egress,
7251                 }
7252         };
7253         struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
7254                                                          table_key.v64);
7255         struct mlx5_flow_tbl_data_entry *tbl_data;
7256         uint32_t idx = 0;
7257         int ret;
7258         void *domain;
7259
7260         if (pos) {
7261                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
7262                                         entry);
7263                 tbl = &tbl_data->tbl;
7264                 rte_atomic32_inc(&tbl->refcnt);
7265                 return tbl;
7266         }
7267         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
7268         if (!tbl_data) {
7269                 rte_flow_error_set(error, ENOMEM,
7270                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7271                                    NULL,
7272                                    "cannot allocate flow table data entry");
7273                 return NULL;
7274         }
7275         tbl_data->idx = idx;
7276         tbl = &tbl_data->tbl;
7277         pos = &tbl_data->entry;
7278         if (transfer)
7279                 domain = sh->fdb_domain;
7280         else if (egress)
7281                 domain = sh->tx_domain;
7282         else
7283                 domain = sh->rx_domain;
7284         tbl->obj = mlx5_glue->dr_create_flow_tbl(domain, table_id);
7285         if (!tbl->obj) {
7286                 rte_flow_error_set(error, ENOMEM,
7287                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7288                                    NULL, "cannot create flow table object");
7289                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7290                 return NULL;
7291         }
7292         /*
7293          * No multi-threads now, but still better to initialize the reference
7294          * count before insert it into the hash list.
7295          */
7296         rte_atomic32_init(&tbl->refcnt);
7297         /* Jump action reference count is initialized here. */
7298         rte_atomic32_init(&tbl_data->jump.refcnt);
7299         pos->key = table_key.v64;
7300         ret = mlx5_hlist_insert(sh->flow_tbls, pos);
7301         if (ret < 0) {
7302                 rte_flow_error_set(error, -ret,
7303                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7304                                    "cannot insert flow table data entry");
7305                 mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
7306                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7307         }
7308         rte_atomic32_inc(&tbl->refcnt);
7309         return tbl;
7310 }
7311
7312 /**
7313  * Release a flow table.
7314  *
7315  * @param[in] dev
7316  *   Pointer to rte_eth_dev structure.
7317  * @param[in] tbl
7318  *   Table resource to be released.
7319  *
7320  * @return
7321  *   Returns 0 if table was released, else return 1;
7322  */
7323 static int
7324 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
7325                              struct mlx5_flow_tbl_resource *tbl)
7326 {
7327         struct mlx5_priv *priv = dev->data->dev_private;
7328         struct mlx5_dev_ctx_shared *sh = priv->sh;
7329         struct mlx5_flow_tbl_data_entry *tbl_data =
7330                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7331
7332         if (!tbl)
7333                 return 0;
7334         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
7335                 struct mlx5_hlist_entry *pos = &tbl_data->entry;
7336
7337                 mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
7338                 tbl->obj = NULL;
7339                 /* remove the entry from the hash list and free memory. */
7340                 mlx5_hlist_remove(sh->flow_tbls, pos);
7341                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
7342                                 tbl_data->idx);
7343                 return 0;
7344         }
7345         return 1;
7346 }
7347
7348 /**
7349  * Register the flow matcher.
7350  *
7351  * @param[in, out] dev
7352  *   Pointer to rte_eth_dev structure.
7353  * @param[in, out] matcher
7354  *   Pointer to flow matcher.
7355  * @param[in, out] key
7356  *   Pointer to flow table key.
7357  * @parm[in, out] dev_flow
7358  *   Pointer to the dev_flow.
7359  * @param[out] error
7360  *   pointer to error structure.
7361  *
7362  * @return
7363  *   0 on success otherwise -errno and errno is set.
7364  */
7365 static int
7366 flow_dv_matcher_register(struct rte_eth_dev *dev,
7367                          struct mlx5_flow_dv_matcher *matcher,
7368                          union mlx5_flow_tbl_key *key,
7369                          struct mlx5_flow *dev_flow,
7370                          struct rte_flow_error *error)
7371 {
7372         struct mlx5_priv *priv = dev->data->dev_private;
7373         struct mlx5_dev_ctx_shared *sh = priv->sh;
7374         struct mlx5_flow_dv_matcher *cache_matcher;
7375         struct mlx5dv_flow_matcher_attr dv_attr = {
7376                 .type = IBV_FLOW_ATTR_NORMAL,
7377                 .match_mask = (void *)&matcher->mask,
7378         };
7379         struct mlx5_flow_tbl_resource *tbl;
7380         struct mlx5_flow_tbl_data_entry *tbl_data;
7381
7382         tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
7383                                        key->domain, error);
7384         if (!tbl)
7385                 return -rte_errno;      /* No need to refill the error info */
7386         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7387         /* Lookup from cache. */
7388         LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
7389                 if (matcher->crc == cache_matcher->crc &&
7390                     matcher->priority == cache_matcher->priority &&
7391                     !memcmp((const void *)matcher->mask.buf,
7392                             (const void *)cache_matcher->mask.buf,
7393                             cache_matcher->mask.size)) {
7394                         DRV_LOG(DEBUG,
7395                                 "%s group %u priority %hd use %s "
7396                                 "matcher %p: refcnt %d++",
7397                                 key->domain ? "FDB" : "NIC", key->table_id,
7398                                 cache_matcher->priority,
7399                                 key->direction ? "tx" : "rx",
7400                                 (void *)cache_matcher,
7401                                 rte_atomic32_read(&cache_matcher->refcnt));
7402                         rte_atomic32_inc(&cache_matcher->refcnt);
7403                         dev_flow->handle->dvh.matcher = cache_matcher;
7404                         /* old matcher should not make the table ref++. */
7405                         flow_dv_tbl_resource_release(dev, tbl);
7406                         return 0;
7407                 }
7408         }
7409         /* Register new matcher. */
7410         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
7411         if (!cache_matcher) {
7412                 flow_dv_tbl_resource_release(dev, tbl);
7413                 return rte_flow_error_set(error, ENOMEM,
7414                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7415                                           "cannot allocate matcher memory");
7416         }
7417         *cache_matcher = *matcher;
7418         dv_attr.match_criteria_enable =
7419                 flow_dv_matcher_enable(cache_matcher->mask.buf);
7420         dv_attr.priority = matcher->priority;
7421         if (key->direction)
7422                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
7423         cache_matcher->matcher_object =
7424                 mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj);
7425         if (!cache_matcher->matcher_object) {
7426                 rte_free(cache_matcher);
7427 #ifdef HAVE_MLX5DV_DR
7428                 flow_dv_tbl_resource_release(dev, tbl);
7429 #endif
7430                 return rte_flow_error_set(error, ENOMEM,
7431                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7432                                           NULL, "cannot create matcher");
7433         }
7434         /* Save the table information */
7435         cache_matcher->tbl = tbl;
7436         rte_atomic32_init(&cache_matcher->refcnt);
7437         /* only matcher ref++, table ref++ already done above in get API. */
7438         rte_atomic32_inc(&cache_matcher->refcnt);
7439         LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
7440         dev_flow->handle->dvh.matcher = cache_matcher;
7441         DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
7442                 key->domain ? "FDB" : "NIC", key->table_id,
7443                 cache_matcher->priority,
7444                 key->direction ? "tx" : "rx", (void *)cache_matcher,
7445                 rte_atomic32_read(&cache_matcher->refcnt));
7446         return 0;
7447 }
7448
7449 /**
7450  * Find existing tag resource or create and register a new one.
7451  *
7452  * @param dev[in, out]
7453  *   Pointer to rte_eth_dev structure.
7454  * @param[in, out] tag_be24
7455  *   Tag value in big endian then R-shift 8.
7456  * @parm[in, out] dev_flow
7457  *   Pointer to the dev_flow.
7458  * @param[out] error
7459  *   pointer to error structure.
7460  *
7461  * @return
7462  *   0 on success otherwise -errno and errno is set.
7463  */
7464 static int
7465 flow_dv_tag_resource_register
7466                         (struct rte_eth_dev *dev,
7467                          uint32_t tag_be24,
7468                          struct mlx5_flow *dev_flow,
7469                          struct rte_flow_error *error)
7470 {
7471         struct mlx5_priv *priv = dev->data->dev_private;
7472         struct mlx5_dev_ctx_shared *sh = priv->sh;
7473         struct mlx5_flow_dv_tag_resource *cache_resource;
7474         struct mlx5_hlist_entry *entry;
7475
7476         /* Lookup a matching resource from cache. */
7477         entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
7478         if (entry) {
7479                 cache_resource = container_of
7480                         (entry, struct mlx5_flow_dv_tag_resource, entry);
7481                 rte_atomic32_inc(&cache_resource->refcnt);
7482                 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
7483                 dev_flow->dv.tag_resource = cache_resource;
7484                 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
7485                         (void *)cache_resource,
7486                         rte_atomic32_read(&cache_resource->refcnt));
7487                 return 0;
7488         }
7489         /* Register new resource. */
7490         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG],
7491                                        &dev_flow->handle->dvh.rix_tag);
7492         if (!cache_resource)
7493                 return rte_flow_error_set(error, ENOMEM,
7494                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7495                                           "cannot allocate resource memory");
7496         cache_resource->entry.key = (uint64_t)tag_be24;
7497         cache_resource->action = mlx5_glue->dv_create_flow_action_tag(tag_be24);
7498         if (!cache_resource->action) {
7499                 rte_free(cache_resource);
7500                 return rte_flow_error_set(error, ENOMEM,
7501                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7502                                           NULL, "cannot create action");
7503         }
7504         rte_atomic32_init(&cache_resource->refcnt);
7505         rte_atomic32_inc(&cache_resource->refcnt);
7506         if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
7507                 mlx5_glue->destroy_flow_action(cache_resource->action);
7508                 rte_free(cache_resource);
7509                 return rte_flow_error_set(error, EEXIST,
7510                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7511                                           NULL, "cannot insert tag");
7512         }
7513         dev_flow->dv.tag_resource = cache_resource;
7514         DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
7515                 (void *)cache_resource,
7516                 rte_atomic32_read(&cache_resource->refcnt));
7517         return 0;
7518 }
7519
7520 /**
7521  * Release the tag.
7522  *
7523  * @param dev
7524  *   Pointer to Ethernet device.
7525  * @param tag_idx
7526  *   Tag index.
7527  *
7528  * @return
7529  *   1 while a reference on it exists, 0 when freed.
7530  */
7531 static int
7532 flow_dv_tag_release(struct rte_eth_dev *dev,
7533                     uint32_t tag_idx)
7534 {
7535         struct mlx5_priv *priv = dev->data->dev_private;
7536         struct mlx5_dev_ctx_shared *sh = priv->sh;
7537         struct mlx5_flow_dv_tag_resource *tag;
7538
7539         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
7540         if (!tag)
7541                 return 0;
7542         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
7543                 dev->data->port_id, (void *)tag,
7544                 rte_atomic32_read(&tag->refcnt));
7545         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
7546                 claim_zero(mlx5_glue->destroy_flow_action(tag->action));
7547                 mlx5_hlist_remove(sh->tag_table, &tag->entry);
7548                 DRV_LOG(DEBUG, "port %u tag %p: removed",
7549                         dev->data->port_id, (void *)tag);
7550                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
7551                 return 0;
7552         }
7553         return 1;
7554 }
7555
7556 /**
7557  * Translate port ID action to vport.
7558  *
7559  * @param[in] dev
7560  *   Pointer to rte_eth_dev structure.
7561  * @param[in] action
7562  *   Pointer to the port ID action.
7563  * @param[out] dst_port_id
7564  *   The target port ID.
7565  * @param[out] error
7566  *   Pointer to the error structure.
7567  *
7568  * @return
7569  *   0 on success, a negative errno value otherwise and rte_errno is set.
7570  */
7571 static int
7572 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
7573                                  const struct rte_flow_action *action,
7574                                  uint32_t *dst_port_id,
7575                                  struct rte_flow_error *error)
7576 {
7577         uint32_t port;
7578         struct mlx5_priv *priv;
7579         const struct rte_flow_action_port_id *conf =
7580                         (const struct rte_flow_action_port_id *)action->conf;
7581
7582         port = conf->original ? dev->data->port_id : conf->id;
7583         priv = mlx5_port_to_eswitch_info(port, false);
7584         if (!priv)
7585                 return rte_flow_error_set(error, -rte_errno,
7586                                           RTE_FLOW_ERROR_TYPE_ACTION,
7587                                           NULL,
7588                                           "No eswitch info was found for port");
7589 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
7590         /*
7591          * This parameter is transferred to
7592          * mlx5dv_dr_action_create_dest_ib_port().
7593          */
7594         *dst_port_id = priv->dev_port;
7595 #else
7596         /*
7597          * Legacy mode, no LAG configurations is supported.
7598          * This parameter is transferred to
7599          * mlx5dv_dr_action_create_dest_vport().
7600          */
7601         *dst_port_id = priv->vport_id;
7602 #endif
7603         return 0;
7604 }
7605
7606 /**
7607  * Create a counter with aging configuration.
7608  *
7609  * @param[in] dev
7610  *   Pointer to rte_eth_dev structure.
7611  * @param[out] count
7612  *   Pointer to the counter action configuration.
7613  * @param[in] age
7614  *   Pointer to the aging action configuration.
7615  *
7616  * @return
7617  *   Index to flow counter on success, 0 otherwise.
7618  */
7619 static uint32_t
7620 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
7621                                 struct mlx5_flow *dev_flow,
7622                                 const struct rte_flow_action_count *count,
7623                                 const struct rte_flow_action_age *age)
7624 {
7625         uint32_t counter;
7626         struct mlx5_age_param *age_param;
7627
7628         counter = flow_dv_counter_alloc(dev,
7629                                 count ? count->shared : 0,
7630                                 count ? count->id : 0,
7631                                 dev_flow->dv.group, !!age);
7632         if (!counter || age == NULL)
7633                 return counter;
7634         age_param  = flow_dv_counter_idx_get_age(dev, counter);
7635         /*
7636          * The counter age accuracy may have a bit delay. Have 3/4
7637          * second bias on the timeount in order to let it age in time.
7638          */
7639         age_param->context = age->context ? age->context :
7640                 (void *)(uintptr_t)(dev_flow->flow_idx);
7641         /*
7642          * The counter age accuracy may have a bit delay. Have 3/4
7643          * second bias on the timeount in order to let it age in time.
7644          */
7645         age_param->timeout = age->timeout * 10 - MLX5_AGING_TIME_DELAY;
7646         /* Set expire time in unit of 0.1 sec. */
7647         age_param->port_id = dev->data->port_id;
7648         age_param->expire = age_param->timeout +
7649                         rte_rdtsc() / (rte_get_tsc_hz() / 10);
7650         rte_atomic16_set(&age_param->state, AGE_CANDIDATE);
7651         return counter;
7652 }
7653 /**
7654  * Add Tx queue matcher
7655  *
7656  * @param[in] dev
7657  *   Pointer to the dev struct.
7658  * @param[in, out] matcher
7659  *   Flow matcher.
7660  * @param[in, out] key
7661  *   Flow matcher value.
7662  * @param[in] item
7663  *   Flow pattern to translate.
7664  * @param[in] inner
7665  *   Item is inner pattern.
7666  */
7667 static void
7668 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
7669                                 void *matcher, void *key,
7670                                 const struct rte_flow_item *item)
7671 {
7672         const struct mlx5_rte_flow_item_tx_queue *queue_m;
7673         const struct mlx5_rte_flow_item_tx_queue *queue_v;
7674         void *misc_m =
7675                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7676         void *misc_v =
7677                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7678         struct mlx5_txq_ctrl *txq;
7679         uint32_t queue;
7680
7681
7682         queue_m = (const void *)item->mask;
7683         if (!queue_m)
7684                 return;
7685         queue_v = (const void *)item->spec;
7686         if (!queue_v)
7687                 return;
7688         txq = mlx5_txq_get(dev, queue_v->queue);
7689         if (!txq)
7690                 return;
7691         queue = txq->obj->sq->id;
7692         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
7693         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
7694                  queue & queue_m->queue);
7695         mlx5_txq_release(dev, queue_v->queue);
7696 }
7697
7698 /**
7699  * Set the hash fields according to the @p flow information.
7700  *
7701  * @param[in] dev_flow
7702  *   Pointer to the mlx5_flow.
7703  * @param[in] rss_desc
7704  *   Pointer to the mlx5_flow_rss_desc.
7705  */
7706 static void
7707 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
7708                        struct mlx5_flow_rss_desc *rss_desc)
7709 {
7710         uint64_t items = dev_flow->handle->layers;
7711         int rss_inner = 0;
7712         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
7713
7714         dev_flow->hash_fields = 0;
7715 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
7716         if (rss_desc->level >= 2) {
7717                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
7718                 rss_inner = 1;
7719         }
7720 #endif
7721         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
7722             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
7723                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
7724                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
7725                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
7726                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
7727                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
7728                         else
7729                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
7730                 }
7731         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
7732                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
7733                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
7734                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
7735                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
7736                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
7737                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
7738                         else
7739                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
7740                 }
7741         }
7742         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
7743             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
7744                 if (rss_types & ETH_RSS_UDP) {
7745                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
7746                                 dev_flow->hash_fields |=
7747                                                 IBV_RX_HASH_SRC_PORT_UDP;
7748                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
7749                                 dev_flow->hash_fields |=
7750                                                 IBV_RX_HASH_DST_PORT_UDP;
7751                         else
7752                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
7753                 }
7754         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
7755                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
7756                 if (rss_types & ETH_RSS_TCP) {
7757                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
7758                                 dev_flow->hash_fields |=
7759                                                 IBV_RX_HASH_SRC_PORT_TCP;
7760                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
7761                                 dev_flow->hash_fields |=
7762                                                 IBV_RX_HASH_DST_PORT_TCP;
7763                         else
7764                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
7765                 }
7766         }
7767 }
7768
7769 /**
7770  * Fill the flow with DV spec, lock free
7771  * (mutex should be acquired by caller).
7772  *
7773  * @param[in] dev
7774  *   Pointer to rte_eth_dev structure.
7775  * @param[in, out] dev_flow
7776  *   Pointer to the sub flow.
7777  * @param[in] attr
7778  *   Pointer to the flow attributes.
7779  * @param[in] items
7780  *   Pointer to the list of items.
7781  * @param[in] actions
7782  *   Pointer to the list of actions.
7783  * @param[out] error
7784  *   Pointer to the error structure.
7785  *
7786  * @return
7787  *   0 on success, a negative errno value otherwise and rte_errno is set.
7788  */
7789 static int
7790 __flow_dv_translate(struct rte_eth_dev *dev,
7791                     struct mlx5_flow *dev_flow,
7792                     const struct rte_flow_attr *attr,
7793                     const struct rte_flow_item items[],
7794                     const struct rte_flow_action actions[],
7795                     struct rte_flow_error *error)
7796 {
7797         struct mlx5_priv *priv = dev->data->dev_private;
7798         struct mlx5_dev_config *dev_conf = &priv->config;
7799         struct rte_flow *flow = dev_flow->flow;
7800         struct mlx5_flow_handle *handle = dev_flow->handle;
7801         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
7802                                               priv->rss_desc)
7803                                               [!!priv->flow_nested_idx];
7804         uint64_t item_flags = 0;
7805         uint64_t last_item = 0;
7806         uint64_t action_flags = 0;
7807         uint64_t priority = attr->priority;
7808         struct mlx5_flow_dv_matcher matcher = {
7809                 .mask = {
7810                         .size = sizeof(matcher.mask.buf),
7811                 },
7812         };
7813         int actions_n = 0;
7814         bool actions_end = false;
7815         union {
7816                 struct mlx5_flow_dv_modify_hdr_resource res;
7817                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
7818                             sizeof(struct mlx5_modification_cmd) *
7819                             (MLX5_MAX_MODIFY_NUM + 1)];
7820         } mhdr_dummy;
7821         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
7822         const struct rte_flow_action_count *count = NULL;
7823         const struct rte_flow_action_age *age = NULL;
7824         union flow_dv_attr flow_attr = { .attr = 0 };
7825         uint32_t tag_be;
7826         union mlx5_flow_tbl_key tbl_key;
7827         uint32_t modify_action_position = UINT32_MAX;
7828         void *match_mask = matcher.mask.buf;
7829         void *match_value = dev_flow->dv.value.buf;
7830         uint8_t next_protocol = 0xff;
7831         struct rte_vlan_hdr vlan = { 0 };
7832         uint32_t table;
7833         int ret = 0;
7834
7835         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
7836                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
7837         ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
7838                                        !!priv->fdb_def_rule, &table, error);
7839         if (ret)
7840                 return ret;
7841         dev_flow->dv.group = table;
7842         if (attr->transfer)
7843                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
7844         if (priority == MLX5_FLOW_PRIO_RSVD)
7845                 priority = dev_conf->flow_prio - 1;
7846         /* number of actions must be set to 0 in case of dirty stack. */
7847         mhdr_res->actions_num = 0;
7848         for (; !actions_end ; actions++) {
7849                 const struct rte_flow_action_queue *queue;
7850                 const struct rte_flow_action_rss *rss;
7851                 const struct rte_flow_action *action = actions;
7852                 const uint8_t *rss_key;
7853                 const struct rte_flow_action_jump *jump_data;
7854                 const struct rte_flow_action_meter *mtr;
7855                 struct mlx5_flow_tbl_resource *tbl;
7856                 uint32_t port_id = 0;
7857                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
7858                 int action_type = actions->type;
7859                 const struct rte_flow_action *found_action = NULL;
7860                 struct mlx5_flow_meter *fm = NULL;
7861
7862                 switch (action_type) {
7863                 case RTE_FLOW_ACTION_TYPE_VOID:
7864                         break;
7865                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
7866                         if (flow_dv_translate_action_port_id(dev, action,
7867                                                              &port_id, error))
7868                                 return -rte_errno;
7869                         port_id_resource.port_id = port_id;
7870                         MLX5_ASSERT(!handle->rix_port_id_action);
7871                         if (flow_dv_port_id_action_resource_register
7872                             (dev, &port_id_resource, dev_flow, error))
7873                                 return -rte_errno;
7874                         dev_flow->dv.actions[actions_n++] =
7875                                         dev_flow->dv.port_id_action->action;
7876                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
7877                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
7878                         break;
7879                 case RTE_FLOW_ACTION_TYPE_FLAG:
7880                         action_flags |= MLX5_FLOW_ACTION_FLAG;
7881                         dev_flow->handle->mark = 1;
7882                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7883                                 struct rte_flow_action_mark mark = {
7884                                         .id = MLX5_FLOW_MARK_DEFAULT,
7885                                 };
7886
7887                                 if (flow_dv_convert_action_mark(dev, &mark,
7888                                                                 mhdr_res,
7889                                                                 error))
7890                                         return -rte_errno;
7891                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
7892                                 break;
7893                         }
7894                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
7895                         /*
7896                          * Only one FLAG or MARK is supported per device flow
7897                          * right now. So the pointer to the tag resource must be
7898                          * zero before the register process.
7899                          */
7900                         MLX5_ASSERT(!handle->dvh.rix_tag);
7901                         if (flow_dv_tag_resource_register(dev, tag_be,
7902                                                           dev_flow, error))
7903                                 return -rte_errno;
7904                         MLX5_ASSERT(dev_flow->dv.tag_resource);
7905                         dev_flow->dv.actions[actions_n++] =
7906                                         dev_flow->dv.tag_resource->action;
7907                         break;
7908                 case RTE_FLOW_ACTION_TYPE_MARK:
7909                         action_flags |= MLX5_FLOW_ACTION_MARK;
7910                         dev_flow->handle->mark = 1;
7911                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7912                                 const struct rte_flow_action_mark *mark =
7913                                         (const struct rte_flow_action_mark *)
7914                                                 actions->conf;
7915
7916                                 if (flow_dv_convert_action_mark(dev, mark,
7917                                                                 mhdr_res,
7918                                                                 error))
7919                                         return -rte_errno;
7920                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
7921                                 break;
7922                         }
7923                         /* Fall-through */
7924                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
7925                         /* Legacy (non-extensive) MARK action. */
7926                         tag_be = mlx5_flow_mark_set
7927                               (((const struct rte_flow_action_mark *)
7928                                (actions->conf))->id);
7929                         MLX5_ASSERT(!handle->dvh.rix_tag);
7930                         if (flow_dv_tag_resource_register(dev, tag_be,
7931                                                           dev_flow, error))
7932                                 return -rte_errno;
7933                         MLX5_ASSERT(dev_flow->dv.tag_resource);
7934                         dev_flow->dv.actions[actions_n++] =
7935                                         dev_flow->dv.tag_resource->action;
7936                         break;
7937                 case RTE_FLOW_ACTION_TYPE_SET_META:
7938                         if (flow_dv_convert_action_set_meta
7939                                 (dev, mhdr_res, attr,
7940                                  (const struct rte_flow_action_set_meta *)
7941                                   actions->conf, error))
7942                                 return -rte_errno;
7943                         action_flags |= MLX5_FLOW_ACTION_SET_META;
7944                         break;
7945                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
7946                         if (flow_dv_convert_action_set_tag
7947                                 (dev, mhdr_res,
7948                                  (const struct rte_flow_action_set_tag *)
7949                                   actions->conf, error))
7950                                 return -rte_errno;
7951                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
7952                         break;
7953                 case RTE_FLOW_ACTION_TYPE_DROP:
7954                         action_flags |= MLX5_FLOW_ACTION_DROP;
7955                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
7956                         break;
7957                 case RTE_FLOW_ACTION_TYPE_QUEUE:
7958                         queue = actions->conf;
7959                         rss_desc->queue_num = 1;
7960                         rss_desc->queue[0] = queue->index;
7961                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
7962                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
7963                         break;
7964                 case RTE_FLOW_ACTION_TYPE_RSS:
7965                         rss = actions->conf;
7966                         memcpy(rss_desc->queue, rss->queue,
7967                                rss->queue_num * sizeof(uint16_t));
7968                         rss_desc->queue_num = rss->queue_num;
7969                         /* NULL RSS key indicates default RSS key. */
7970                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
7971                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
7972                         /*
7973                          * rss->level and rss.types should be set in advance
7974                          * when expanding items for RSS.
7975                          */
7976                         action_flags |= MLX5_FLOW_ACTION_RSS;
7977                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
7978                         break;
7979                 case RTE_FLOW_ACTION_TYPE_AGE:
7980                 case RTE_FLOW_ACTION_TYPE_COUNT:
7981                         if (!dev_conf->devx) {
7982                                 return rte_flow_error_set
7983                                               (error, ENOTSUP,
7984                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7985                                                NULL,
7986                                                "count action not supported");
7987                         }
7988                         /* Save information first, will apply later. */
7989                         if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT)
7990                                 count = action->conf;
7991                         else
7992                                 age = action->conf;
7993                         action_flags |= MLX5_FLOW_ACTION_COUNT;
7994                         break;
7995                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
7996                         dev_flow->dv.actions[actions_n++] =
7997                                                 priv->sh->pop_vlan_action;
7998                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
7999                         break;
8000                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
8001                         if (!(action_flags &
8002                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
8003                                 flow_dev_get_vlan_info_from_items(items, &vlan);
8004                         vlan.eth_proto = rte_be_to_cpu_16
8005                              ((((const struct rte_flow_action_of_push_vlan *)
8006                                                    actions->conf)->ethertype));
8007                         found_action = mlx5_flow_find_action
8008                                         (actions + 1,
8009                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
8010                         if (found_action)
8011                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
8012                         found_action = mlx5_flow_find_action
8013                                         (actions + 1,
8014                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
8015                         if (found_action)
8016                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
8017                         if (flow_dv_create_action_push_vlan
8018                                             (dev, attr, &vlan, dev_flow, error))
8019                                 return -rte_errno;
8020                         dev_flow->dv.actions[actions_n++] =
8021                                         dev_flow->dv.push_vlan_res->action;
8022                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
8023                         break;
8024                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
8025                         /* of_vlan_push action handled this action */
8026                         MLX5_ASSERT(action_flags &
8027                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
8028                         break;
8029                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
8030                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
8031                                 break;
8032                         flow_dev_get_vlan_info_from_items(items, &vlan);
8033                         mlx5_update_vlan_vid_pcp(actions, &vlan);
8034                         /* If no VLAN push - this is a modify header action */
8035                         if (flow_dv_convert_action_modify_vlan_vid
8036                                                 (mhdr_res, actions, error))
8037                                 return -rte_errno;
8038                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
8039                         break;
8040                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
8041                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
8042                         if (flow_dv_create_action_l2_encap(dev, actions,
8043                                                            dev_flow,
8044                                                            attr->transfer,
8045                                                            error))
8046                                 return -rte_errno;
8047                         dev_flow->dv.actions[actions_n++] =
8048                                         dev_flow->dv.encap_decap->verbs_action;
8049                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
8050                         break;
8051                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
8052                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
8053                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
8054                                                            attr->transfer,
8055                                                            error))
8056                                 return -rte_errno;
8057                         dev_flow->dv.actions[actions_n++] =
8058                                         dev_flow->dv.encap_decap->verbs_action;
8059                         action_flags |= MLX5_FLOW_ACTION_DECAP;
8060                         break;
8061                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
8062                         /* Handle encap with preceding decap. */
8063                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
8064                                 if (flow_dv_create_action_raw_encap
8065                                         (dev, actions, dev_flow, attr, error))
8066                                         return -rte_errno;
8067                                 dev_flow->dv.actions[actions_n++] =
8068                                         dev_flow->dv.encap_decap->verbs_action;
8069                         } else {
8070                                 /* Handle encap without preceding decap. */
8071                                 if (flow_dv_create_action_l2_encap
8072                                     (dev, actions, dev_flow, attr->transfer,
8073                                      error))
8074                                         return -rte_errno;
8075                                 dev_flow->dv.actions[actions_n++] =
8076                                         dev_flow->dv.encap_decap->verbs_action;
8077                         }
8078                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
8079                         break;
8080                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
8081                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
8082                                 ;
8083                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
8084                                 if (flow_dv_create_action_l2_decap
8085                                     (dev, dev_flow, attr->transfer, error))
8086                                         return -rte_errno;
8087                                 dev_flow->dv.actions[actions_n++] =
8088                                         dev_flow->dv.encap_decap->verbs_action;
8089                         }
8090                         /* If decap is followed by encap, handle it at encap. */
8091                         action_flags |= MLX5_FLOW_ACTION_DECAP;
8092                         break;
8093                 case RTE_FLOW_ACTION_TYPE_JUMP:
8094                         jump_data = action->conf;
8095                         ret = mlx5_flow_group_to_table(attr, dev_flow->external,
8096                                                        jump_data->group,
8097                                                        !!priv->fdb_def_rule,
8098                                                        &table, error);
8099                         if (ret)
8100                                 return ret;
8101                         tbl = flow_dv_tbl_resource_get(dev, table,
8102                                                        attr->egress,
8103                                                        attr->transfer, error);
8104                         if (!tbl)
8105                                 return rte_flow_error_set
8106                                                 (error, errno,
8107                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8108                                                  NULL,
8109                                                  "cannot create jump action.");
8110                         if (flow_dv_jump_tbl_resource_register
8111                             (dev, tbl, dev_flow, error)) {
8112                                 flow_dv_tbl_resource_release(dev, tbl);
8113                                 return rte_flow_error_set
8114                                                 (error, errno,
8115                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8116                                                  NULL,
8117                                                  "cannot create jump action.");
8118                         }
8119                         dev_flow->dv.actions[actions_n++] =
8120                                         dev_flow->dv.jump->action;
8121                         action_flags |= MLX5_FLOW_ACTION_JUMP;
8122                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
8123                         break;
8124                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
8125                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
8126                         if (flow_dv_convert_action_modify_mac
8127                                         (mhdr_res, actions, error))
8128                                 return -rte_errno;
8129                         action_flags |= actions->type ==
8130                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
8131                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
8132                                         MLX5_FLOW_ACTION_SET_MAC_DST;
8133                         break;
8134                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
8135                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
8136                         if (flow_dv_convert_action_modify_ipv4
8137                                         (mhdr_res, actions, error))
8138                                 return -rte_errno;
8139                         action_flags |= actions->type ==
8140                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
8141                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
8142                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
8143                         break;
8144                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
8145                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
8146                         if (flow_dv_convert_action_modify_ipv6
8147                                         (mhdr_res, actions, error))
8148                                 return -rte_errno;
8149                         action_flags |= actions->type ==
8150                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
8151                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
8152                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
8153                         break;
8154                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
8155                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
8156                         if (flow_dv_convert_action_modify_tp
8157                                         (mhdr_res, actions, items,
8158                                          &flow_attr, dev_flow, !!(action_flags &
8159                                          MLX5_FLOW_ACTION_DECAP), error))
8160                                 return -rte_errno;
8161                         action_flags |= actions->type ==
8162                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
8163                                         MLX5_FLOW_ACTION_SET_TP_SRC :
8164                                         MLX5_FLOW_ACTION_SET_TP_DST;
8165                         break;
8166                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
8167                         if (flow_dv_convert_action_modify_dec_ttl
8168                                         (mhdr_res, items, &flow_attr, dev_flow,
8169                                          !!(action_flags &
8170                                          MLX5_FLOW_ACTION_DECAP), error))
8171                                 return -rte_errno;
8172                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
8173                         break;
8174                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
8175                         if (flow_dv_convert_action_modify_ttl
8176                                         (mhdr_res, actions, items, &flow_attr,
8177                                          dev_flow, !!(action_flags &
8178                                          MLX5_FLOW_ACTION_DECAP), error))
8179                                 return -rte_errno;
8180                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
8181                         break;
8182                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
8183                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
8184                         if (flow_dv_convert_action_modify_tcp_seq
8185                                         (mhdr_res, actions, error))
8186                                 return -rte_errno;
8187                         action_flags |= actions->type ==
8188                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
8189                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
8190                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
8191                         break;
8192
8193                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
8194                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
8195                         if (flow_dv_convert_action_modify_tcp_ack
8196                                         (mhdr_res, actions, error))
8197                                 return -rte_errno;
8198                         action_flags |= actions->type ==
8199                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
8200                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
8201                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
8202                         break;
8203                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
8204                         if (flow_dv_convert_action_set_reg
8205                                         (mhdr_res, actions, error))
8206                                 return -rte_errno;
8207                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8208                         break;
8209                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
8210                         if (flow_dv_convert_action_copy_mreg
8211                                         (dev, mhdr_res, actions, error))
8212                                 return -rte_errno;
8213                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8214                         break;
8215                 case RTE_FLOW_ACTION_TYPE_METER:
8216                         mtr = actions->conf;
8217                         if (!flow->meter) {
8218                                 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
8219                                                             attr, error);
8220                                 if (!fm)
8221                                         return rte_flow_error_set(error,
8222                                                 rte_errno,
8223                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8224                                                 NULL,
8225                                                 "meter not found "
8226                                                 "or invalid parameters");
8227                                 flow->meter = fm->idx;
8228                         }
8229                         /* Set the meter action. */
8230                         if (!fm) {
8231                                 fm = mlx5_ipool_get(priv->sh->ipool
8232                                                 [MLX5_IPOOL_MTR], flow->meter);
8233                                 if (!fm)
8234                                         return rte_flow_error_set(error,
8235                                                 rte_errno,
8236                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8237                                                 NULL,
8238                                                 "meter not found "
8239                                                 "or invalid parameters");
8240                         }
8241                         dev_flow->dv.actions[actions_n++] =
8242                                 fm->mfts->meter_action;
8243                         action_flags |= MLX5_FLOW_ACTION_METER;
8244                         break;
8245                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
8246                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
8247                                                               actions, error))
8248                                 return -rte_errno;
8249                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
8250                         break;
8251                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
8252                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
8253                                                               actions, error))
8254                                 return -rte_errno;
8255                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
8256                         break;
8257                 case RTE_FLOW_ACTION_TYPE_END:
8258                         actions_end = true;
8259                         if (mhdr_res->actions_num) {
8260                                 /* create modify action if needed. */
8261                                 if (flow_dv_modify_hdr_resource_register
8262                                         (dev, mhdr_res, dev_flow, error))
8263                                         return -rte_errno;
8264                                 dev_flow->dv.actions[modify_action_position] =
8265                                         handle->dvh.modify_hdr->verbs_action;
8266                         }
8267                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
8268                                 flow->counter =
8269                                         flow_dv_translate_create_counter(dev,
8270                                                 dev_flow, count, age);
8271
8272                                 if (!flow->counter)
8273                                         return rte_flow_error_set
8274                                                 (error, rte_errno,
8275                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8276                                                 NULL,
8277                                                 "cannot create counter"
8278                                                 " object.");
8279                                 dev_flow->dv.actions[actions_n++] =
8280                                           (flow_dv_counter_get_by_idx(dev,
8281                                           flow->counter, NULL))->action;
8282                         }
8283                         break;
8284                 default:
8285                         break;
8286                 }
8287                 if (mhdr_res->actions_num &&
8288                     modify_action_position == UINT32_MAX)
8289                         modify_action_position = actions_n++;
8290         }
8291         dev_flow->dv.actions_n = actions_n;
8292         dev_flow->act_flags = action_flags;
8293         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
8294                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
8295                 int item_type = items->type;
8296
8297                 switch (item_type) {
8298                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
8299                         flow_dv_translate_item_port_id(dev, match_mask,
8300                                                        match_value, items);
8301                         last_item = MLX5_FLOW_ITEM_PORT_ID;
8302                         break;
8303                 case RTE_FLOW_ITEM_TYPE_ETH:
8304                         flow_dv_translate_item_eth(match_mask, match_value,
8305                                                    items, tunnel,
8306                                                    dev_flow->dv.group);
8307                         matcher.priority = MLX5_PRIORITY_MAP_L2;
8308                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
8309                                              MLX5_FLOW_LAYER_OUTER_L2;
8310                         break;
8311                 case RTE_FLOW_ITEM_TYPE_VLAN:
8312                         flow_dv_translate_item_vlan(dev_flow,
8313                                                     match_mask, match_value,
8314                                                     items, tunnel,
8315                                                     dev_flow->dv.group);
8316                         matcher.priority = MLX5_PRIORITY_MAP_L2;
8317                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
8318                                               MLX5_FLOW_LAYER_INNER_VLAN) :
8319                                              (MLX5_FLOW_LAYER_OUTER_L2 |
8320                                               MLX5_FLOW_LAYER_OUTER_VLAN);
8321                         break;
8322                 case RTE_FLOW_ITEM_TYPE_IPV4:
8323                         mlx5_flow_tunnel_ip_check(items, next_protocol,
8324                                                   &item_flags, &tunnel);
8325                         flow_dv_translate_item_ipv4(match_mask, match_value,
8326                                                     items, item_flags, tunnel,
8327                                                     dev_flow->dv.group);
8328                         matcher.priority = MLX5_PRIORITY_MAP_L3;
8329                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
8330                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
8331                         if (items->mask != NULL &&
8332                             ((const struct rte_flow_item_ipv4 *)
8333                              items->mask)->hdr.next_proto_id) {
8334                                 next_protocol =
8335                                         ((const struct rte_flow_item_ipv4 *)
8336                                          (items->spec))->hdr.next_proto_id;
8337                                 next_protocol &=
8338                                         ((const struct rte_flow_item_ipv4 *)
8339                                          (items->mask))->hdr.next_proto_id;
8340                         } else {
8341                                 /* Reset for inner layer. */
8342                                 next_protocol = 0xff;
8343                         }
8344                         break;
8345                 case RTE_FLOW_ITEM_TYPE_IPV6:
8346                         mlx5_flow_tunnel_ip_check(items, next_protocol,
8347                                                   &item_flags, &tunnel);
8348                         flow_dv_translate_item_ipv6(match_mask, match_value,
8349                                                     items, item_flags, tunnel,
8350                                                     dev_flow->dv.group);
8351                         matcher.priority = MLX5_PRIORITY_MAP_L3;
8352                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
8353                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
8354                         if (items->mask != NULL &&
8355                             ((const struct rte_flow_item_ipv6 *)
8356                              items->mask)->hdr.proto) {
8357                                 next_protocol =
8358                                         ((const struct rte_flow_item_ipv6 *)
8359                                          items->spec)->hdr.proto;
8360                                 next_protocol &=
8361                                         ((const struct rte_flow_item_ipv6 *)
8362                                          items->mask)->hdr.proto;
8363                         } else {
8364                                 /* Reset for inner layer. */
8365                                 next_protocol = 0xff;
8366                         }
8367                         break;
8368                 case RTE_FLOW_ITEM_TYPE_TCP:
8369                         flow_dv_translate_item_tcp(match_mask, match_value,
8370                                                    items, tunnel);
8371                         matcher.priority = MLX5_PRIORITY_MAP_L4;
8372                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
8373                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
8374                         break;
8375                 case RTE_FLOW_ITEM_TYPE_UDP:
8376                         flow_dv_translate_item_udp(match_mask, match_value,
8377                                                    items, tunnel);
8378                         matcher.priority = MLX5_PRIORITY_MAP_L4;
8379                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
8380                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
8381                         break;
8382                 case RTE_FLOW_ITEM_TYPE_GRE:
8383                         flow_dv_translate_item_gre(match_mask, match_value,
8384                                                    items, tunnel);
8385                         matcher.priority = rss_desc->level >= 2 ?
8386                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8387                         last_item = MLX5_FLOW_LAYER_GRE;
8388                         break;
8389                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
8390                         flow_dv_translate_item_gre_key(match_mask,
8391                                                        match_value, items);
8392                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
8393                         break;
8394                 case RTE_FLOW_ITEM_TYPE_NVGRE:
8395                         flow_dv_translate_item_nvgre(match_mask, match_value,
8396                                                      items, tunnel);
8397                         matcher.priority = rss_desc->level >= 2 ?
8398                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8399                         last_item = MLX5_FLOW_LAYER_GRE;
8400                         break;
8401                 case RTE_FLOW_ITEM_TYPE_VXLAN:
8402                         flow_dv_translate_item_vxlan(match_mask, match_value,
8403                                                      items, tunnel);
8404                         matcher.priority = rss_desc->level >= 2 ?
8405                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8406                         last_item = MLX5_FLOW_LAYER_VXLAN;
8407                         break;
8408                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
8409                         flow_dv_translate_item_vxlan_gpe(match_mask,
8410                                                          match_value, items,
8411                                                          tunnel);
8412                         matcher.priority = rss_desc->level >= 2 ?
8413                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8414                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
8415                         break;
8416                 case RTE_FLOW_ITEM_TYPE_GENEVE:
8417                         flow_dv_translate_item_geneve(match_mask, match_value,
8418                                                       items, tunnel);
8419                         matcher.priority = rss_desc->level >= 2 ?
8420                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8421                         last_item = MLX5_FLOW_LAYER_GENEVE;
8422                         break;
8423                 case RTE_FLOW_ITEM_TYPE_MPLS:
8424                         flow_dv_translate_item_mpls(match_mask, match_value,
8425                                                     items, last_item, tunnel);
8426                         matcher.priority = rss_desc->level >= 2 ?
8427                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8428                         last_item = MLX5_FLOW_LAYER_MPLS;
8429                         break;
8430                 case RTE_FLOW_ITEM_TYPE_MARK:
8431                         flow_dv_translate_item_mark(dev, match_mask,
8432                                                     match_value, items);
8433                         last_item = MLX5_FLOW_ITEM_MARK;
8434                         break;
8435                 case RTE_FLOW_ITEM_TYPE_META:
8436                         flow_dv_translate_item_meta(dev, match_mask,
8437                                                     match_value, attr, items);
8438                         last_item = MLX5_FLOW_ITEM_METADATA;
8439                         break;
8440                 case RTE_FLOW_ITEM_TYPE_ICMP:
8441                         flow_dv_translate_item_icmp(match_mask, match_value,
8442                                                     items, tunnel);
8443                         last_item = MLX5_FLOW_LAYER_ICMP;
8444                         break;
8445                 case RTE_FLOW_ITEM_TYPE_ICMP6:
8446                         flow_dv_translate_item_icmp6(match_mask, match_value,
8447                                                       items, tunnel);
8448                         last_item = MLX5_FLOW_LAYER_ICMP6;
8449                         break;
8450                 case RTE_FLOW_ITEM_TYPE_TAG:
8451                         flow_dv_translate_item_tag(dev, match_mask,
8452                                                    match_value, items);
8453                         last_item = MLX5_FLOW_ITEM_TAG;
8454                         break;
8455                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
8456                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
8457                                                         match_value, items);
8458                         last_item = MLX5_FLOW_ITEM_TAG;
8459                         break;
8460                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
8461                         flow_dv_translate_item_tx_queue(dev, match_mask,
8462                                                         match_value,
8463                                                         items);
8464                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
8465                         break;
8466                 case RTE_FLOW_ITEM_TYPE_GTP:
8467                         flow_dv_translate_item_gtp(match_mask, match_value,
8468                                                    items, tunnel);
8469                         matcher.priority = rss_desc->level >= 2 ?
8470                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8471                         last_item = MLX5_FLOW_LAYER_GTP;
8472                         break;
8473                 default:
8474                         break;
8475                 }
8476                 item_flags |= last_item;
8477         }
8478         /*
8479          * When E-Switch mode is enabled, we have two cases where we need to
8480          * set the source port manually.
8481          * The first one, is in case of Nic steering rule, and the second is
8482          * E-Switch rule where no port_id item was found. In both cases
8483          * the source port is set according the current port in use.
8484          */
8485         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
8486             (priv->representor || priv->master)) {
8487                 if (flow_dv_translate_item_port_id(dev, match_mask,
8488                                                    match_value, NULL))
8489                         return -rte_errno;
8490         }
8491 #ifdef RTE_LIBRTE_MLX5_DEBUG
8492         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
8493                                               dev_flow->dv.value.buf));
8494 #endif
8495         /*
8496          * Layers may be already initialized from prefix flow if this dev_flow
8497          * is the suffix flow.
8498          */
8499         handle->layers |= item_flags;
8500         if (action_flags & MLX5_FLOW_ACTION_RSS)
8501                 flow_dv_hashfields_set(dev_flow, rss_desc);
8502         /* Register matcher. */
8503         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
8504                                     matcher.mask.size);
8505         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
8506                                                      matcher.priority);
8507         /* reserved field no needs to be set to 0 here. */
8508         tbl_key.domain = attr->transfer;
8509         tbl_key.direction = attr->egress;
8510         tbl_key.table_id = dev_flow->dv.group;
8511         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
8512                 return -rte_errno;
8513         return 0;
8514 }
8515
8516 /**
8517  * Apply the flow to the NIC, lock free,
8518  * (mutex should be acquired by caller).
8519  *
8520  * @param[in] dev
8521  *   Pointer to the Ethernet device structure.
8522  * @param[in, out] flow
8523  *   Pointer to flow structure.
8524  * @param[out] error
8525  *   Pointer to error structure.
8526  *
8527  * @return
8528  *   0 on success, a negative errno value otherwise and rte_errno is set.
8529  */
8530 static int
8531 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
8532                 struct rte_flow_error *error)
8533 {
8534         struct mlx5_flow_dv_workspace *dv;
8535         struct mlx5_flow_handle *dh;
8536         struct mlx5_flow_handle_dv *dv_h;
8537         struct mlx5_flow *dev_flow;
8538         struct mlx5_priv *priv = dev->data->dev_private;
8539         uint32_t handle_idx;
8540         int n;
8541         int err;
8542         int idx;
8543
8544         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
8545                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
8546                 dv = &dev_flow->dv;
8547                 dh = dev_flow->handle;
8548                 dv_h = &dh->dvh;
8549                 n = dv->actions_n;
8550                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
8551                         if (dv->transfer) {
8552                                 dv->actions[n++] = priv->sh->esw_drop_action;
8553                         } else {
8554                                 struct mlx5_hrxq *drop_hrxq;
8555                                 drop_hrxq = mlx5_hrxq_drop_new(dev);
8556                                 if (!drop_hrxq) {
8557                                         rte_flow_error_set
8558                                                 (error, errno,
8559                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8560                                                  NULL,
8561                                                  "cannot get drop hash queue");
8562                                         goto error;
8563                                 }
8564                                 /*
8565                                  * Drop queues will be released by the specify
8566                                  * mlx5_hrxq_drop_release() function. Assign
8567                                  * the special index to hrxq to mark the queue
8568                                  * has been allocated.
8569                                  */
8570                                 dh->rix_hrxq = UINT32_MAX;
8571                                 dv->actions[n++] = drop_hrxq->action;
8572                         }
8573                 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
8574                         struct mlx5_hrxq *hrxq;
8575                         uint32_t hrxq_idx;
8576                         struct mlx5_flow_rss_desc *rss_desc =
8577                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
8578                                 [!!priv->flow_nested_idx];
8579
8580                         MLX5_ASSERT(rss_desc->queue_num);
8581                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
8582                                                  MLX5_RSS_HASH_KEY_LEN,
8583                                                  dev_flow->hash_fields,
8584                                                  rss_desc->queue,
8585                                                  rss_desc->queue_num);
8586                         if (!hrxq_idx) {
8587                                 hrxq_idx = mlx5_hrxq_new
8588                                                 (dev, rss_desc->key,
8589                                                 MLX5_RSS_HASH_KEY_LEN,
8590                                                 dev_flow->hash_fields,
8591                                                 rss_desc->queue,
8592                                                 rss_desc->queue_num,
8593                                                 !!(dh->layers &
8594                                                 MLX5_FLOW_LAYER_TUNNEL));
8595                         }
8596                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
8597                                               hrxq_idx);
8598                         if (!hrxq) {
8599                                 rte_flow_error_set
8600                                         (error, rte_errno,
8601                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8602                                          "cannot get hash queue");
8603                                 goto error;
8604                         }
8605                         dh->rix_hrxq = hrxq_idx;
8606                         dv->actions[n++] = hrxq->action;
8607                 }
8608                 dh->ib_flow =
8609                         mlx5_glue->dv_create_flow(dv_h->matcher->matcher_object,
8610                                                   (void *)&dv->value, n,
8611                                                   dv->actions);
8612                 if (!dh->ib_flow) {
8613                         rte_flow_error_set(error, errno,
8614                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8615                                            NULL,
8616                                            "hardware refuses to create flow");
8617                         goto error;
8618                 }
8619                 if (priv->vmwa_context &&
8620                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
8621                         /*
8622                          * The rule contains the VLAN pattern.
8623                          * For VF we are going to create VLAN
8624                          * interface to make hypervisor set correct
8625                          * e-Switch vport context.
8626                          */
8627                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
8628                 }
8629         }
8630         return 0;
8631 error:
8632         err = rte_errno; /* Save rte_errno before cleanup. */
8633         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
8634                        handle_idx, dh, next) {
8635                 /* hrxq is union, don't clear it if the flag is not set. */
8636                 if (dh->rix_hrxq) {
8637                         if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
8638                                 mlx5_hrxq_drop_release(dev);
8639                                 dh->rix_hrxq = 0;
8640                         } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
8641                                 mlx5_hrxq_release(dev, dh->rix_hrxq);
8642                                 dh->rix_hrxq = 0;
8643                         }
8644                 }
8645                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
8646                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
8647         }
8648         rte_errno = err; /* Restore rte_errno. */
8649         return -rte_errno;
8650 }
8651
8652 /**
8653  * Release the flow matcher.
8654  *
8655  * @param dev
8656  *   Pointer to Ethernet device.
8657  * @param handle
8658  *   Pointer to mlx5_flow_handle.
8659  *
8660  * @return
8661  *   1 while a reference on it exists, 0 when freed.
8662  */
8663 static int
8664 flow_dv_matcher_release(struct rte_eth_dev *dev,
8665                         struct mlx5_flow_handle *handle)
8666 {
8667         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
8668
8669         MLX5_ASSERT(matcher->matcher_object);
8670         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
8671                 dev->data->port_id, (void *)matcher,
8672                 rte_atomic32_read(&matcher->refcnt));
8673         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
8674                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
8675                            (matcher->matcher_object));
8676                 LIST_REMOVE(matcher, next);
8677                 /* table ref-- in release interface. */
8678                 flow_dv_tbl_resource_release(dev, matcher->tbl);
8679                 rte_free(matcher);
8680                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
8681                         dev->data->port_id, (void *)matcher);
8682                 return 0;
8683         }
8684         return 1;
8685 }
8686
8687 /**
8688  * Release an encap/decap resource.
8689  *
8690  * @param dev
8691  *   Pointer to Ethernet device.
8692  * @param handle
8693  *   Pointer to mlx5_flow_handle.
8694  *
8695  * @return
8696  *   1 while a reference on it exists, 0 when freed.
8697  */
8698 static int
8699 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
8700                                      struct mlx5_flow_handle *handle)
8701 {
8702         struct mlx5_priv *priv = dev->data->dev_private;
8703         uint32_t idx = handle->dvh.rix_encap_decap;
8704         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
8705
8706         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
8707                          idx);
8708         if (!cache_resource)
8709                 return 0;
8710         MLX5_ASSERT(cache_resource->verbs_action);
8711         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
8712                 (void *)cache_resource,
8713                 rte_atomic32_read(&cache_resource->refcnt));
8714         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8715                 claim_zero(mlx5_glue->destroy_flow_action
8716                                 (cache_resource->verbs_action));
8717                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
8718                              &priv->sh->encaps_decaps, idx,
8719                              cache_resource, next);
8720                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
8721                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
8722                         (void *)cache_resource);
8723                 return 0;
8724         }
8725         return 1;
8726 }
8727
8728 /**
8729  * Release an jump to table action resource.
8730  *
8731  * @param dev
8732  *   Pointer to Ethernet device.
8733  * @param handle
8734  *   Pointer to mlx5_flow_handle.
8735  *
8736  * @return
8737  *   1 while a reference on it exists, 0 when freed.
8738  */
8739 static int
8740 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
8741                                   struct mlx5_flow_handle *handle)
8742 {
8743         struct mlx5_priv *priv = dev->data->dev_private;
8744         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
8745         struct mlx5_flow_tbl_data_entry *tbl_data;
8746
8747         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
8748                              handle->rix_jump);
8749         if (!tbl_data)
8750                 return 0;
8751         cache_resource = &tbl_data->jump;
8752         MLX5_ASSERT(cache_resource->action);
8753         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
8754                 (void *)cache_resource,
8755                 rte_atomic32_read(&cache_resource->refcnt));
8756         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8757                 claim_zero(mlx5_glue->destroy_flow_action
8758                                 (cache_resource->action));
8759                 /* jump action memory free is inside the table release. */
8760                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
8761                 DRV_LOG(DEBUG, "jump table resource %p: removed",
8762                         (void *)cache_resource);
8763                 return 0;
8764         }
8765         return 1;
8766 }
8767
8768 /**
8769  * Release a modify-header resource.
8770  *
8771  * @param handle
8772  *   Pointer to mlx5_flow_handle.
8773  *
8774  * @return
8775  *   1 while a reference on it exists, 0 when freed.
8776  */
8777 static int
8778 flow_dv_modify_hdr_resource_release(struct mlx5_flow_handle *handle)
8779 {
8780         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
8781                                                         handle->dvh.modify_hdr;
8782
8783         MLX5_ASSERT(cache_resource->verbs_action);
8784         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
8785                 (void *)cache_resource,
8786                 rte_atomic32_read(&cache_resource->refcnt));
8787         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8788                 claim_zero(mlx5_glue->destroy_flow_action
8789                                 (cache_resource->verbs_action));
8790                 LIST_REMOVE(cache_resource, next);
8791                 rte_free(cache_resource);
8792                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
8793                         (void *)cache_resource);
8794                 return 0;
8795         }
8796         return 1;
8797 }
8798
8799 /**
8800  * Release port ID action resource.
8801  *
8802  * @param dev
8803  *   Pointer to Ethernet device.
8804  * @param handle
8805  *   Pointer to mlx5_flow_handle.
8806  *
8807  * @return
8808  *   1 while a reference on it exists, 0 when freed.
8809  */
8810 static int
8811 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
8812                                         struct mlx5_flow_handle *handle)
8813 {
8814         struct mlx5_priv *priv = dev->data->dev_private;
8815         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
8816         uint32_t idx = handle->rix_port_id_action;
8817
8818         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
8819                                         idx);
8820         if (!cache_resource)
8821                 return 0;
8822         MLX5_ASSERT(cache_resource->action);
8823         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
8824                 (void *)cache_resource,
8825                 rte_atomic32_read(&cache_resource->refcnt));
8826         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8827                 claim_zero(mlx5_glue->destroy_flow_action
8828                                 (cache_resource->action));
8829                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
8830                              &priv->sh->port_id_action_list, idx,
8831                              cache_resource, next);
8832                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx);
8833                 DRV_LOG(DEBUG, "port id action resource %p: removed",
8834                         (void *)cache_resource);
8835                 return 0;
8836         }
8837         return 1;
8838 }
8839
8840 /**
8841  * Release push vlan action resource.
8842  *
8843  * @param dev
8844  *   Pointer to Ethernet device.
8845  * @param handle
8846  *   Pointer to mlx5_flow_handle.
8847  *
8848  * @return
8849  *   1 while a reference on it exists, 0 when freed.
8850  */
8851 static int
8852 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
8853                                           struct mlx5_flow_handle *handle)
8854 {
8855         struct mlx5_priv *priv = dev->data->dev_private;
8856         uint32_t idx = handle->dvh.rix_push_vlan;
8857         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
8858
8859         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
8860                                         idx);
8861         if (!cache_resource)
8862                 return 0;
8863         MLX5_ASSERT(cache_resource->action);
8864         DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
8865                 (void *)cache_resource,
8866                 rte_atomic32_read(&cache_resource->refcnt));
8867         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8868                 claim_zero(mlx5_glue->destroy_flow_action
8869                                 (cache_resource->action));
8870                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
8871                              &priv->sh->push_vlan_action_list, idx,
8872                              cache_resource, next);
8873                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
8874                 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
8875                         (void *)cache_resource);
8876                 return 0;
8877         }
8878         return 1;
8879 }
8880
8881 /**
8882  * Release the fate resource.
8883  *
8884  * @param dev
8885  *   Pointer to Ethernet device.
8886  * @param handle
8887  *   Pointer to mlx5_flow_handle.
8888  */
8889 static void
8890 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
8891                                struct mlx5_flow_handle *handle)
8892 {
8893         if (!handle->rix_fate)
8894                 return;
8895         if (handle->fate_action == MLX5_FLOW_FATE_DROP)
8896                 mlx5_hrxq_drop_release(dev);
8897         else if (handle->fate_action == MLX5_FLOW_FATE_QUEUE)
8898                 mlx5_hrxq_release(dev, handle->rix_hrxq);
8899         else if (handle->fate_action == MLX5_FLOW_FATE_JUMP)
8900                 flow_dv_jump_tbl_resource_release(dev, handle);
8901         else if (handle->fate_action == MLX5_FLOW_FATE_PORT_ID)
8902                 flow_dv_port_id_action_resource_release(dev, handle);
8903         else
8904                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
8905         handle->rix_fate = 0;
8906 }
8907
8908 /**
8909  * Remove the flow from the NIC but keeps it in memory.
8910  * Lock free, (mutex should be acquired by caller).
8911  *
8912  * @param[in] dev
8913  *   Pointer to Ethernet device.
8914  * @param[in, out] flow
8915  *   Pointer to flow structure.
8916  */
8917 static void
8918 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
8919 {
8920         struct mlx5_flow_handle *dh;
8921         uint32_t handle_idx;
8922         struct mlx5_priv *priv = dev->data->dev_private;
8923
8924         if (!flow)
8925                 return;
8926         handle_idx = flow->dev_handles;
8927         while (handle_idx) {
8928                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
8929                                     handle_idx);
8930                 if (!dh)
8931                         return;
8932                 if (dh->ib_flow) {
8933                         claim_zero(mlx5_glue->dv_destroy_flow(dh->ib_flow));
8934                         dh->ib_flow = NULL;
8935                 }
8936                 if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
8937                     dh->fate_action == MLX5_FLOW_FATE_QUEUE)
8938                         flow_dv_fate_resource_release(dev, dh);
8939                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
8940                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
8941                 handle_idx = dh->next.next;
8942         }
8943 }
8944
8945 /**
8946  * Remove the flow from the NIC and the memory.
8947  * Lock free, (mutex should be acquired by caller).
8948  *
8949  * @param[in] dev
8950  *   Pointer to the Ethernet device structure.
8951  * @param[in, out] flow
8952  *   Pointer to flow structure.
8953  */
8954 static void
8955 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
8956 {
8957         struct mlx5_flow_handle *dev_handle;
8958         struct mlx5_priv *priv = dev->data->dev_private;
8959
8960         if (!flow)
8961                 return;
8962         __flow_dv_remove(dev, flow);
8963         if (flow->counter) {
8964                 flow_dv_counter_release(dev, flow->counter);
8965                 flow->counter = 0;
8966         }
8967         if (flow->meter) {
8968                 struct mlx5_flow_meter *fm;
8969
8970                 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
8971                                     flow->meter);
8972                 if (fm)
8973                         mlx5_flow_meter_detach(fm);
8974                 flow->meter = 0;
8975         }
8976         while (flow->dev_handles) {
8977                 uint32_t tmp_idx = flow->dev_handles;
8978
8979                 dev_handle = mlx5_ipool_get(priv->sh->ipool
8980                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
8981                 if (!dev_handle)
8982                         return;
8983                 flow->dev_handles = dev_handle->next.next;
8984                 if (dev_handle->dvh.matcher)
8985                         flow_dv_matcher_release(dev, dev_handle);
8986                 if (dev_handle->dvh.rix_encap_decap)
8987                         flow_dv_encap_decap_resource_release(dev, dev_handle);
8988                 if (dev_handle->dvh.modify_hdr)
8989                         flow_dv_modify_hdr_resource_release(dev_handle);
8990                 if (dev_handle->dvh.rix_push_vlan)
8991                         flow_dv_push_vlan_action_resource_release(dev,
8992                                                                   dev_handle);
8993                 if (dev_handle->dvh.rix_tag)
8994                         flow_dv_tag_release(dev,
8995                                             dev_handle->dvh.rix_tag);
8996                 flow_dv_fate_resource_release(dev, dev_handle);
8997                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
8998                            tmp_idx);
8999         }
9000 }
9001
9002 /**
9003  * Query a dv flow  rule for its statistics via devx.
9004  *
9005  * @param[in] dev
9006  *   Pointer to Ethernet device.
9007  * @param[in] flow
9008  *   Pointer to the sub flow.
9009  * @param[out] data
9010  *   data retrieved by the query.
9011  * @param[out] error
9012  *   Perform verbose error reporting if not NULL.
9013  *
9014  * @return
9015  *   0 on success, a negative errno value otherwise and rte_errno is set.
9016  */
9017 static int
9018 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
9019                     void *data, struct rte_flow_error *error)
9020 {
9021         struct mlx5_priv *priv = dev->data->dev_private;
9022         struct rte_flow_query_count *qc = data;
9023
9024         if (!priv->config.devx)
9025                 return rte_flow_error_set(error, ENOTSUP,
9026                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9027                                           NULL,
9028                                           "counters are not supported");
9029         if (flow->counter) {
9030                 uint64_t pkts, bytes;
9031                 struct mlx5_flow_counter *cnt;
9032
9033                 cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
9034                                                  NULL);
9035                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
9036                                                &bytes);
9037
9038                 if (err)
9039                         return rte_flow_error_set(error, -err,
9040                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9041                                         NULL, "cannot read counters");
9042                 qc->hits_set = 1;
9043                 qc->bytes_set = 1;
9044                 qc->hits = pkts - cnt->hits;
9045                 qc->bytes = bytes - cnt->bytes;
9046                 if (qc->reset) {
9047                         cnt->hits = pkts;
9048                         cnt->bytes = bytes;
9049                 }
9050                 return 0;
9051         }
9052         return rte_flow_error_set(error, EINVAL,
9053                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9054                                   NULL,
9055                                   "counters are not available");
9056 }
9057
9058 /**
9059  * Query a flow.
9060  *
9061  * @see rte_flow_query()
9062  * @see rte_flow_ops
9063  */
9064 static int
9065 flow_dv_query(struct rte_eth_dev *dev,
9066               struct rte_flow *flow __rte_unused,
9067               const struct rte_flow_action *actions __rte_unused,
9068               void *data __rte_unused,
9069               struct rte_flow_error *error __rte_unused)
9070 {
9071         int ret = -EINVAL;
9072
9073         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
9074                 switch (actions->type) {
9075                 case RTE_FLOW_ACTION_TYPE_VOID:
9076                         break;
9077                 case RTE_FLOW_ACTION_TYPE_COUNT:
9078                         ret = flow_dv_query_count(dev, flow, data, error);
9079                         break;
9080                 default:
9081                         return rte_flow_error_set(error, ENOTSUP,
9082                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9083                                                   actions,
9084                                                   "action not supported");
9085                 }
9086         }
9087         return ret;
9088 }
9089
9090 /**
9091  * Destroy the meter table set.
9092  * Lock free, (mutex should be acquired by caller).
9093  *
9094  * @param[in] dev
9095  *   Pointer to Ethernet device.
9096  * @param[in] tbl
9097  *   Pointer to the meter table set.
9098  *
9099  * @return
9100  *   Always 0.
9101  */
9102 static int
9103 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
9104                         struct mlx5_meter_domains_infos *tbl)
9105 {
9106         struct mlx5_priv *priv = dev->data->dev_private;
9107         struct mlx5_meter_domains_infos *mtd =
9108                                 (struct mlx5_meter_domains_infos *)tbl;
9109
9110         if (!mtd || !priv->config.dv_flow_en)
9111                 return 0;
9112         if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
9113                 claim_zero(mlx5_glue->dv_destroy_flow
9114                           (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
9115         if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
9116                 claim_zero(mlx5_glue->dv_destroy_flow
9117                           (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
9118         if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
9119                 claim_zero(mlx5_glue->dv_destroy_flow
9120                           (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
9121         if (mtd->egress.color_matcher)
9122                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9123                           (mtd->egress.color_matcher));
9124         if (mtd->egress.any_matcher)
9125                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9126                           (mtd->egress.any_matcher));
9127         if (mtd->egress.tbl)
9128                 flow_dv_tbl_resource_release(dev, mtd->egress.tbl);
9129         if (mtd->egress.sfx_tbl)
9130                 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl);
9131         if (mtd->ingress.color_matcher)
9132                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9133                           (mtd->ingress.color_matcher));
9134         if (mtd->ingress.any_matcher)
9135                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9136                           (mtd->ingress.any_matcher));
9137         if (mtd->ingress.tbl)
9138                 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl);
9139         if (mtd->ingress.sfx_tbl)
9140                 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl);
9141         if (mtd->transfer.color_matcher)
9142                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9143                           (mtd->transfer.color_matcher));
9144         if (mtd->transfer.any_matcher)
9145                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9146                           (mtd->transfer.any_matcher));
9147         if (mtd->transfer.tbl)
9148                 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl);
9149         if (mtd->transfer.sfx_tbl)
9150                 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl);
9151         if (mtd->drop_actn)
9152                 claim_zero(mlx5_glue->destroy_flow_action(mtd->drop_actn));
9153         rte_free(mtd);
9154         return 0;
9155 }
9156
9157 /* Number of meter flow actions, count and jump or count and drop. */
9158 #define METER_ACTIONS 2
9159
9160 /**
9161  * Create specify domain meter table and suffix table.
9162  *
9163  * @param[in] dev
9164  *   Pointer to Ethernet device.
9165  * @param[in,out] mtb
9166  *   Pointer to DV meter table set.
9167  * @param[in] egress
9168  *   Table attribute.
9169  * @param[in] transfer
9170  *   Table attribute.
9171  * @param[in] color_reg_c_idx
9172  *   Reg C index for color match.
9173  *
9174  * @return
9175  *   0 on success, -1 otherwise and rte_errno is set.
9176  */
9177 static int
9178 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
9179                            struct mlx5_meter_domains_infos *mtb,
9180                            uint8_t egress, uint8_t transfer,
9181                            uint32_t color_reg_c_idx)
9182 {
9183         struct mlx5_priv *priv = dev->data->dev_private;
9184         struct mlx5_dev_ctx_shared *sh = priv->sh;
9185         struct mlx5_flow_dv_match_params mask = {
9186                 .size = sizeof(mask.buf),
9187         };
9188         struct mlx5_flow_dv_match_params value = {
9189                 .size = sizeof(value.buf),
9190         };
9191         struct mlx5dv_flow_matcher_attr dv_attr = {
9192                 .type = IBV_FLOW_ATTR_NORMAL,
9193                 .priority = 0,
9194                 .match_criteria_enable = 0,
9195                 .match_mask = (void *)&mask,
9196         };
9197         void *actions[METER_ACTIONS];
9198         struct mlx5_meter_domain_info *dtb;
9199         struct rte_flow_error error;
9200         int i = 0;
9201
9202         if (transfer)
9203                 dtb = &mtb->transfer;
9204         else if (egress)
9205                 dtb = &mtb->egress;
9206         else
9207                 dtb = &mtb->ingress;
9208         /* Create the meter table with METER level. */
9209         dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
9210                                             egress, transfer, &error);
9211         if (!dtb->tbl) {
9212                 DRV_LOG(ERR, "Failed to create meter policer table.");
9213                 return -1;
9214         }
9215         /* Create the meter suffix table with SUFFIX level. */
9216         dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
9217                                             MLX5_FLOW_TABLE_LEVEL_SUFFIX,
9218                                             egress, transfer, &error);
9219         if (!dtb->sfx_tbl) {
9220                 DRV_LOG(ERR, "Failed to create meter suffix table.");
9221                 return -1;
9222         }
9223         /* Create matchers, Any and Color. */
9224         dv_attr.priority = 3;
9225         dv_attr.match_criteria_enable = 0;
9226         dtb->any_matcher = mlx5_glue->dv_create_flow_matcher(sh->ctx,
9227                                                              &dv_attr,
9228                                                              dtb->tbl->obj);
9229         if (!dtb->any_matcher) {
9230                 DRV_LOG(ERR, "Failed to create meter"
9231                              " policer default matcher.");
9232                 goto error_exit;
9233         }
9234         dv_attr.priority = 0;
9235         dv_attr.match_criteria_enable =
9236                                 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
9237         flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
9238                                rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
9239         dtb->color_matcher = mlx5_glue->dv_create_flow_matcher(sh->ctx,
9240                                                                &dv_attr,
9241                                                                dtb->tbl->obj);
9242         if (!dtb->color_matcher) {
9243                 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
9244                 goto error_exit;
9245         }
9246         if (mtb->count_actns[RTE_MTR_DROPPED])
9247                 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
9248         actions[i++] = mtb->drop_actn;
9249         /* Default rule: lowest priority, match any, actions: drop. */
9250         dtb->policer_rules[RTE_MTR_DROPPED] =
9251                         mlx5_glue->dv_create_flow(dtb->any_matcher,
9252                                                  (void *)&value, i, actions);
9253         if (!dtb->policer_rules[RTE_MTR_DROPPED]) {
9254                 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
9255                 goto error_exit;
9256         }
9257         return 0;
9258 error_exit:
9259         return -1;
9260 }
9261
9262 /**
9263  * Create the needed meter and suffix tables.
9264  * Lock free, (mutex should be acquired by caller).
9265  *
9266  * @param[in] dev
9267  *   Pointer to Ethernet device.
9268  * @param[in] fm
9269  *   Pointer to the flow meter.
9270  *
9271  * @return
9272  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
9273  */
9274 static struct mlx5_meter_domains_infos *
9275 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
9276                        const struct mlx5_flow_meter *fm)
9277 {
9278         struct mlx5_priv *priv = dev->data->dev_private;
9279         struct mlx5_meter_domains_infos *mtb;
9280         int ret;
9281         int i;
9282
9283         if (!priv->mtr_en) {
9284                 rte_errno = ENOTSUP;
9285                 return NULL;
9286         }
9287         mtb = rte_calloc(__func__, 1, sizeof(*mtb), 0);
9288         if (!mtb) {
9289                 DRV_LOG(ERR, "Failed to allocate memory for meter.");
9290                 return NULL;
9291         }
9292         /* Create meter count actions */
9293         for (i = 0; i <= RTE_MTR_DROPPED; i++) {
9294                 struct mlx5_flow_counter *cnt;
9295                 if (!fm->policer_stats.cnt[i])
9296                         continue;
9297                 cnt = flow_dv_counter_get_by_idx(dev,
9298                       fm->policer_stats.cnt[i], NULL);
9299                 mtb->count_actns[i] = cnt->action;
9300         }
9301         /* Create drop action. */
9302         mtb->drop_actn = mlx5_glue->dr_create_flow_action_drop();
9303         if (!mtb->drop_actn) {
9304                 DRV_LOG(ERR, "Failed to create drop action.");
9305                 goto error_exit;
9306         }
9307         /* Egress meter table. */
9308         ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
9309         if (ret) {
9310                 DRV_LOG(ERR, "Failed to prepare egress meter table.");
9311                 goto error_exit;
9312         }
9313         /* Ingress meter table. */
9314         ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
9315         if (ret) {
9316                 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
9317                 goto error_exit;
9318         }
9319         /* FDB meter table. */
9320         if (priv->config.dv_esw_en) {
9321                 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
9322                                                  priv->mtr_color_reg);
9323                 if (ret) {
9324                         DRV_LOG(ERR, "Failed to prepare fdb meter table.");
9325                         goto error_exit;
9326                 }
9327         }
9328         return mtb;
9329 error_exit:
9330         flow_dv_destroy_mtr_tbl(dev, mtb);
9331         return NULL;
9332 }
9333
9334 /**
9335  * Destroy domain policer rule.
9336  *
9337  * @param[in] dt
9338  *   Pointer to domain table.
9339  */
9340 static void
9341 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
9342 {
9343         int i;
9344
9345         for (i = 0; i < RTE_MTR_DROPPED; i++) {
9346                 if (dt->policer_rules[i]) {
9347                         claim_zero(mlx5_glue->dv_destroy_flow
9348                                   (dt->policer_rules[i]));
9349                         dt->policer_rules[i] = NULL;
9350                 }
9351         }
9352         if (dt->jump_actn) {
9353                 claim_zero(mlx5_glue->destroy_flow_action(dt->jump_actn));
9354                 dt->jump_actn = NULL;
9355         }
9356 }
9357
9358 /**
9359  * Destroy policer rules.
9360  *
9361  * @param[in] dev
9362  *   Pointer to Ethernet device.
9363  * @param[in] fm
9364  *   Pointer to flow meter structure.
9365  * @param[in] attr
9366  *   Pointer to flow attributes.
9367  *
9368  * @return
9369  *   Always 0.
9370  */
9371 static int
9372 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
9373                               const struct mlx5_flow_meter *fm,
9374                               const struct rte_flow_attr *attr)
9375 {
9376         struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
9377
9378         if (!mtb)
9379                 return 0;
9380         if (attr->egress)
9381                 flow_dv_destroy_domain_policer_rule(&mtb->egress);
9382         if (attr->ingress)
9383                 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
9384         if (attr->transfer)
9385                 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
9386         return 0;
9387 }
9388
9389 /**
9390  * Create specify domain meter policer rule.
9391  *
9392  * @param[in] fm
9393  *   Pointer to flow meter structure.
9394  * @param[in] mtb
9395  *   Pointer to DV meter table set.
9396  * @param[in] mtr_reg_c
9397  *   Color match REG_C.
9398  *
9399  * @return
9400  *   0 on success, -1 otherwise.
9401  */
9402 static int
9403 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
9404                                     struct mlx5_meter_domain_info *dtb,
9405                                     uint8_t mtr_reg_c)
9406 {
9407         struct mlx5_flow_dv_match_params matcher = {
9408                 .size = sizeof(matcher.buf),
9409         };
9410         struct mlx5_flow_dv_match_params value = {
9411                 .size = sizeof(value.buf),
9412         };
9413         struct mlx5_meter_domains_infos *mtb = fm->mfts;
9414         void *actions[METER_ACTIONS];
9415         int i;
9416
9417         /* Create jump action. */
9418         if (!dtb->jump_actn)
9419                 dtb->jump_actn =
9420                         mlx5_glue->dr_create_flow_action_dest_flow_tbl
9421                                                         (dtb->sfx_tbl->obj);
9422         if (!dtb->jump_actn) {
9423                 DRV_LOG(ERR, "Failed to create policer jump action.");
9424                 goto error;
9425         }
9426         for (i = 0; i < RTE_MTR_DROPPED; i++) {
9427                 int j = 0;
9428
9429                 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
9430                                        rte_col_2_mlx5_col(i), UINT8_MAX);
9431                 if (mtb->count_actns[i])
9432                         actions[j++] = mtb->count_actns[i];
9433                 if (fm->action[i] == MTR_POLICER_ACTION_DROP)
9434                         actions[j++] = mtb->drop_actn;
9435                 else
9436                         actions[j++] = dtb->jump_actn;
9437                 dtb->policer_rules[i] =
9438                         mlx5_glue->dv_create_flow(dtb->color_matcher,
9439                                                  (void *)&value,
9440                                                   j, actions);
9441                 if (!dtb->policer_rules[i]) {
9442                         DRV_LOG(ERR, "Failed to create policer rule.");
9443                         goto error;
9444                 }
9445         }
9446         return 0;
9447 error:
9448         rte_errno = errno;
9449         return -1;
9450 }
9451
9452 /**
9453  * Create policer rules.
9454  *
9455  * @param[in] dev
9456  *   Pointer to Ethernet device.
9457  * @param[in] fm
9458  *   Pointer to flow meter structure.
9459  * @param[in] attr
9460  *   Pointer to flow attributes.
9461  *
9462  * @return
9463  *   0 on success, -1 otherwise.
9464  */
9465 static int
9466 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
9467                              struct mlx5_flow_meter *fm,
9468                              const struct rte_flow_attr *attr)
9469 {
9470         struct mlx5_priv *priv = dev->data->dev_private;
9471         struct mlx5_meter_domains_infos *mtb = fm->mfts;
9472         int ret;
9473
9474         if (attr->egress) {
9475                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
9476                                                 priv->mtr_color_reg);
9477                 if (ret) {
9478                         DRV_LOG(ERR, "Failed to create egress policer.");
9479                         goto error;
9480                 }
9481         }
9482         if (attr->ingress) {
9483                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
9484                                                 priv->mtr_color_reg);
9485                 if (ret) {
9486                         DRV_LOG(ERR, "Failed to create ingress policer.");
9487                         goto error;
9488                 }
9489         }
9490         if (attr->transfer) {
9491                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
9492                                                 priv->mtr_color_reg);
9493                 if (ret) {
9494                         DRV_LOG(ERR, "Failed to create transfer policer.");
9495                         goto error;
9496                 }
9497         }
9498         return 0;
9499 error:
9500         flow_dv_destroy_policer_rules(dev, fm, attr);
9501         return -1;
9502 }
9503
9504 /**
9505  * Query a devx counter.
9506  *
9507  * @param[in] dev
9508  *   Pointer to the Ethernet device structure.
9509  * @param[in] cnt
9510  *   Index to the flow counter.
9511  * @param[in] clear
9512  *   Set to clear the counter statistics.
9513  * @param[out] pkts
9514  *   The statistics value of packets.
9515  * @param[out] bytes
9516  *   The statistics value of bytes.
9517  *
9518  * @return
9519  *   0 on success, otherwise return -1.
9520  */
9521 static int
9522 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
9523                       uint64_t *pkts, uint64_t *bytes)
9524 {
9525         struct mlx5_priv *priv = dev->data->dev_private;
9526         struct mlx5_flow_counter *cnt;
9527         uint64_t inn_pkts, inn_bytes;
9528         int ret;
9529
9530         if (!priv->config.devx)
9531                 return -1;
9532
9533         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
9534         if (ret)
9535                 return -1;
9536         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
9537         *pkts = inn_pkts - cnt->hits;
9538         *bytes = inn_bytes - cnt->bytes;
9539         if (clear) {
9540                 cnt->hits = inn_pkts;
9541                 cnt->bytes = inn_bytes;
9542         }
9543         return 0;
9544 }
9545
9546 /**
9547  * Get aged-out flows.
9548  *
9549  * @param[in] dev
9550  *   Pointer to the Ethernet device structure.
9551  * @param[in] context
9552  *   The address of an array of pointers to the aged-out flows contexts.
9553  * @param[in] nb_contexts
9554  *   The length of context array pointers.
9555  * @param[out] error
9556  *   Perform verbose error reporting if not NULL. Initialized in case of
9557  *   error only.
9558  *
9559  * @return
9560  *   how many contexts get in success, otherwise negative errno value.
9561  *   if nb_contexts is 0, return the amount of all aged contexts.
9562  *   if nb_contexts is not 0 , return the amount of aged flows reported
9563  *   in the context array.
9564  * @note: only stub for now
9565  */
9566 static int
9567 flow_get_aged_flows(struct rte_eth_dev *dev,
9568                     void **context,
9569                     uint32_t nb_contexts,
9570                     struct rte_flow_error *error)
9571 {
9572         struct mlx5_priv *priv = dev->data->dev_private;
9573         struct mlx5_age_info *age_info;
9574         struct mlx5_age_param *age_param;
9575         struct mlx5_flow_counter *counter;
9576         int nb_flows = 0;
9577
9578         if (nb_contexts && !context)
9579                 return rte_flow_error_set(error, EINVAL,
9580                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9581                                           NULL,
9582                                           "Should assign at least one flow or"
9583                                           " context to get if nb_contexts != 0");
9584         age_info = GET_PORT_AGE_INFO(priv);
9585         rte_spinlock_lock(&age_info->aged_sl);
9586         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
9587                 nb_flows++;
9588                 if (nb_contexts) {
9589                         age_param = MLX5_CNT_TO_AGE(counter);
9590                         context[nb_flows - 1] = age_param->context;
9591                         if (!(--nb_contexts))
9592                                 break;
9593                 }
9594         }
9595         rte_spinlock_unlock(&age_info->aged_sl);
9596         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
9597         return nb_flows;
9598 }
9599
9600 /*
9601  * Mutex-protected thunk to lock-free  __flow_dv_translate().
9602  */
9603 static int
9604 flow_dv_translate(struct rte_eth_dev *dev,
9605                   struct mlx5_flow *dev_flow,
9606                   const struct rte_flow_attr *attr,
9607                   const struct rte_flow_item items[],
9608                   const struct rte_flow_action actions[],
9609                   struct rte_flow_error *error)
9610 {
9611         int ret;
9612
9613         flow_dv_shared_lock(dev);
9614         ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
9615         flow_dv_shared_unlock(dev);
9616         return ret;
9617 }
9618
9619 /*
9620  * Mutex-protected thunk to lock-free  __flow_dv_apply().
9621  */
9622 static int
9623 flow_dv_apply(struct rte_eth_dev *dev,
9624               struct rte_flow *flow,
9625               struct rte_flow_error *error)
9626 {
9627         int ret;
9628
9629         flow_dv_shared_lock(dev);
9630         ret = __flow_dv_apply(dev, flow, error);
9631         flow_dv_shared_unlock(dev);
9632         return ret;
9633 }
9634
9635 /*
9636  * Mutex-protected thunk to lock-free __flow_dv_remove().
9637  */
9638 static void
9639 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
9640 {
9641         flow_dv_shared_lock(dev);
9642         __flow_dv_remove(dev, flow);
9643         flow_dv_shared_unlock(dev);
9644 }
9645
9646 /*
9647  * Mutex-protected thunk to lock-free __flow_dv_destroy().
9648  */
9649 static void
9650 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
9651 {
9652         flow_dv_shared_lock(dev);
9653         __flow_dv_destroy(dev, flow);
9654         flow_dv_shared_unlock(dev);
9655 }
9656
9657 /*
9658  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
9659  */
9660 static uint32_t
9661 flow_dv_counter_allocate(struct rte_eth_dev *dev)
9662 {
9663         uint32_t cnt;
9664
9665         flow_dv_shared_lock(dev);
9666         cnt = flow_dv_counter_alloc(dev, 0, 0, 1, 0);
9667         flow_dv_shared_unlock(dev);
9668         return cnt;
9669 }
9670
9671 /*
9672  * Mutex-protected thunk to lock-free flow_dv_counter_release().
9673  */
9674 static void
9675 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
9676 {
9677         flow_dv_shared_lock(dev);
9678         flow_dv_counter_release(dev, cnt);
9679         flow_dv_shared_unlock(dev);
9680 }
9681
9682 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
9683         .validate = flow_dv_validate,
9684         .prepare = flow_dv_prepare,
9685         .translate = flow_dv_translate,
9686         .apply = flow_dv_apply,
9687         .remove = flow_dv_remove,
9688         .destroy = flow_dv_destroy,
9689         .query = flow_dv_query,
9690         .create_mtr_tbls = flow_dv_create_mtr_tbl,
9691         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
9692         .create_policer_rules = flow_dv_create_policer_rules,
9693         .destroy_policer_rules = flow_dv_destroy_policer_rules,
9694         .counter_alloc = flow_dv_counter_allocate,
9695         .counter_free = flow_dv_counter_free,
9696         .counter_query = flow_dv_counter_query,
9697         .get_aged_flows = flow_get_aged_flows,
9698 };
9699
9700 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */