drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_cycles.h>
  28 #include <rte_ip.h>
  29 #include <rte_gre.h>
  30 #include <rte_vxlan.h>
  31 #include <rte_gtp.h>
  32
  33 #include <mlx5_glue.h>
  34 #include <mlx5_devx_cmds.h>
  35 #include <mlx5_prm.h>
  36
  37 #include "mlx5_defs.h"
  38 #include "mlx5.h"
  39 #include "mlx5_common_os.h"
  40 #include "mlx5_flow.h"
  41 #include "mlx5_rxtx.h"
  42
  43 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  44
  45 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  46 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  47 #endif
  48
  49 #ifndef HAVE_MLX5DV_DR_ESWITCH
  50 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  51 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  52 #endif
  53 #endif
  54
  55 #ifndef HAVE_MLX5DV_DR
  56 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  57 #endif
  58
  59 /* VLAN header definitions */
  60 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  61 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  62 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  63 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  64 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  65
  66 union flow_dv_attr {
  67         struct {
  68                 uint32_t valid:1;
  69                 uint32_t ipv4:1;
  70                 uint32_t ipv6:1;
  71                 uint32_t tcp:1;
  72                 uint32_t udp:1;
  73                 uint32_t reserved:27;
  74         };
  75         uint32_t attr;
  76 };
  77
  78 static int
  79 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
  80                              struct mlx5_flow_tbl_resource *tbl);
  81
  82 static int
  83 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev);
  84
  85 /**
  86  * Initialize flow attributes structure according to flow items' types.
  87  *
  88  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  89  * mode. For tunnel mode, the items to be modified are the outermost ones.
  90  *
  91  * @param[in] item
  92  *   Pointer to item specification.
  93  * @param[out] attr
  94  *   Pointer to flow attributes structure.
  95  * @param[in] dev_flow
  96  *   Pointer to the sub flow.
  97  * @param[in] tunnel_decap
  98  *   Whether action is after tunnel decapsulation.
  99  */
 100 static void
 101 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 102                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 103 {
 104         uint64_t layers = dev_flow->handle->layers;
 105
 106         /*
 107          * If layers is already initialized, it means this dev_flow is the
 108          * suffix flow, the layers flags is set by the prefix flow. Need to
 109          * use the layer flags from prefix flow as the suffix flow may not
 110          * have the user defined items as the flow is split.
 111          */
 112         if (layers) {
 113                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 114                         attr->ipv4 = 1;
 115                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 116                         attr->ipv6 = 1;
 117                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 118                         attr->tcp = 1;
 119                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 120                         attr->udp = 1;
 121                 attr->valid = 1;
 122                 return;
 123         }
 124         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 125                 uint8_t next_protocol = 0xff;
 126                 switch (item->type) {
 127                 case RTE_FLOW_ITEM_TYPE_GRE:
 128                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 129                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 130                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 131                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 132                 case RTE_FLOW_ITEM_TYPE_MPLS:
 133                         if (tunnel_decap)
 134                                 attr->attr = 0;
 135                         break;
 136                 case RTE_FLOW_ITEM_TYPE_IPV4:
 137                         if (!attr->ipv6)
 138                                 attr->ipv4 = 1;
 139                         if (item->mask != NULL &&
 140                             ((const struct rte_flow_item_ipv4 *)
 141                             item->mask)->hdr.next_proto_id)
 142                                 next_protocol =
 143                                     ((const struct rte_flow_item_ipv4 *)
 144                                       (item->spec))->hdr.next_proto_id &
 145                                     ((const struct rte_flow_item_ipv4 *)
 146                                       (item->mask))->hdr.next_proto_id;
 147                         if ((next_protocol == IPPROTO_IPIP ||
 148                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 149                                 attr->attr = 0;
 150                         break;
 151                 case RTE_FLOW_ITEM_TYPE_IPV6:
 152                         if (!attr->ipv4)
 153                                 attr->ipv6 = 1;
 154                         if (item->mask != NULL &&
 155                             ((const struct rte_flow_item_ipv6 *)
 156                             item->mask)->hdr.proto)
 157                                 next_protocol =
 158                                     ((const struct rte_flow_item_ipv6 *)
 159                                       (item->spec))->hdr.proto &
 160                                     ((const struct rte_flow_item_ipv6 *)
 161                                       (item->mask))->hdr.proto;
 162                         if ((next_protocol == IPPROTO_IPIP ||
 163                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 164                                 attr->attr = 0;
 165                         break;
 166                 case RTE_FLOW_ITEM_TYPE_UDP:
 167                         if (!attr->tcp)
 168                                 attr->udp = 1;
 169                         break;
 170                 case RTE_FLOW_ITEM_TYPE_TCP:
 171                         if (!attr->udp)
 172                                 attr->tcp = 1;
 173                         break;
 174                 default:
 175                         break;
 176                 }
 177         }
 178         attr->valid = 1;
 179 }
 180
 181 /**
 182  * Convert rte_mtr_color to mlx5 color.
 183  *
 184  * @param[in] rcol
 185  *   rte_mtr_color.
 186  *
 187  * @return
 188  *   mlx5 color.
 189  */
 190 static int
 191 rte_col_2_mlx5_col(enum rte_color rcol)
 192 {
 193         switch (rcol) {
 194         case RTE_COLOR_GREEN:
 195                 return MLX5_FLOW_COLOR_GREEN;
 196         case RTE_COLOR_YELLOW:
 197                 return MLX5_FLOW_COLOR_YELLOW;
 198         case RTE_COLOR_RED:
 199                 return MLX5_FLOW_COLOR_RED;
 200         default:
 201                 break;
 202         }
 203         return MLX5_FLOW_COLOR_UNDEFINED;
 204 }
 205
 206 struct field_modify_info {
 207         uint32_t size; /* Size of field in protocol header, in bytes. */
 208         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 209         enum mlx5_modification_field id;
 210 };
 211
 212 struct field_modify_info modify_eth[] = {
 213         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 214         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 215         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 216         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 217         {0, 0, 0},
 218 };
 219
 220 struct field_modify_info modify_vlan_out_first_vid[] = {
 221         /* Size in bits !!! */
 222         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 223         {0, 0, 0},
 224 };
 225
 226 struct field_modify_info modify_ipv4[] = {
 227         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 228         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 229         {4, 12, MLX5_MODI_OUT_SIPV4},
 230         {4, 16, MLX5_MODI_OUT_DIPV4},
 231         {0, 0, 0},
 232 };
 233
 234 struct field_modify_info modify_ipv6[] = {
 235         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 236         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 237         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 238         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 239         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 240         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 241         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 242         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 243         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 244         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 245         {0, 0, 0},
 246 };
 247
 248 struct field_modify_info modify_udp[] = {
 249         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 250         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 251         {0, 0, 0},
 252 };
 253
 254 struct field_modify_info modify_tcp[] = {
 255         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 256         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 257         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 258         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 259         {0, 0, 0},
 260 };
 261
 262 static void
 263 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 264                           uint8_t next_protocol, uint64_t *item_flags,
 265                           int *tunnel)
 266 {
 267         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 268                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 269         if (next_protocol == IPPROTO_IPIP) {
 270                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 271                 *tunnel = 1;
 272         }
 273         if (next_protocol == IPPROTO_IPV6) {
 274                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 275                 *tunnel = 1;
 276         }
 277 }
 278
 279 /**
 280  * Acquire the synchronizing object to protect multithreaded access
 281  * to shared dv context. Lock occurs only if context is actually
 282  * shared, i.e. we have multiport IB device and representors are
 283  * created.
 284  *
 285  * @param[in] dev
 286  *   Pointer to the rte_eth_dev structure.
 287  */
 288 static void
 289 flow_dv_shared_lock(struct rte_eth_dev *dev)
 290 {
 291         struct mlx5_priv *priv = dev->data->dev_private;
 292         struct mlx5_dev_ctx_shared *sh = priv->sh;
 293
 294         if (sh->dv_refcnt > 1) {
 295                 int ret;
 296
 297                 ret = pthread_mutex_lock(&sh->dv_mutex);
 298                 MLX5_ASSERT(!ret);
 299                 (void)ret;
 300         }
 301 }
 302
 303 static void
 304 flow_dv_shared_unlock(struct rte_eth_dev *dev)
 305 {
 306         struct mlx5_priv *priv = dev->data->dev_private;
 307         struct mlx5_dev_ctx_shared *sh = priv->sh;
 308
 309         if (sh->dv_refcnt > 1) {
 310                 int ret;
 311
 312                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 313                 MLX5_ASSERT(!ret);
 314                 (void)ret;
 315         }
 316 }
 317
 318 /* Update VLAN's VID/PCP based on input rte_flow_action.
 319  *
 320  * @param[in] action
 321  *   Pointer to struct rte_flow_action.
 322  * @param[out] vlan
 323  *   Pointer to struct rte_vlan_hdr.
 324  */
 325 static void
 326 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 327                          struct rte_vlan_hdr *vlan)
 328 {
 329         uint16_t vlan_tci;
 330         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 331                 vlan_tci =
 332                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 333                                                action->conf)->vlan_pcp;
 334                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 335                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 336                 vlan->vlan_tci |= vlan_tci;
 337         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 338                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 339                 vlan->vlan_tci |= rte_be_to_cpu_16
 340                     (((const struct rte_flow_action_of_set_vlan_vid *)
 341                                              action->conf)->vlan_vid);
 342         }
 343 }
 344
 345 /**
 346  * Fetch 1, 2, 3 or 4 byte field from the byte array
 347  * and return as unsigned integer in host-endian format.
 348  *
 349  * @param[in] data
 350  *   Pointer to data array.
 351  * @param[in] size
 352  *   Size of field to extract.
 353  *
 354  * @return
 355  *   converted field in host endian format.
 356  */
 357 static inline uint32_t
 358 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 359 {
 360         uint32_t ret;
 361
 362         switch (size) {
 363         case 1:
 364                 ret = *data;
 365                 break;
 366         case 2:
 367                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 368                 break;
 369         case 3:
 370                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 371                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 372                 break;
 373         case 4:
 374                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 375                 break;
 376         default:
 377                 MLX5_ASSERT(false);
 378                 ret = 0;
 379                 break;
 380         }
 381         return ret;
 382 }
 383
 384 /**
 385  * Convert modify-header action to DV specification.
 386  *
 387  * Data length of each action is determined by provided field description
 388  * and the item mask. Data bit offset and width of each action is determined
 389  * by provided item mask.
 390  *
 391  * @param[in] item
 392  *   Pointer to item specification.
 393  * @param[in] field
 394  *   Pointer to field modification information.
 395  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 396  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 397  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 398  * @param[in] dcopy
 399  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 400  *   Negative offset value sets the same offset as source offset.
 401  *   size field is ignored, value is taken from source field.
 402  * @param[in,out] resource
 403  *   Pointer to the modify-header resource.
 404  * @param[in] type
 405  *   Type of modification.
 406  * @param[out] error
 407  *   Pointer to the error structure.
 408  *
 409  * @return
 410  *   0 on success, a negative errno value otherwise and rte_errno is set.
 411  */
 412 static int
 413 flow_dv_convert_modify_action(struct rte_flow_item *item,
 414                               struct field_modify_info *field,
 415                               struct field_modify_info *dcopy,
 416                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 417                               uint32_t type, struct rte_flow_error *error)
 418 {
 419         uint32_t i = resource->actions_num;
 420         struct mlx5_modification_cmd *actions = resource->actions;
 421
 422         /*
 423          * The item and mask are provided in big-endian format.
 424          * The fields should be presented as in big-endian format either.
 425          * Mask must be always present, it defines the actual field width.
 426          */
 427         MLX5_ASSERT(item->mask);
 428         MLX5_ASSERT(field->size);
 429         do {
 430                 unsigned int size_b;
 431                 unsigned int off_b;
 432                 uint32_t mask;
 433                 uint32_t data;
 434
 435                 if (i >= MLX5_MAX_MODIFY_NUM)
 436                         return rte_flow_error_set(error, EINVAL,
 437                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 438                                  "too many items to modify");
 439                 /* Fetch variable byte size mask from the array. */
 440                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 441                                            field->offset, field->size);
 442                 if (!mask) {
 443                         ++field;
 444                         continue;
 445                 }
 446                 /* Deduce actual data width in bits from mask value. */
 447                 off_b = rte_bsf32(mask);
 448                 size_b = sizeof(uint32_t) * CHAR_BIT -
 449                          off_b - __builtin_clz(mask);
 450                 MLX5_ASSERT(size_b);
 451                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 452                 actions[i] = (struct mlx5_modification_cmd) {
 453                         .action_type = type,
 454                         .field = field->id,
 455                         .offset = off_b,
 456                         .length = size_b,
 457                 };
 458                 /* Convert entire record to expected big-endian format. */
 459                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 460                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 461                         MLX5_ASSERT(dcopy);
 462                         actions[i].dst_field = dcopy->id;
 463                         actions[i].dst_offset =
 464                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 465                         /* Convert entire record to big-endian format. */
 466                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 467                 } else {
 468                         MLX5_ASSERT(item->spec);
 469                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 470                                                    field->offset, field->size);
 471                         /* Shift out the trailing masked bits from data. */
 472                         data = (data & mask) >> off_b;
 473                         actions[i].data1 = rte_cpu_to_be_32(data);
 474                 }
 475                 ++i;
 476                 ++field;
 477         } while (field->size);
 478         if (resource->actions_num == i)
 479                 return rte_flow_error_set(error, EINVAL,
 480                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 481                                           "invalid modification flow item");
 482         resource->actions_num = i;
 483         return 0;
 484 }
 485
 486 /**
 487  * Convert modify-header set IPv4 address action to DV specification.
 488  *
 489  * @param[in,out] resource
 490  *   Pointer to the modify-header resource.
 491  * @param[in] action
 492  *   Pointer to action specification.
 493  * @param[out] error
 494  *   Pointer to the error structure.
 495  *
 496  * @return
 497  *   0 on success, a negative errno value otherwise and rte_errno is set.
 498  */
 499 static int
 500 flow_dv_convert_action_modify_ipv4
 501                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 502                          const struct rte_flow_action *action,
 503                          struct rte_flow_error *error)
 504 {
 505         const struct rte_flow_action_set_ipv4 *conf =
 506                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 507         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 508         struct rte_flow_item_ipv4 ipv4;
 509         struct rte_flow_item_ipv4 ipv4_mask;
 510
 511         memset(&ipv4, 0, sizeof(ipv4));
 512         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 513         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 514                 ipv4.hdr.src_addr = conf->ipv4_addr;
 515                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 516         } else {
 517                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 518                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 519         }
 520         item.spec = &ipv4;
 521         item.mask = &ipv4_mask;
 522         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 523                                              MLX5_MODIFICATION_TYPE_SET, error);
 524 }
 525
 526 /**
 527  * Convert modify-header set IPv6 address action to DV specification.
 528  *
 529  * @param[in,out] resource
 530  *   Pointer to the modify-header resource.
 531  * @param[in] action
 532  *   Pointer to action specification.
 533  * @param[out] error
 534  *   Pointer to the error structure.
 535  *
 536  * @return
 537  *   0 on success, a negative errno value otherwise and rte_errno is set.
 538  */
 539 static int
 540 flow_dv_convert_action_modify_ipv6
 541                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 542                          const struct rte_flow_action *action,
 543                          struct rte_flow_error *error)
 544 {
 545         const struct rte_flow_action_set_ipv6 *conf =
 546                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 547         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 548         struct rte_flow_item_ipv6 ipv6;
 549         struct rte_flow_item_ipv6 ipv6_mask;
 550
 551         memset(&ipv6, 0, sizeof(ipv6));
 552         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 553         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 554                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 555                        sizeof(ipv6.hdr.src_addr));
 556                 memcpy(&ipv6_mask.hdr.src_addr,
 557                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 558                        sizeof(ipv6.hdr.src_addr));
 559         } else {
 560                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 561                        sizeof(ipv6.hdr.dst_addr));
 562                 memcpy(&ipv6_mask.hdr.dst_addr,
 563                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 564                        sizeof(ipv6.hdr.dst_addr));
 565         }
 566         item.spec = &ipv6;
 567         item.mask = &ipv6_mask;
 568         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 569                                              MLX5_MODIFICATION_TYPE_SET, error);
 570 }
 571
 572 /**
 573  * Convert modify-header set MAC address action to DV specification.
 574  *
 575  * @param[in,out] resource
 576  *   Pointer to the modify-header resource.
 577  * @param[in] action
 578  *   Pointer to action specification.
 579  * @param[out] error
 580  *   Pointer to the error structure.
 581  *
 582  * @return
 583  *   0 on success, a negative errno value otherwise and rte_errno is set.
 584  */
 585 static int
 586 flow_dv_convert_action_modify_mac
 587                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 588                          const struct rte_flow_action *action,
 589                          struct rte_flow_error *error)
 590 {
 591         const struct rte_flow_action_set_mac *conf =
 592                 (const struct rte_flow_action_set_mac *)(action->conf);
 593         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 594         struct rte_flow_item_eth eth;
 595         struct rte_flow_item_eth eth_mask;
 596
 597         memset(&eth, 0, sizeof(eth));
 598         memset(&eth_mask, 0, sizeof(eth_mask));
 599         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 600                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 601                        sizeof(eth.src.addr_bytes));
 602                 memcpy(&eth_mask.src.addr_bytes,
 603                        &rte_flow_item_eth_mask.src.addr_bytes,
 604                        sizeof(eth_mask.src.addr_bytes));
 605         } else {
 606                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 607                        sizeof(eth.dst.addr_bytes));
 608                 memcpy(&eth_mask.dst.addr_bytes,
 609                        &rte_flow_item_eth_mask.dst.addr_bytes,
 610                        sizeof(eth_mask.dst.addr_bytes));
 611         }
 612         item.spec = &eth;
 613         item.mask = &eth_mask;
 614         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 615                                              MLX5_MODIFICATION_TYPE_SET, error);
 616 }
 617
 618 /**
 619  * Convert modify-header set VLAN VID action to DV specification.
 620  *
 621  * @param[in,out] resource
 622  *   Pointer to the modify-header resource.
 623  * @param[in] action
 624  *   Pointer to action specification.
 625  * @param[out] error
 626  *   Pointer to the error structure.
 627  *
 628  * @return
 629  *   0 on success, a negative errno value otherwise and rte_errno is set.
 630  */
 631 static int
 632 flow_dv_convert_action_modify_vlan_vid
 633                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 634                          const struct rte_flow_action *action,
 635                          struct rte_flow_error *error)
 636 {
 637         const struct rte_flow_action_of_set_vlan_vid *conf =
 638                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 639         int i = resource->actions_num;
 640         struct mlx5_modification_cmd *actions = resource->actions;
 641         struct field_modify_info *field = modify_vlan_out_first_vid;
 642
 643         if (i >= MLX5_MAX_MODIFY_NUM)
 644                 return rte_flow_error_set(error, EINVAL,
 645                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 646                          "too many items to modify");
 647         actions[i] = (struct mlx5_modification_cmd) {
 648                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 649                 .field = field->id,
 650                 .length = field->size,
 651                 .offset = field->offset,
 652         };
 653         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 654         actions[i].data1 = conf->vlan_vid;
 655         actions[i].data1 = actions[i].data1 << 16;
 656         resource->actions_num = ++i;
 657         return 0;
 658 }
 659
 660 /**
 661  * Convert modify-header set TP action to DV specification.
 662  *
 663  * @param[in,out] resource
 664  *   Pointer to the modify-header resource.
 665  * @param[in] action
 666  *   Pointer to action specification.
 667  * @param[in] items
 668  *   Pointer to rte_flow_item objects list.
 669  * @param[in] attr
 670  *   Pointer to flow attributes structure.
 671  * @param[in] dev_flow
 672  *   Pointer to the sub flow.
 673  * @param[in] tunnel_decap
 674  *   Whether action is after tunnel decapsulation.
 675  * @param[out] error
 676  *   Pointer to the error structure.
 677  *
 678  * @return
 679  *   0 on success, a negative errno value otherwise and rte_errno is set.
 680  */
 681 static int
 682 flow_dv_convert_action_modify_tp
 683                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 684                          const struct rte_flow_action *action,
 685                          const struct rte_flow_item *items,
 686                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 687                          bool tunnel_decap, struct rte_flow_error *error)
 688 {
 689         const struct rte_flow_action_set_tp *conf =
 690                 (const struct rte_flow_action_set_tp *)(action->conf);
 691         struct rte_flow_item item;
 692         struct rte_flow_item_udp udp;
 693         struct rte_flow_item_udp udp_mask;
 694         struct rte_flow_item_tcp tcp;
 695         struct rte_flow_item_tcp tcp_mask;
 696         struct field_modify_info *field;
 697
 698         if (!attr->valid)
 699                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 700         if (attr->udp) {
 701                 memset(&udp, 0, sizeof(udp));
 702                 memset(&udp_mask, 0, sizeof(udp_mask));
 703                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 704                         udp.hdr.src_port = conf->port;
 705                         udp_mask.hdr.src_port =
 706                                         rte_flow_item_udp_mask.hdr.src_port;
 707                 } else {
 708                         udp.hdr.dst_port = conf->port;
 709                         udp_mask.hdr.dst_port =
 710                                         rte_flow_item_udp_mask.hdr.dst_port;
 711                 }
 712                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 713                 item.spec = &udp;
 714                 item.mask = &udp_mask;
 715                 field = modify_udp;
 716         } else {
 717                 MLX5_ASSERT(attr->tcp);
 718                 memset(&tcp, 0, sizeof(tcp));
 719                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 720                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 721                         tcp.hdr.src_port = conf->port;
 722                         tcp_mask.hdr.src_port =
 723                                         rte_flow_item_tcp_mask.hdr.src_port;
 724                 } else {
 725                         tcp.hdr.dst_port = conf->port;
 726                         tcp_mask.hdr.dst_port =
 727                                         rte_flow_item_tcp_mask.hdr.dst_port;
 728                 }
 729                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 730                 item.spec = &tcp;
 731                 item.mask = &tcp_mask;
 732                 field = modify_tcp;
 733         }
 734         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 735                                              MLX5_MODIFICATION_TYPE_SET, error);
 736 }
 737
 738 /**
 739  * Convert modify-header set TTL action to DV specification.
 740  *
 741  * @param[in,out] resource
 742  *   Pointer to the modify-header resource.
 743  * @param[in] action
 744  *   Pointer to action specification.
 745  * @param[in] items
 746  *   Pointer to rte_flow_item objects list.
 747  * @param[in] attr
 748  *   Pointer to flow attributes structure.
 749  * @param[in] dev_flow
 750  *   Pointer to the sub flow.
 751  * @param[in] tunnel_decap
 752  *   Whether action is after tunnel decapsulation.
 753  * @param[out] error
 754  *   Pointer to the error structure.
 755  *
 756  * @return
 757  *   0 on success, a negative errno value otherwise and rte_errno is set.
 758  */
 759 static int
 760 flow_dv_convert_action_modify_ttl
 761                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 762                          const struct rte_flow_action *action,
 763                          const struct rte_flow_item *items,
 764                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 765                          bool tunnel_decap, struct rte_flow_error *error)
 766 {
 767         const struct rte_flow_action_set_ttl *conf =
 768                 (const struct rte_flow_action_set_ttl *)(action->conf);
 769         struct rte_flow_item item;
 770         struct rte_flow_item_ipv4 ipv4;
 771         struct rte_flow_item_ipv4 ipv4_mask;
 772         struct rte_flow_item_ipv6 ipv6;
 773         struct rte_flow_item_ipv6 ipv6_mask;
 774         struct field_modify_info *field;
 775
 776         if (!attr->valid)
 777                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 778         if (attr->ipv4) {
 779                 memset(&ipv4, 0, sizeof(ipv4));
 780                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 781                 ipv4.hdr.time_to_live = conf->ttl_value;
 782                 ipv4_mask.hdr.time_to_live = 0xFF;
 783                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 784                 item.spec = &ipv4;
 785                 item.mask = &ipv4_mask;
 786                 field = modify_ipv4;
 787         } else {
 788                 MLX5_ASSERT(attr->ipv6);
 789                 memset(&ipv6, 0, sizeof(ipv6));
 790                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 791                 ipv6.hdr.hop_limits = conf->ttl_value;
 792                 ipv6_mask.hdr.hop_limits = 0xFF;
 793                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 794                 item.spec = &ipv6;
 795                 item.mask = &ipv6_mask;
 796                 field = modify_ipv6;
 797         }
 798         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 799                                              MLX5_MODIFICATION_TYPE_SET, error);
 800 }
 801
 802 /**
 803  * Convert modify-header decrement TTL action to DV specification.
 804  *
 805  * @param[in,out] resource
 806  *   Pointer to the modify-header resource.
 807  * @param[in] action
 808  *   Pointer to action specification.
 809  * @param[in] items
 810  *   Pointer to rte_flow_item objects list.
 811  * @param[in] attr
 812  *   Pointer to flow attributes structure.
 813  * @param[in] dev_flow
 814  *   Pointer to the sub flow.
 815  * @param[in] tunnel_decap
 816  *   Whether action is after tunnel decapsulation.
 817  * @param[out] error
 818  *   Pointer to the error structure.
 819  *
 820  * @return
 821  *   0 on success, a negative errno value otherwise and rte_errno is set.
 822  */
 823 static int
 824 flow_dv_convert_action_modify_dec_ttl
 825                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 826                          const struct rte_flow_item *items,
 827                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 828                          bool tunnel_decap, struct rte_flow_error *error)
 829 {
 830         struct rte_flow_item item;
 831         struct rte_flow_item_ipv4 ipv4;
 832         struct rte_flow_item_ipv4 ipv4_mask;
 833         struct rte_flow_item_ipv6 ipv6;
 834         struct rte_flow_item_ipv6 ipv6_mask;
 835         struct field_modify_info *field;
 836
 837         if (!attr->valid)
 838                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 839         if (attr->ipv4) {
 840                 memset(&ipv4, 0, sizeof(ipv4));
 841                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 842                 ipv4.hdr.time_to_live = 0xFF;
 843                 ipv4_mask.hdr.time_to_live = 0xFF;
 844                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 845                 item.spec = &ipv4;
 846                 item.mask = &ipv4_mask;
 847                 field = modify_ipv4;
 848         } else {
 849                 MLX5_ASSERT(attr->ipv6);
 850                 memset(&ipv6, 0, sizeof(ipv6));
 851                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 852                 ipv6.hdr.hop_limits = 0xFF;
 853                 ipv6_mask.hdr.hop_limits = 0xFF;
 854                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 855                 item.spec = &ipv6;
 856                 item.mask = &ipv6_mask;
 857                 field = modify_ipv6;
 858         }
 859         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 860                                              MLX5_MODIFICATION_TYPE_ADD, error);
 861 }
 862
 863 /**
 864  * Convert modify-header increment/decrement TCP Sequence number
 865  * to DV specification.
 866  *
 867  * @param[in,out] resource
 868  *   Pointer to the modify-header resource.
 869  * @param[in] action
 870  *   Pointer to action specification.
 871  * @param[out] error
 872  *   Pointer to the error structure.
 873  *
 874  * @return
 875  *   0 on success, a negative errno value otherwise and rte_errno is set.
 876  */
 877 static int
 878 flow_dv_convert_action_modify_tcp_seq
 879                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 880                          const struct rte_flow_action *action,
 881                          struct rte_flow_error *error)
 882 {
 883         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 884         uint64_t value = rte_be_to_cpu_32(*conf);
 885         struct rte_flow_item item;
 886         struct rte_flow_item_tcp tcp;
 887         struct rte_flow_item_tcp tcp_mask;
 888
 889         memset(&tcp, 0, sizeof(tcp));
 890         memset(&tcp_mask, 0, sizeof(tcp_mask));
 891         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 892                 /*
 893                  * The HW has no decrement operation, only increment operation.
 894                  * To simulate decrement X from Y using increment operation
 895                  * we need to add UINT32_MAX X times to Y.
 896                  * Each adding of UINT32_MAX decrements Y by 1.
 897                  */
 898                 value *= UINT32_MAX;
 899         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 900         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 901         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 902         item.spec = &tcp;
 903         item.mask = &tcp_mask;
 904         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 905                                              MLX5_MODIFICATION_TYPE_ADD, error);
 906 }
 907
 908 /**
 909  * Convert modify-header increment/decrement TCP Acknowledgment number
 910  * to DV specification.
 911  *
 912  * @param[in,out] resource
 913  *   Pointer to the modify-header resource.
 914  * @param[in] action
 915  *   Pointer to action specification.
 916  * @param[out] error
 917  *   Pointer to the error structure.
 918  *
 919  * @return
 920  *   0 on success, a negative errno value otherwise and rte_errno is set.
 921  */
 922 static int
 923 flow_dv_convert_action_modify_tcp_ack
 924                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 925                          const struct rte_flow_action *action,
 926                          struct rte_flow_error *error)
 927 {
 928         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 929         uint64_t value = rte_be_to_cpu_32(*conf);
 930         struct rte_flow_item item;
 931         struct rte_flow_item_tcp tcp;
 932         struct rte_flow_item_tcp tcp_mask;
 933
 934         memset(&tcp, 0, sizeof(tcp));
 935         memset(&tcp_mask, 0, sizeof(tcp_mask));
 936         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 937                 /*
 938                  * The HW has no decrement operation, only increment operation.
 939                  * To simulate decrement X from Y using increment operation
 940                  * we need to add UINT32_MAX X times to Y.
 941                  * Each adding of UINT32_MAX decrements Y by 1.
 942                  */
 943                 value *= UINT32_MAX;
 944         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 945         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 946         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 947         item.spec = &tcp;
 948         item.mask = &tcp_mask;
 949         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 950                                              MLX5_MODIFICATION_TYPE_ADD, error);
 951 }
 952
 953 static enum mlx5_modification_field reg_to_field[] = {
 954         [REG_NONE] = MLX5_MODI_OUT_NONE,
 955         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 956         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 957         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 958         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 959         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 960         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 961         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 962         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 963         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 964         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 965 };
 966
 967 /**
 968  * Convert register set to DV specification.
 969  *
 970  * @param[in,out] resource
 971  *   Pointer to the modify-header resource.
 972  * @param[in] action
 973  *   Pointer to action specification.
 974  * @param[out] error
 975  *   Pointer to the error structure.
 976  *
 977  * @return
 978  *   0 on success, a negative errno value otherwise and rte_errno is set.
 979  */
 980 static int
 981 flow_dv_convert_action_set_reg
 982                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 983                          const struct rte_flow_action *action,
 984                          struct rte_flow_error *error)
 985 {
 986         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 987         struct mlx5_modification_cmd *actions = resource->actions;
 988         uint32_t i = resource->actions_num;
 989
 990         if (i >= MLX5_MAX_MODIFY_NUM)
 991                 return rte_flow_error_set(error, EINVAL,
 992                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 993                                           "too many items to modify");
 994         MLX5_ASSERT(conf->id != REG_NONE);
 995         MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field));
 996         actions[i] = (struct mlx5_modification_cmd) {
 997                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 998                 .field = reg_to_field[conf->id],
 999         };
1000         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1001         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1002         ++i;
1003         resource->actions_num = i;
1004         return 0;
1005 }
1006
1007 /**
1008  * Convert SET_TAG action to DV specification.
1009  *
1010  * @param[in] dev
1011  *   Pointer to the rte_eth_dev structure.
1012  * @param[in,out] resource
1013  *   Pointer to the modify-header resource.
1014  * @param[in] conf
1015  *   Pointer to action specification.
1016  * @param[out] error
1017  *   Pointer to the error structure.
1018  *
1019  * @return
1020  *   0 on success, a negative errno value otherwise and rte_errno is set.
1021  */
1022 static int
1023 flow_dv_convert_action_set_tag
1024                         (struct rte_eth_dev *dev,
1025                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1026                          const struct rte_flow_action_set_tag *conf,
1027                          struct rte_flow_error *error)
1028 {
1029         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1030         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1031         struct rte_flow_item item = {
1032                 .spec = &data,
1033                 .mask = &mask,
1034         };
1035         struct field_modify_info reg_c_x[] = {
1036                 [1] = {0, 0, 0},
1037         };
1038         enum mlx5_modification_field reg_type;
1039         int ret;
1040
1041         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1042         if (ret < 0)
1043                 return ret;
1044         MLX5_ASSERT(ret != REG_NONE);
1045         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1046         reg_type = reg_to_field[ret];
1047         MLX5_ASSERT(reg_type > 0);
1048         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1049         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1050                                              MLX5_MODIFICATION_TYPE_SET, error);
1051 }
1052
1053 /**
1054  * Convert internal COPY_REG action to DV specification.
1055  *
1056  * @param[in] dev
1057  *   Pointer to the rte_eth_dev structure.
1058  * @param[in,out] res
1059  *   Pointer to the modify-header resource.
1060  * @param[in] action
1061  *   Pointer to action specification.
1062  * @param[out] error
1063  *   Pointer to the error structure.
1064  *
1065  * @return
1066  *   0 on success, a negative errno value otherwise and rte_errno is set.
1067  */
1068 static int
1069 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1070                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1071                                  const struct rte_flow_action *action,
1072                                  struct rte_flow_error *error)
1073 {
1074         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1075         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1076         struct rte_flow_item item = {
1077                 .spec = NULL,
1078                 .mask = &mask,
1079         };
1080         struct field_modify_info reg_src[] = {
1081                 {4, 0, reg_to_field[conf->src]},
1082                 {0, 0, 0},
1083         };
1084         struct field_modify_info reg_dst = {
1085                 .offset = 0,
1086                 .id = reg_to_field[conf->dst],
1087         };
1088         /* Adjust reg_c[0] usage according to reported mask. */
1089         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1090                 struct mlx5_priv *priv = dev->data->dev_private;
1091                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1092
1093                 MLX5_ASSERT(reg_c0);
1094                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1095                 if (conf->dst == REG_C_0) {
1096                         /* Copy to reg_c[0], within mask only. */
1097                         reg_dst.offset = rte_bsf32(reg_c0);
1098                         /*
1099                          * Mask is ignoring the enianness, because
1100                          * there is no conversion in datapath.
1101                          */
1102 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1103                         /* Copy from destination lower bits to reg_c[0]. */
1104                         mask = reg_c0 >> reg_dst.offset;
1105 #else
1106                         /* Copy from destination upper bits to reg_c[0]. */
1107                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1108                                           rte_fls_u32(reg_c0));
1109 #endif
1110                 } else {
1111                         mask = rte_cpu_to_be_32(reg_c0);
1112 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1113                         /* Copy from reg_c[0] to destination lower bits. */
1114                         reg_dst.offset = 0;
1115 #else
1116                         /* Copy from reg_c[0] to destination upper bits. */
1117                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1118                                          (rte_fls_u32(reg_c0) -
1119                                           rte_bsf32(reg_c0));
1120 #endif
1121                 }
1122         }
1123         return flow_dv_convert_modify_action(&item,
1124                                              reg_src, &reg_dst, res,
1125                                              MLX5_MODIFICATION_TYPE_COPY,
1126                                              error);
1127 }
1128
1129 /**
1130  * Convert MARK action to DV specification. This routine is used
1131  * in extensive metadata only and requires metadata register to be
1132  * handled. In legacy mode hardware tag resource is engaged.
1133  *
1134  * @param[in] dev
1135  *   Pointer to the rte_eth_dev structure.
1136  * @param[in] conf
1137  *   Pointer to MARK action specification.
1138  * @param[in,out] resource
1139  *   Pointer to the modify-header resource.
1140  * @param[out] error
1141  *   Pointer to the error structure.
1142  *
1143  * @return
1144  *   0 on success, a negative errno value otherwise and rte_errno is set.
1145  */
1146 static int
1147 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1148                             const struct rte_flow_action_mark *conf,
1149                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1150                             struct rte_flow_error *error)
1151 {
1152         struct mlx5_priv *priv = dev->data->dev_private;
1153         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1154                                            priv->sh->dv_mark_mask);
1155         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1156         struct rte_flow_item item = {
1157                 .spec = &data,
1158                 .mask = &mask,
1159         };
1160         struct field_modify_info reg_c_x[] = {
1161                 {4, 0, 0}, /* dynamic instead of MLX5_MODI_META_REG_C_1. */
1162                 {0, 0, 0},
1163         };
1164         int reg;
1165
1166         if (!mask)
1167                 return rte_flow_error_set(error, EINVAL,
1168                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1169                                           NULL, "zero mark action mask");
1170         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1171         if (reg < 0)
1172                 return reg;
1173         MLX5_ASSERT(reg > 0);
1174         if (reg == REG_C_0) {
1175                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1176                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1177
1178                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1179                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1180                 mask = rte_cpu_to_be_32(mask << shl_c0);
1181         }
1182         reg_c_x[0].id = reg_to_field[reg];
1183         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1184                                              MLX5_MODIFICATION_TYPE_SET, error);
1185 }
1186
1187 /**
1188  * Get metadata register index for specified steering domain.
1189  *
1190  * @param[in] dev
1191  *   Pointer to the rte_eth_dev structure.
1192  * @param[in] attr
1193  *   Attributes of flow to determine steering domain.
1194  * @param[out] error
1195  *   Pointer to the error structure.
1196  *
1197  * @return
1198  *   positive index on success, a negative errno value otherwise
1199  *   and rte_errno is set.
1200  */
1201 static enum modify_reg
1202 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1203                          const struct rte_flow_attr *attr,
1204                          struct rte_flow_error *error)
1205 {
1206         int reg =
1207                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1208                                           MLX5_METADATA_FDB :
1209                                             attr->egress ?
1210                                             MLX5_METADATA_TX :
1211                                             MLX5_METADATA_RX, 0, error);
1212         if (reg < 0)
1213                 return rte_flow_error_set(error,
1214                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1215                                           NULL, "unavailable "
1216                                           "metadata register");
1217         return reg;
1218 }
1219
1220 /**
1221  * Convert SET_META action to DV specification.
1222  *
1223  * @param[in] dev
1224  *   Pointer to the rte_eth_dev structure.
1225  * @param[in,out] resource
1226  *   Pointer to the modify-header resource.
1227  * @param[in] attr
1228  *   Attributes of flow that includes this item.
1229  * @param[in] conf
1230  *   Pointer to action specification.
1231  * @param[out] error
1232  *   Pointer to the error structure.
1233  *
1234  * @return
1235  *   0 on success, a negative errno value otherwise and rte_errno is set.
1236  */
1237 static int
1238 flow_dv_convert_action_set_meta
1239                         (struct rte_eth_dev *dev,
1240                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1241                          const struct rte_flow_attr *attr,
1242                          const struct rte_flow_action_set_meta *conf,
1243                          struct rte_flow_error *error)
1244 {
1245         uint32_t data = conf->data;
1246         uint32_t mask = conf->mask;
1247         struct rte_flow_item item = {
1248                 .spec = &data,
1249                 .mask = &mask,
1250         };
1251         struct field_modify_info reg_c_x[] = {
1252                 [1] = {0, 0, 0},
1253         };
1254         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1255
1256         if (reg < 0)
1257                 return reg;
1258         /*
1259          * In datapath code there is no endianness
1260          * coversions for perfromance reasons, all
1261          * pattern conversions are done in rte_flow.
1262          */
1263         if (reg == REG_C_0) {
1264                 struct mlx5_priv *priv = dev->data->dev_private;
1265                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1266                 uint32_t shl_c0;
1267
1268                 MLX5_ASSERT(msk_c0);
1269 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1270                 shl_c0 = rte_bsf32(msk_c0);
1271 #else
1272                 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1273 #endif
1274                 mask <<= shl_c0;
1275                 data <<= shl_c0;
1276                 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1277         }
1278         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1279         /* The routine expects parameters in memory as big-endian ones. */
1280         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1281                                              MLX5_MODIFICATION_TYPE_SET, error);
1282 }
1283
1284 /**
1285  * Convert modify-header set IPv4 DSCP action to DV specification.
1286  *
1287  * @param[in,out] resource
1288  *   Pointer to the modify-header resource.
1289  * @param[in] action
1290  *   Pointer to action specification.
1291  * @param[out] error
1292  *   Pointer to the error structure.
1293  *
1294  * @return
1295  *   0 on success, a negative errno value otherwise and rte_errno is set.
1296  */
1297 static int
1298 flow_dv_convert_action_modify_ipv4_dscp
1299                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1300                          const struct rte_flow_action *action,
1301                          struct rte_flow_error *error)
1302 {
1303         const struct rte_flow_action_set_dscp *conf =
1304                 (const struct rte_flow_action_set_dscp *)(action->conf);
1305         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1306         struct rte_flow_item_ipv4 ipv4;
1307         struct rte_flow_item_ipv4 ipv4_mask;
1308
1309         memset(&ipv4, 0, sizeof(ipv4));
1310         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1311         ipv4.hdr.type_of_service = conf->dscp;
1312         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1313         item.spec = &ipv4;
1314         item.mask = &ipv4_mask;
1315         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1316                                              MLX5_MODIFICATION_TYPE_SET, error);
1317 }
1318
1319 /**
1320  * Convert modify-header set IPv6 DSCP action to DV specification.
1321  *
1322  * @param[in,out] resource
1323  *   Pointer to the modify-header resource.
1324  * @param[in] action
1325  *   Pointer to action specification.
1326  * @param[out] error
1327  *   Pointer to the error structure.
1328  *
1329  * @return
1330  *   0 on success, a negative errno value otherwise and rte_errno is set.
1331  */
1332 static int
1333 flow_dv_convert_action_modify_ipv6_dscp
1334                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1335                          const struct rte_flow_action *action,
1336                          struct rte_flow_error *error)
1337 {
1338         const struct rte_flow_action_set_dscp *conf =
1339                 (const struct rte_flow_action_set_dscp *)(action->conf);
1340         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1341         struct rte_flow_item_ipv6 ipv6;
1342         struct rte_flow_item_ipv6 ipv6_mask;
1343
1344         memset(&ipv6, 0, sizeof(ipv6));
1345         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1346         /*
1347          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1348          * rdma-core only accept the DSCP bits byte aligned start from
1349          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1350          * bits in IPv6 case as rdma-core requires byte aligned value.
1351          */
1352         ipv6.hdr.vtc_flow = conf->dscp;
1353         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1354         item.spec = &ipv6;
1355         item.mask = &ipv6_mask;
1356         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1357                                              MLX5_MODIFICATION_TYPE_SET, error);
1358 }
1359
1360 /**
1361  * Validate MARK item.
1362  *
1363  * @param[in] dev
1364  *   Pointer to the rte_eth_dev structure.
1365  * @param[in] item
1366  *   Item specification.
1367  * @param[in] attr
1368  *   Attributes of flow that includes this item.
1369  * @param[out] error
1370  *   Pointer to error structure.
1371  *
1372  * @return
1373  *   0 on success, a negative errno value otherwise and rte_errno is set.
1374  */
1375 static int
1376 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1377                            const struct rte_flow_item *item,
1378                            const struct rte_flow_attr *attr __rte_unused,
1379                            struct rte_flow_error *error)
1380 {
1381         struct mlx5_priv *priv = dev->data->dev_private;
1382         struct mlx5_dev_config *config = &priv->config;
1383         const struct rte_flow_item_mark *spec = item->spec;
1384         const struct rte_flow_item_mark *mask = item->mask;
1385         const struct rte_flow_item_mark nic_mask = {
1386                 .id = priv->sh->dv_mark_mask,
1387         };
1388         int ret;
1389
1390         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1391                 return rte_flow_error_set(error, ENOTSUP,
1392                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1393                                           "extended metadata feature"
1394                                           " isn't enabled");
1395         if (!mlx5_flow_ext_mreg_supported(dev))
1396                 return rte_flow_error_set(error, ENOTSUP,
1397                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1398                                           "extended metadata register"
1399                                           " isn't supported");
1400         if (!nic_mask.id)
1401                 return rte_flow_error_set(error, ENOTSUP,
1402                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1403                                           "extended metadata register"
1404                                           " isn't available");
1405         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1406         if (ret < 0)
1407                 return ret;
1408         if (!spec)
1409                 return rte_flow_error_set(error, EINVAL,
1410                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1411                                           item->spec,
1412                                           "data cannot be empty");
1413         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1414                 return rte_flow_error_set(error, EINVAL,
1415                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1416                                           &spec->id,
1417                                           "mark id exceeds the limit");
1418         if (!mask)
1419                 mask = &nic_mask;
1420         if (!mask->id)
1421                 return rte_flow_error_set(error, EINVAL,
1422                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1423                                         "mask cannot be zero");
1424
1425         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1426                                         (const uint8_t *)&nic_mask,
1427                                         sizeof(struct rte_flow_item_mark),
1428                                         error);
1429         if (ret < 0)
1430                 return ret;
1431         return 0;
1432 }
1433
1434 /**
1435  * Validate META item.
1436  *
1437  * @param[in] dev
1438  *   Pointer to the rte_eth_dev structure.
1439  * @param[in] item
1440  *   Item specification.
1441  * @param[in] attr
1442  *   Attributes of flow that includes this item.
1443  * @param[out] error
1444  *   Pointer to error structure.
1445  *
1446  * @return
1447  *   0 on success, a negative errno value otherwise and rte_errno is set.
1448  */
1449 static int
1450 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1451                            const struct rte_flow_item *item,
1452                            const struct rte_flow_attr *attr,
1453                            struct rte_flow_error *error)
1454 {
1455         struct mlx5_priv *priv = dev->data->dev_private;
1456         struct mlx5_dev_config *config = &priv->config;
1457         const struct rte_flow_item_meta *spec = item->spec;
1458         const struct rte_flow_item_meta *mask = item->mask;
1459         struct rte_flow_item_meta nic_mask = {
1460                 .data = UINT32_MAX
1461         };
1462         int reg;
1463         int ret;
1464
1465         if (!spec)
1466                 return rte_flow_error_set(error, EINVAL,
1467                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1468                                           item->spec,
1469                                           "data cannot be empty");
1470         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1471                 if (!mlx5_flow_ext_mreg_supported(dev))
1472                         return rte_flow_error_set(error, ENOTSUP,
1473                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1474                                           "extended metadata register"
1475                                           " isn't supported");
1476                 reg = flow_dv_get_metadata_reg(dev, attr, error);
1477                 if (reg < 0)
1478                         return reg;
1479                 if (reg == REG_B)
1480                         return rte_flow_error_set(error, ENOTSUP,
1481                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1482                                           "match on reg_b "
1483                                           "isn't supported");
1484                 if (reg != REG_A)
1485                         nic_mask.data = priv->sh->dv_meta_mask;
1486         }
1487         if (!mask)
1488                 mask = &rte_flow_item_meta_mask;
1489         if (!mask->data)
1490                 return rte_flow_error_set(error, EINVAL,
1491                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1492                                         "mask cannot be zero");
1493
1494         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1495                                         (const uint8_t *)&nic_mask,
1496                                         sizeof(struct rte_flow_item_meta),
1497                                         error);
1498         return ret;
1499 }
1500
1501 /**
1502  * Validate TAG item.
1503  *
1504  * @param[in] dev
1505  *   Pointer to the rte_eth_dev structure.
1506  * @param[in] item
1507  *   Item specification.
1508  * @param[in] attr
1509  *   Attributes of flow that includes this item.
1510  * @param[out] error
1511  *   Pointer to error structure.
1512  *
1513  * @return
1514  *   0 on success, a negative errno value otherwise and rte_errno is set.
1515  */
1516 static int
1517 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1518                           const struct rte_flow_item *item,
1519                           const struct rte_flow_attr *attr __rte_unused,
1520                           struct rte_flow_error *error)
1521 {
1522         const struct rte_flow_item_tag *spec = item->spec;
1523         const struct rte_flow_item_tag *mask = item->mask;
1524         const struct rte_flow_item_tag nic_mask = {
1525                 .data = RTE_BE32(UINT32_MAX),
1526                 .index = 0xff,
1527         };
1528         int ret;
1529
1530         if (!mlx5_flow_ext_mreg_supported(dev))
1531                 return rte_flow_error_set(error, ENOTSUP,
1532                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1533                                           "extensive metadata register"
1534                                           " isn't supported");
1535         if (!spec)
1536                 return rte_flow_error_set(error, EINVAL,
1537                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1538                                           item->spec,
1539                                           "data cannot be empty");
1540         if (!mask)
1541                 mask = &rte_flow_item_tag_mask;
1542         if (!mask->data)
1543                 return rte_flow_error_set(error, EINVAL,
1544                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1545                                         "mask cannot be zero");
1546
1547         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1548                                         (const uint8_t *)&nic_mask,
1549                                         sizeof(struct rte_flow_item_tag),
1550                                         error);
1551         if (ret < 0)
1552                 return ret;
1553         if (mask->index != 0xff)
1554                 return rte_flow_error_set(error, EINVAL,
1555                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1556                                           "partial mask for tag index"
1557                                           " is not supported");
1558         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1559         if (ret < 0)
1560                 return ret;
1561         MLX5_ASSERT(ret != REG_NONE);
1562         return 0;
1563 }
1564
1565 /**
1566  * Validate vport item.
1567  *
1568  * @param[in] dev
1569  *   Pointer to the rte_eth_dev structure.
1570  * @param[in] item
1571  *   Item specification.
1572  * @param[in] attr
1573  *   Attributes of flow that includes this item.
1574  * @param[in] item_flags
1575  *   Bit-fields that holds the items detected until now.
1576  * @param[out] error
1577  *   Pointer to error structure.
1578  *
1579  * @return
1580  *   0 on success, a negative errno value otherwise and rte_errno is set.
1581  */
1582 static int
1583 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1584                               const struct rte_flow_item *item,
1585                               const struct rte_flow_attr *attr,
1586                               uint64_t item_flags,
1587                               struct rte_flow_error *error)
1588 {
1589         const struct rte_flow_item_port_id *spec = item->spec;
1590         const struct rte_flow_item_port_id *mask = item->mask;
1591         const struct rte_flow_item_port_id switch_mask = {
1592                         .id = 0xffffffff,
1593         };
1594         struct mlx5_priv *esw_priv;
1595         struct mlx5_priv *dev_priv;
1596         int ret;
1597
1598         if (!attr->transfer)
1599                 return rte_flow_error_set(error, EINVAL,
1600                                           RTE_FLOW_ERROR_TYPE_ITEM,
1601                                           NULL,
1602                                           "match on port id is valid only"
1603                                           " when transfer flag is enabled");
1604         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1605                 return rte_flow_error_set(error, ENOTSUP,
1606                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1607                                           "multiple source ports are not"
1608                                           " supported");
1609         if (!mask)
1610                 mask = &switch_mask;
1611         if (mask->id != 0xffffffff)
1612                 return rte_flow_error_set(error, ENOTSUP,
1613                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1614                                            mask,
1615                                            "no support for partial mask on"
1616                                            " \"id\" field");
1617         ret = mlx5_flow_item_acceptable
1618                                 (item, (const uint8_t *)mask,
1619                                  (const uint8_t *)&rte_flow_item_port_id_mask,
1620                                  sizeof(struct rte_flow_item_port_id),
1621                                  error);
1622         if (ret)
1623                 return ret;
1624         if (!spec)
1625                 return 0;
1626         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1627         if (!esw_priv)
1628                 return rte_flow_error_set(error, rte_errno,
1629                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1630                                           "failed to obtain E-Switch info for"
1631                                           " port");
1632         dev_priv = mlx5_dev_to_eswitch_info(dev);
1633         if (!dev_priv)
1634                 return rte_flow_error_set(error, rte_errno,
1635                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1636                                           NULL,
1637                                           "failed to obtain E-Switch info");
1638         if (esw_priv->domain_id != dev_priv->domain_id)
1639                 return rte_flow_error_set(error, EINVAL,
1640                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1641                                           "cannot match on a port from a"
1642                                           " different E-Switch");
1643         return 0;
1644 }
1645
1646 /**
1647  * Validate VLAN item.
1648  *
1649  * @param[in] item
1650  *   Item specification.
1651  * @param[in] item_flags
1652  *   Bit-fields that holds the items detected until now.
1653  * @param[in] dev
1654  *   Ethernet device flow is being created on.
1655  * @param[out] error
1656  *   Pointer to error structure.
1657  *
1658  * @return
1659  *   0 on success, a negative errno value otherwise and rte_errno is set.
1660  */
1661 static int
1662 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
1663                            uint64_t item_flags,
1664                            struct rte_eth_dev *dev,
1665                            struct rte_flow_error *error)
1666 {
1667         const struct rte_flow_item_vlan *mask = item->mask;
1668         const struct rte_flow_item_vlan nic_mask = {
1669                 .tci = RTE_BE16(UINT16_MAX),
1670                 .inner_type = RTE_BE16(UINT16_MAX),
1671         };
1672         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1673         int ret;
1674         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1675                                         MLX5_FLOW_LAYER_INNER_L4) :
1676                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1677                                         MLX5_FLOW_LAYER_OUTER_L4);
1678         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1679                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1680
1681         if (item_flags & vlanm)
1682                 return rte_flow_error_set(error, EINVAL,
1683                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1684                                           "multiple VLAN layers not supported");
1685         else if ((item_flags & l34m) != 0)
1686                 return rte_flow_error_set(error, EINVAL,
1687                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1688                                           "VLAN cannot follow L3/L4 layer");
1689         if (!mask)
1690                 mask = &rte_flow_item_vlan_mask;
1691         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1692                                         (const uint8_t *)&nic_mask,
1693                                         sizeof(struct rte_flow_item_vlan),
1694                                         error);
1695         if (ret)
1696                 return ret;
1697         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1698                 struct mlx5_priv *priv = dev->data->dev_private;
1699
1700                 if (priv->vmwa_context) {
1701                         /*
1702                          * Non-NULL context means we have a virtual machine
1703                          * and SR-IOV enabled, we have to create VLAN interface
1704                          * to make hypervisor to setup E-Switch vport
1705                          * context correctly. We avoid creating the multiple
1706                          * VLAN interfaces, so we cannot support VLAN tag mask.
1707                          */
1708                         return rte_flow_error_set(error, EINVAL,
1709                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1710                                                   item,
1711                                                   "VLAN tag mask is not"
1712                                                   " supported in virtual"
1713                                                   " environment");
1714                 }
1715         }
1716         return 0;
1717 }
1718
1719 /*
1720  * GTP flags are contained in 1 byte of the format:
1721  * -------------------------------------------
1722  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
1723  * |-----------------------------------------|
1724  * | value | Version | PT | Res | E | S | PN |
1725  * -------------------------------------------
1726  *
1727  * Matching is supported only for GTP flags E, S, PN.
1728  */
1729 #define MLX5_GTP_FLAGS_MASK     0x07
1730
1731 /**
1732  * Validate GTP item.
1733  *
1734  * @param[in] dev
1735  *   Pointer to the rte_eth_dev structure.
1736  * @param[in] item
1737  *   Item specification.
1738  * @param[in] item_flags
1739  *   Bit-fields that holds the items detected until now.
1740  * @param[out] error
1741  *   Pointer to error structure.
1742  *
1743  * @return
1744  *   0 on success, a negative errno value otherwise and rte_errno is set.
1745  */
1746 static int
1747 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
1748                           const struct rte_flow_item *item,
1749                           uint64_t item_flags,
1750                           struct rte_flow_error *error)
1751 {
1752         struct mlx5_priv *priv = dev->data->dev_private;
1753         const struct rte_flow_item_gtp *spec = item->spec;
1754         const struct rte_flow_item_gtp *mask = item->mask;
1755         const struct rte_flow_item_gtp nic_mask = {
1756                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
1757                 .msg_type = 0xff,
1758                 .teid = RTE_BE32(0xffffffff),
1759         };
1760
1761         if (!priv->config.hca_attr.tunnel_stateless_gtp)
1762                 return rte_flow_error_set(error, ENOTSUP,
1763                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1764                                           "GTP support is not enabled");
1765         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1766                 return rte_flow_error_set(error, ENOTSUP,
1767                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1768                                           "multiple tunnel layers not"
1769                                           " supported");
1770         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1771                 return rte_flow_error_set(error, EINVAL,
1772                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1773                                           "no outer UDP layer found");
1774         if (!mask)
1775                 mask = &rte_flow_item_gtp_mask;
1776         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
1777                 return rte_flow_error_set(error, ENOTSUP,
1778                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1779                                           "Match is supported for GTP"
1780                                           " flags only");
1781         return mlx5_flow_item_acceptable
1782                 (item, (const uint8_t *)mask,
1783                  (const uint8_t *)&nic_mask,
1784                  sizeof(struct rte_flow_item_gtp),
1785                  error);
1786 }
1787
1788 /**
1789  * Validate the pop VLAN action.
1790  *
1791  * @param[in] dev
1792  *   Pointer to the rte_eth_dev structure.
1793  * @param[in] action_flags
1794  *   Holds the actions detected until now.
1795  * @param[in] action
1796  *   Pointer to the pop vlan action.
1797  * @param[in] item_flags
1798  *   The items found in this flow rule.
1799  * @param[in] attr
1800  *   Pointer to flow attributes.
1801  * @param[out] error
1802  *   Pointer to error structure.
1803  *
1804  * @return
1805  *   0 on success, a negative errno value otherwise and rte_errno is set.
1806  */
1807 static int
1808 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
1809                                  uint64_t action_flags,
1810                                  const struct rte_flow_action *action,
1811                                  uint64_t item_flags,
1812                                  const struct rte_flow_attr *attr,
1813                                  struct rte_flow_error *error)
1814 {
1815         const struct mlx5_priv *priv = dev->data->dev_private;
1816
1817         (void)action;
1818         (void)attr;
1819         if (!priv->sh->pop_vlan_action)
1820                 return rte_flow_error_set(error, ENOTSUP,
1821                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1822                                           NULL,
1823                                           "pop vlan action is not supported");
1824         if (attr->egress)
1825                 return rte_flow_error_set(error, ENOTSUP,
1826                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1827                                           NULL,
1828                                           "pop vlan action not supported for "
1829                                           "egress");
1830         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
1831                 return rte_flow_error_set(error, ENOTSUP,
1832                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1833                                           "no support for multiple VLAN "
1834                                           "actions");
1835         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
1836                 return rte_flow_error_set(error, ENOTSUP,
1837                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1838                                           NULL,
1839                                           "cannot pop vlan without a "
1840                                           "match on (outer) vlan in the flow");
1841         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1842                 return rte_flow_error_set(error, EINVAL,
1843                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1844                                           "wrong action order, port_id should "
1845                                           "be after pop VLAN action");
1846         if (!attr->transfer && priv->representor)
1847                 return rte_flow_error_set(error, ENOTSUP,
1848                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1849                                           "pop vlan action for VF representor "
1850                                           "not supported on NIC table");
1851         return 0;
1852 }
1853
1854 /**
1855  * Get VLAN default info from vlan match info.
1856  *
1857  * @param[in] items
1858  *   the list of item specifications.
1859  * @param[out] vlan
1860  *   pointer VLAN info to fill to.
1861  *
1862  * @return
1863  *   0 on success, a negative errno value otherwise and rte_errno is set.
1864  */
1865 static void
1866 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
1867                                   struct rte_vlan_hdr *vlan)
1868 {
1869         const struct rte_flow_item_vlan nic_mask = {
1870                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
1871                                 MLX5DV_FLOW_VLAN_VID_MASK),
1872                 .inner_type = RTE_BE16(0xffff),
1873         };
1874
1875         if (items == NULL)
1876                 return;
1877         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1878                 int type = items->type;
1879
1880                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
1881                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
1882                         break;
1883         }
1884         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
1885                 const struct rte_flow_item_vlan *vlan_m = items->mask;
1886                 const struct rte_flow_item_vlan *vlan_v = items->spec;
1887
1888                 /* If VLAN item in pattern doesn't contain data, return here. */
1889                 if (!vlan_v)
1890                         return;
1891                 if (!vlan_m)
1892                         vlan_m = &nic_mask;
1893                 /* Only full match values are accepted */
1894                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
1895                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
1896                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
1897                         vlan->vlan_tci |=
1898                                 rte_be_to_cpu_16(vlan_v->tci &
1899                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
1900                 }
1901                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
1902                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
1903                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
1904                         vlan->vlan_tci |=
1905                                 rte_be_to_cpu_16(vlan_v->tci &
1906                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
1907                 }
1908                 if (vlan_m->inner_type == nic_mask.inner_type)
1909                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
1910                                                            vlan_m->inner_type);
1911         }
1912 }
1913
1914 /**
1915  * Validate the push VLAN action.
1916  *
1917  * @param[in] dev
1918  *   Pointer to the rte_eth_dev structure.
1919  * @param[in] action_flags
1920  *   Holds the actions detected until now.
1921  * @param[in] item_flags
1922  *   The items found in this flow rule.
1923  * @param[in] action
1924  *   Pointer to the action structure.
1925  * @param[in] attr
1926  *   Pointer to flow attributes
1927  * @param[out] error
1928  *   Pointer to error structure.
1929  *
1930  * @return
1931  *   0 on success, a negative errno value otherwise and rte_errno is set.
1932  */
1933 static int
1934 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
1935                                   uint64_t action_flags,
1936                                   const struct rte_flow_item_vlan *vlan_m,
1937                                   const struct rte_flow_action *action,
1938                                   const struct rte_flow_attr *attr,
1939                                   struct rte_flow_error *error)
1940 {
1941         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
1942         const struct mlx5_priv *priv = dev->data->dev_private;
1943
1944         if (!attr->transfer && attr->ingress)
1945                 return rte_flow_error_set(error, ENOTSUP,
1946                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1947                                           NULL,
1948                                           "push VLAN action not supported for "
1949                                           "ingress");
1950         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
1951             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
1952                 return rte_flow_error_set(error, EINVAL,
1953                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1954                                           "invalid vlan ethertype");
1955         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
1956                 return rte_flow_error_set(error, ENOTSUP,
1957                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1958                                           "no support for multiple VLAN "
1959                                           "actions");
1960         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1961                 return rte_flow_error_set(error, EINVAL,
1962                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1963                                           "wrong action order, port_id should "
1964                                           "be after push VLAN");
1965         if (!attr->transfer && priv->representor)
1966                 return rte_flow_error_set(error, ENOTSUP,
1967                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1968                                           "push vlan action for VF representor "
1969                                           "not supported on NIC table");
1970         if (vlan_m &&
1971             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
1972             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
1973                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
1974             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
1975             !(mlx5_flow_find_action
1976                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
1977                 return rte_flow_error_set(error, EINVAL,
1978                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1979                                           "not full match mask on VLAN PCP and "
1980                                           "there is no of_set_vlan_pcp action, "
1981                                           "push VLAN action cannot figure out "
1982                                           "PCP value");
1983         if (vlan_m &&
1984             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
1985             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
1986                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
1987             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
1988             !(mlx5_flow_find_action
1989                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
1990                 return rte_flow_error_set(error, EINVAL,
1991                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1992                                           "not full match mask on VLAN VID and "
1993                                           "there is no of_set_vlan_vid action, "
1994                                           "push VLAN action cannot figure out "
1995                                           "VID value");
1996         (void)attr;
1997         return 0;
1998 }
1999
2000 /**
2001  * Validate the set VLAN PCP.
2002  *
2003  * @param[in] action_flags
2004  *   Holds the actions detected until now.
2005  * @param[in] actions
2006  *   Pointer to the list of actions remaining in the flow rule.
2007  * @param[out] error
2008  *   Pointer to error structure.
2009  *
2010  * @return
2011  *   0 on success, a negative errno value otherwise and rte_errno is set.
2012  */
2013 static int
2014 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2015                                      const struct rte_flow_action actions[],
2016                                      struct rte_flow_error *error)
2017 {
2018         const struct rte_flow_action *action = actions;
2019         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2020
2021         if (conf->vlan_pcp > 7)
2022                 return rte_flow_error_set(error, EINVAL,
2023                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2024                                           "VLAN PCP value is too big");
2025         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2026                 return rte_flow_error_set(error, ENOTSUP,
2027                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2028                                           "set VLAN PCP action must follow "
2029                                           "the push VLAN action");
2030         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2031                 return rte_flow_error_set(error, ENOTSUP,
2032                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2033                                           "Multiple VLAN PCP modification are "
2034                                           "not supported");
2035         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2036                 return rte_flow_error_set(error, EINVAL,
2037                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2038                                           "wrong action order, port_id should "
2039                                           "be after set VLAN PCP");
2040         return 0;
2041 }
2042
2043 /**
2044  * Validate the set VLAN VID.
2045  *
2046  * @param[in] item_flags
2047  *   Holds the items detected in this rule.
2048  * @param[in] action_flags
2049  *   Holds the actions detected until now.
2050  * @param[in] actions
2051  *   Pointer to the list of actions remaining in the flow rule.
2052  * @param[out] error
2053  *   Pointer to error structure.
2054  *
2055  * @return
2056  *   0 on success, a negative errno value otherwise and rte_errno is set.
2057  */
2058 static int
2059 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2060                                      uint64_t action_flags,
2061                                      const struct rte_flow_action actions[],
2062                                      struct rte_flow_error *error)
2063 {
2064         const struct rte_flow_action *action = actions;
2065         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2066
2067         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2068                 return rte_flow_error_set(error, EINVAL,
2069                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2070                                           "VLAN VID value is too big");
2071         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2072             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2073                 return rte_flow_error_set(error, ENOTSUP,
2074                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2075                                           "set VLAN VID action must follow push"
2076                                           " VLAN action or match on VLAN item");
2077         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2078                 return rte_flow_error_set(error, ENOTSUP,
2079                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2080                                           "Multiple VLAN VID modifications are "
2081                                           "not supported");
2082         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2083                 return rte_flow_error_set(error, EINVAL,
2084                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2085                                           "wrong action order, port_id should "
2086                                           "be after set VLAN VID");
2087         return 0;
2088 }
2089
2090 /*
2091  * Validate the FLAG action.
2092  *
2093  * @param[in] dev
2094  *   Pointer to the rte_eth_dev structure.
2095  * @param[in] action_flags
2096  *   Holds the actions detected until now.
2097  * @param[in] attr
2098  *   Pointer to flow attributes
2099  * @param[out] error
2100  *   Pointer to error structure.
2101  *
2102  * @return
2103  *   0 on success, a negative errno value otherwise and rte_errno is set.
2104  */
2105 static int
2106 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
2107                              uint64_t action_flags,
2108                              const struct rte_flow_attr *attr,
2109                              struct rte_flow_error *error)
2110 {
2111         struct mlx5_priv *priv = dev->data->dev_private;
2112         struct mlx5_dev_config *config = &priv->config;
2113         int ret;
2114
2115         /* Fall back if no extended metadata register support. */
2116         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2117                 return mlx5_flow_validate_action_flag(action_flags, attr,
2118                                                       error);
2119         /* Extensive metadata mode requires registers. */
2120         if (!mlx5_flow_ext_mreg_supported(dev))
2121                 return rte_flow_error_set(error, ENOTSUP,
2122                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2123                                           "no metadata registers "
2124                                           "to support flag action");
2125         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
2126                 return rte_flow_error_set(error, ENOTSUP,
2127                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2128                                           "extended metadata register"
2129                                           " isn't available");
2130         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2131         if (ret < 0)
2132                 return ret;
2133         MLX5_ASSERT(ret > 0);
2134         if (action_flags & MLX5_FLOW_ACTION_MARK)
2135                 return rte_flow_error_set(error, EINVAL,
2136                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2137                                           "can't mark and flag in same flow");
2138         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2139                 return rte_flow_error_set(error, EINVAL,
2140                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2141                                           "can't have 2 flag"
2142                                           " actions in same flow");
2143         return 0;
2144 }
2145
2146 /**
2147  * Validate MARK action.
2148  *
2149  * @param[in] dev
2150  *   Pointer to the rte_eth_dev structure.
2151  * @param[in] action
2152  *   Pointer to action.
2153  * @param[in] action_flags
2154  *   Holds the actions detected until now.
2155  * @param[in] attr
2156  *   Pointer to flow attributes
2157  * @param[out] error
2158  *   Pointer to error structure.
2159  *
2160  * @return
2161  *   0 on success, a negative errno value otherwise and rte_errno is set.
2162  */
2163 static int
2164 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
2165                              const struct rte_flow_action *action,
2166                              uint64_t action_flags,
2167                              const struct rte_flow_attr *attr,
2168                              struct rte_flow_error *error)
2169 {
2170         struct mlx5_priv *priv = dev->data->dev_private;
2171         struct mlx5_dev_config *config = &priv->config;
2172         const struct rte_flow_action_mark *mark = action->conf;
2173         int ret;
2174
2175         /* Fall back if no extended metadata register support. */
2176         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2177                 return mlx5_flow_validate_action_mark(action, action_flags,
2178                                                       attr, error);
2179         /* Extensive metadata mode requires registers. */
2180         if (!mlx5_flow_ext_mreg_supported(dev))
2181                 return rte_flow_error_set(error, ENOTSUP,
2182                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2183                                           "no metadata registers "
2184                                           "to support mark action");
2185         if (!priv->sh->dv_mark_mask)
2186                 return rte_flow_error_set(error, ENOTSUP,
2187                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2188                                           "extended metadata register"
2189                                           " isn't available");
2190         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2191         if (ret < 0)
2192                 return ret;
2193         MLX5_ASSERT(ret > 0);
2194         if (!mark)
2195                 return rte_flow_error_set(error, EINVAL,
2196                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2197                                           "configuration cannot be null");
2198         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
2199                 return rte_flow_error_set(error, EINVAL,
2200                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2201                                           &mark->id,
2202                                           "mark id exceeds the limit");
2203         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2204                 return rte_flow_error_set(error, EINVAL,
2205                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2206                                           "can't flag and mark in same flow");
2207         if (action_flags & MLX5_FLOW_ACTION_MARK)
2208                 return rte_flow_error_set(error, EINVAL,
2209                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2210                                           "can't have 2 mark actions in same"
2211                                           " flow");
2212         return 0;
2213 }
2214
2215 /**
2216  * Validate SET_META action.
2217  *
2218  * @param[in] dev
2219  *   Pointer to the rte_eth_dev structure.
2220  * @param[in] action
2221  *   Pointer to the action structure.
2222  * @param[in] action_flags
2223  *   Holds the actions detected until now.
2224  * @param[in] attr
2225  *   Pointer to flow attributes
2226  * @param[out] error
2227  *   Pointer to error structure.
2228  *
2229  * @return
2230  *   0 on success, a negative errno value otherwise and rte_errno is set.
2231  */
2232 static int
2233 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
2234                                  const struct rte_flow_action *action,
2235                                  uint64_t action_flags __rte_unused,
2236                                  const struct rte_flow_attr *attr,
2237                                  struct rte_flow_error *error)
2238 {
2239         const struct rte_flow_action_set_meta *conf;
2240         uint32_t nic_mask = UINT32_MAX;
2241         int reg;
2242
2243         if (!mlx5_flow_ext_mreg_supported(dev))
2244                 return rte_flow_error_set(error, ENOTSUP,
2245                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2246                                           "extended metadata register"
2247                                           " isn't supported");
2248         reg = flow_dv_get_metadata_reg(dev, attr, error);
2249         if (reg < 0)
2250                 return reg;
2251         if (reg != REG_A && reg != REG_B) {
2252                 struct mlx5_priv *priv = dev->data->dev_private;
2253
2254                 nic_mask = priv->sh->dv_meta_mask;
2255         }
2256         if (!(action->conf))
2257                 return rte_flow_error_set(error, EINVAL,
2258                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2259                                           "configuration cannot be null");
2260         conf = (const struct rte_flow_action_set_meta *)action->conf;
2261         if (!conf->mask)
2262                 return rte_flow_error_set(error, EINVAL,
2263                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2264                                           "zero mask doesn't have any effect");
2265         if (conf->mask & ~nic_mask)
2266                 return rte_flow_error_set(error, EINVAL,
2267                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2268                                           "meta data must be within reg C0");
2269         return 0;
2270 }
2271
2272 /**
2273  * Validate SET_TAG action.
2274  *
2275  * @param[in] dev
2276  *   Pointer to the rte_eth_dev structure.
2277  * @param[in] action
2278  *   Pointer to the action structure.
2279  * @param[in] action_flags
2280  *   Holds the actions detected until now.
2281  * @param[in] attr
2282  *   Pointer to flow attributes
2283  * @param[out] error
2284  *   Pointer to error structure.
2285  *
2286  * @return
2287  *   0 on success, a negative errno value otherwise and rte_errno is set.
2288  */
2289 static int
2290 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
2291                                 const struct rte_flow_action *action,
2292                                 uint64_t action_flags,
2293                                 const struct rte_flow_attr *attr,
2294                                 struct rte_flow_error *error)
2295 {
2296         const struct rte_flow_action_set_tag *conf;
2297         const uint64_t terminal_action_flags =
2298                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
2299                 MLX5_FLOW_ACTION_RSS;
2300         int ret;
2301
2302         if (!mlx5_flow_ext_mreg_supported(dev))
2303                 return rte_flow_error_set(error, ENOTSUP,
2304                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2305                                           "extensive metadata register"
2306                                           " isn't supported");
2307         if (!(action->conf))
2308                 return rte_flow_error_set(error, EINVAL,
2309                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2310                                           "configuration cannot be null");
2311         conf = (const struct rte_flow_action_set_tag *)action->conf;
2312         if (!conf->mask)
2313                 return rte_flow_error_set(error, EINVAL,
2314                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2315                                           "zero mask doesn't have any effect");
2316         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
2317         if (ret < 0)
2318                 return ret;
2319         if (!attr->transfer && attr->ingress &&
2320             (action_flags & terminal_action_flags))
2321                 return rte_flow_error_set(error, EINVAL,
2322                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2323                                           "set_tag has no effect"
2324                                           " with terminal actions");
2325         return 0;
2326 }
2327
2328 /**
2329  * Validate count action.
2330  *
2331  * @param[in] dev
2332  *   Pointer to rte_eth_dev structure.
2333  * @param[out] error
2334  *   Pointer to error structure.
2335  *
2336  * @return
2337  *   0 on success, a negative errno value otherwise and rte_errno is set.
2338  */
2339 static int
2340 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2341                               struct rte_flow_error *error)
2342 {
2343         struct mlx5_priv *priv = dev->data->dev_private;
2344
2345         if (!priv->config.devx)
2346                 goto notsup_err;
2347 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2348         return 0;
2349 #endif
2350 notsup_err:
2351         return rte_flow_error_set
2352                       (error, ENOTSUP,
2353                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2354                        NULL,
2355                        "count action not supported");
2356 }
2357
2358 /**
2359  * Validate the L2 encap action.
2360  *
2361  * @param[in] dev
2362  *   Pointer to the rte_eth_dev structure.
2363  * @param[in] action_flags
2364  *   Holds the actions detected until now.
2365  * @param[in] action
2366  *   Pointer to the action structure.
2367  * @param[in] attr
2368  *   Pointer to flow attributes.
2369  * @param[out] error
2370  *   Pointer to error structure.
2371  *
2372  * @return
2373  *   0 on success, a negative errno value otherwise and rte_errno is set.
2374  */
2375 static int
2376 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
2377                                  uint64_t action_flags,
2378                                  const struct rte_flow_action *action,
2379                                  const struct rte_flow_attr *attr,
2380                                  struct rte_flow_error *error)
2381 {
2382         const struct mlx5_priv *priv = dev->data->dev_private;
2383
2384         if (!(action->conf))
2385                 return rte_flow_error_set(error, EINVAL,
2386                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2387                                           "configuration cannot be null");
2388         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
2389                 return rte_flow_error_set(error, EINVAL,
2390                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2391                                           "can only have a single encap action "
2392                                           "in a flow");
2393         if (!attr->transfer && priv->representor)
2394                 return rte_flow_error_set(error, ENOTSUP,
2395                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2396                                           "encap action for VF representor "
2397                                           "not supported on NIC table");
2398         return 0;
2399 }
2400
2401 /**
2402  * Validate a decap action.
2403  *
2404  * @param[in] dev
2405  *   Pointer to the rte_eth_dev structure.
2406  * @param[in] action_flags
2407  *   Holds the actions detected until now.
2408  * @param[in] attr
2409  *   Pointer to flow attributes
2410  * @param[out] error
2411  *   Pointer to error structure.
2412  *
2413  * @return
2414  *   0 on success, a negative errno value otherwise and rte_errno is set.
2415  */
2416 static int
2417 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
2418                               uint64_t action_flags,
2419                               const struct rte_flow_attr *attr,
2420                               struct rte_flow_error *error)
2421 {
2422         const struct mlx5_priv *priv = dev->data->dev_private;
2423
2424         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
2425                 return rte_flow_error_set(error, ENOTSUP,
2426                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2427                                           action_flags &
2428                                           MLX5_FLOW_ACTION_DECAP ? "can only "
2429                                           "have a single decap action" : "decap "
2430                                           "after encap is not supported");
2431         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2432                 return rte_flow_error_set(error, EINVAL,
2433                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2434                                           "can't have decap action after"
2435                                           " modify action");
2436         if (attr->egress)
2437                 return rte_flow_error_set(error, ENOTSUP,
2438                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2439                                           NULL,
2440                                           "decap action not supported for "
2441                                           "egress");
2442         if (!attr->transfer && priv->representor)
2443                 return rte_flow_error_set(error, ENOTSUP,
2444                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2445                                           "decap action for VF representor "
2446                                           "not supported on NIC table");
2447         return 0;
2448 }
2449
2450 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
2451
2452 /**
2453  * Validate the raw encap and decap actions.
2454  *
2455  * @param[in] dev
2456  *   Pointer to the rte_eth_dev structure.
2457  * @param[in] decap
2458  *   Pointer to the decap action.
2459  * @param[in] encap
2460  *   Pointer to the encap action.
2461  * @param[in] attr
2462  *   Pointer to flow attributes
2463  * @param[in/out] action_flags
2464  *   Holds the actions detected until now.
2465  * @param[out] actions_n
2466  *   pointer to the number of actions counter.
2467  * @param[out] error
2468  *   Pointer to error structure.
2469  *
2470  * @return
2471  *   0 on success, a negative errno value otherwise and rte_errno is set.
2472  */
2473 static int
2474 flow_dv_validate_action_raw_encap_decap
2475         (struct rte_eth_dev *dev,
2476          const struct rte_flow_action_raw_decap *decap,
2477          const struct rte_flow_action_raw_encap *encap,
2478          const struct rte_flow_attr *attr, uint64_t *action_flags,
2479          int *actions_n, struct rte_flow_error *error)
2480 {
2481         const struct mlx5_priv *priv = dev->data->dev_private;
2482         int ret;
2483
2484         if (encap && (!encap->size || !encap->data))
2485                 return rte_flow_error_set(error, EINVAL,
2486                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2487                                           "raw encap data cannot be empty");
2488         if (decap && encap) {
2489                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
2490                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2491                         /* L3 encap. */
2492                         decap = NULL;
2493                 else if (encap->size <=
2494                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2495                            decap->size >
2496                            MLX5_ENCAPSULATION_DECISION_SIZE)
2497                         /* L3 decap. */
2498                         encap = NULL;
2499                 else if (encap->size >
2500                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2501                            decap->size >
2502                            MLX5_ENCAPSULATION_DECISION_SIZE)
2503                         /* 2 L2 actions: encap and decap. */
2504                         ;
2505                 else
2506                         return rte_flow_error_set(error,
2507                                 ENOTSUP,
2508                                 RTE_FLOW_ERROR_TYPE_ACTION,
2509                                 NULL, "unsupported too small "
2510                                 "raw decap and too small raw "
2511                                 "encap combination");
2512         }
2513         if (decap) {
2514                 ret = flow_dv_validate_action_decap(dev, *action_flags, attr,
2515                                                     error);
2516                 if (ret < 0)
2517                         return ret;
2518                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
2519                 ++(*actions_n);
2520         }
2521         if (encap) {
2522                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
2523                         return rte_flow_error_set(error, ENOTSUP,
2524                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2525                                                   NULL,
2526                                                   "small raw encap size");
2527                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
2528                         return rte_flow_error_set(error, EINVAL,
2529                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2530                                                   NULL,
2531                                                   "more than one encap action");
2532                 if (!attr->transfer && priv->representor)
2533                         return rte_flow_error_set
2534                                         (error, ENOTSUP,
2535                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2536                                          "encap action for VF representor "
2537                                          "not supported on NIC table");
2538                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
2539                 ++(*actions_n);
2540         }
2541         return 0;
2542 }
2543
2544 /**
2545  * Find existing encap/decap resource or create and register a new one.
2546  *
2547  * @param[in, out] dev
2548  *   Pointer to rte_eth_dev structure.
2549  * @param[in, out] resource
2550  *   Pointer to encap/decap resource.
2551  * @parm[in, out] dev_flow
2552  *   Pointer to the dev_flow.
2553  * @param[out] error
2554  *   pointer to error structure.
2555  *
2556  * @return
2557  *   0 on success otherwise -errno and errno is set.
2558  */
2559 static int
2560 flow_dv_encap_decap_resource_register
2561                         (struct rte_eth_dev *dev,
2562                          struct mlx5_flow_dv_encap_decap_resource *resource,
2563                          struct mlx5_flow *dev_flow,
2564                          struct rte_flow_error *error)
2565 {
2566         struct mlx5_priv *priv = dev->data->dev_private;
2567         struct mlx5_dev_ctx_shared *sh = priv->sh;
2568         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2569         struct mlx5dv_dr_domain *domain;
2570         uint32_t idx = 0;
2571
2572         resource->flags = dev_flow->dv.group ? 0 : 1;
2573         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2574                 domain = sh->fdb_domain;
2575         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2576                 domain = sh->rx_domain;
2577         else
2578                 domain = sh->tx_domain;
2579         /* Lookup a matching resource from cache. */
2580         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], sh->encaps_decaps, idx,
2581                       cache_resource, next) {
2582                 if (resource->reformat_type == cache_resource->reformat_type &&
2583                     resource->ft_type == cache_resource->ft_type &&
2584                     resource->flags == cache_resource->flags &&
2585                     resource->size == cache_resource->size &&
2586                     !memcmp((const void *)resource->buf,
2587                             (const void *)cache_resource->buf,
2588                             resource->size)) {
2589                         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2590                                 (void *)cache_resource,
2591                                 rte_atomic32_read(&cache_resource->refcnt));
2592                         rte_atomic32_inc(&cache_resource->refcnt);
2593                         dev_flow->handle->dvh.rix_encap_decap = idx;
2594                         dev_flow->dv.encap_decap = cache_resource;
2595                         return 0;
2596                 }
2597         }
2598         /* Register new encap/decap resource. */
2599         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2600                                        &dev_flow->handle->dvh.rix_encap_decap);
2601         if (!cache_resource)
2602                 return rte_flow_error_set(error, ENOMEM,
2603                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2604                                           "cannot allocate resource memory");
2605         *cache_resource = *resource;
2606         cache_resource->verbs_action =
2607                 mlx5_glue->dv_create_flow_action_packet_reformat
2608                         (sh->ctx, cache_resource->reformat_type,
2609                          cache_resource->ft_type, domain, cache_resource->flags,
2610                          cache_resource->size,
2611                          (cache_resource->size ? cache_resource->buf : NULL));
2612         if (!cache_resource->verbs_action) {
2613                 rte_free(cache_resource);
2614                 return rte_flow_error_set(error, ENOMEM,
2615                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2616                                           NULL, "cannot create action");
2617         }
2618         rte_atomic32_init(&cache_resource->refcnt);
2619         rte_atomic32_inc(&cache_resource->refcnt);
2620         ILIST_INSERT(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &sh->encaps_decaps,
2621                      dev_flow->handle->dvh.rix_encap_decap, cache_resource,
2622                      next);
2623         dev_flow->dv.encap_decap = cache_resource;
2624         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2625                 (void *)cache_resource,
2626                 rte_atomic32_read(&cache_resource->refcnt));
2627         return 0;
2628 }
2629
2630 /**
2631  * Find existing table jump resource or create and register a new one.
2632  *
2633  * @param[in, out] dev
2634  *   Pointer to rte_eth_dev structure.
2635  * @param[in, out] tbl
2636  *   Pointer to flow table resource.
2637  * @parm[in, out] dev_flow
2638  *   Pointer to the dev_flow.
2639  * @param[out] error
2640  *   pointer to error structure.
2641  *
2642  * @return
2643  *   0 on success otherwise -errno and errno is set.
2644  */
2645 static int
2646 flow_dv_jump_tbl_resource_register
2647                         (struct rte_eth_dev *dev __rte_unused,
2648                          struct mlx5_flow_tbl_resource *tbl,
2649                          struct mlx5_flow *dev_flow,
2650                          struct rte_flow_error *error)
2651 {
2652         struct mlx5_flow_tbl_data_entry *tbl_data =
2653                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2654         int cnt;
2655
2656         MLX5_ASSERT(tbl);
2657         cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
2658         if (!cnt) {
2659                 tbl_data->jump.action =
2660                         mlx5_glue->dr_create_flow_action_dest_flow_tbl
2661                         (tbl->obj);
2662                 if (!tbl_data->jump.action)
2663                         return rte_flow_error_set(error, ENOMEM,
2664                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2665                                         NULL, "cannot create jump action");
2666                 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2667                         (void *)&tbl_data->jump, cnt);
2668         } else {
2669                 /* old jump should not make the table ref++. */
2670                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
2671                 MLX5_ASSERT(tbl_data->jump.action);
2672                 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2673                         (void *)&tbl_data->jump, cnt);
2674         }
2675         rte_atomic32_inc(&tbl_data->jump.refcnt);
2676         dev_flow->handle->rix_jump = tbl_data->idx;
2677         dev_flow->dv.jump = &tbl_data->jump;
2678         return 0;
2679 }
2680
2681 /**
2682  * Find existing default miss resource or create and register a new one.
2683  *
2684  * @param[in, out] dev
2685  *   Pointer to rte_eth_dev structure.
2686  * @param[out] error
2687  *   pointer to error structure.
2688  *
2689  * @return
2690  *   0 on success otherwise -errno and errno is set.
2691  */
2692 static int
2693 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
2694                 struct rte_flow_error *error)
2695 {
2696         struct mlx5_priv *priv = dev->data->dev_private;
2697         struct mlx5_dev_ctx_shared *sh = priv->sh;
2698         struct mlx5_flow_default_miss_resource *cache_resource =
2699                         &sh->default_miss;
2700         int cnt = rte_atomic32_read(&cache_resource->refcnt);
2701
2702         if (!cnt) {
2703                 MLX5_ASSERT(cache_resource->action);
2704                 cache_resource->action =
2705                 mlx5_glue->dr_create_flow_action_default_miss();
2706                 if (!cache_resource->action)
2707                         return rte_flow_error_set(error, ENOMEM,
2708                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2709                                         "cannot create default miss action");
2710                 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++",
2711                                 (void *)cache_resource->action, cnt);
2712         }
2713         rte_atomic32_inc(&cache_resource->refcnt);
2714         return 0;
2715 }
2716
2717 /**
2718  * Find existing table port ID resource or create and register a new one.
2719  *
2720  * @param[in, out] dev
2721  *   Pointer to rte_eth_dev structure.
2722  * @param[in, out] resource
2723  *   Pointer to port ID action resource.
2724  * @parm[in, out] dev_flow
2725  *   Pointer to the dev_flow.
2726  * @param[out] error
2727  *   pointer to error structure.
2728  *
2729  * @return
2730  *   0 on success otherwise -errno and errno is set.
2731  */
2732 static int
2733 flow_dv_port_id_action_resource_register
2734                         (struct rte_eth_dev *dev,
2735                          struct mlx5_flow_dv_port_id_action_resource *resource,
2736                          struct mlx5_flow *dev_flow,
2737                          struct rte_flow_error *error)
2738 {
2739         struct mlx5_priv *priv = dev->data->dev_private;
2740         struct mlx5_dev_ctx_shared *sh = priv->sh;
2741         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
2742         uint32_t idx = 0;
2743
2744         /* Lookup a matching resource from cache. */
2745         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list,
2746                       idx, cache_resource, next) {
2747                 if (resource->port_id == cache_resource->port_id) {
2748                         DRV_LOG(DEBUG, "port id action resource resource %p: "
2749                                 "refcnt %d++",
2750                                 (void *)cache_resource,
2751                                 rte_atomic32_read(&cache_resource->refcnt));
2752                         rte_atomic32_inc(&cache_resource->refcnt);
2753                         dev_flow->handle->rix_port_id_action = idx;
2754                         dev_flow->dv.port_id_action = cache_resource;
2755                         return 0;
2756                 }
2757         }
2758         /* Register new port id action resource. */
2759         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID],
2760                                        &dev_flow->handle->rix_port_id_action);
2761         if (!cache_resource)
2762                 return rte_flow_error_set(error, ENOMEM,
2763                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2764                                           "cannot allocate resource memory");
2765         *cache_resource = *resource;
2766         /*
2767          * Depending on rdma_core version the glue routine calls
2768          * either mlx5dv_dr_action_create_dest_ib_port(domain, dev_port)
2769          * or mlx5dv_dr_action_create_dest_vport(domain, vport_id).
2770          */
2771         cache_resource->action =
2772                 mlx5_glue->dr_create_flow_action_dest_port
2773                         (priv->sh->fdb_domain, resource->port_id);
2774         if (!cache_resource->action) {
2775                 rte_free(cache_resource);
2776                 return rte_flow_error_set(error, ENOMEM,
2777                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2778                                           NULL, "cannot create action");
2779         }
2780         rte_atomic32_init(&cache_resource->refcnt);
2781         rte_atomic32_inc(&cache_resource->refcnt);
2782         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
2783                      dev_flow->handle->rix_port_id_action, cache_resource,
2784                      next);
2785         dev_flow->dv.port_id_action = cache_resource;
2786         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
2787                 (void *)cache_resource,
2788                 rte_atomic32_read(&cache_resource->refcnt));
2789         return 0;
2790 }
2791
2792 /**
2793  * Find existing push vlan resource or create and register a new one.
2794  *
2795  * @param [in, out] dev
2796  *   Pointer to rte_eth_dev structure.
2797  * @param[in, out] resource
2798  *   Pointer to port ID action resource.
2799  * @parm[in, out] dev_flow
2800  *   Pointer to the dev_flow.
2801  * @param[out] error
2802  *   pointer to error structure.
2803  *
2804  * @return
2805  *   0 on success otherwise -errno and errno is set.
2806  */
2807 static int
2808 flow_dv_push_vlan_action_resource_register
2809                        (struct rte_eth_dev *dev,
2810                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
2811                         struct mlx5_flow *dev_flow,
2812                         struct rte_flow_error *error)
2813 {
2814         struct mlx5_priv *priv = dev->data->dev_private;
2815         struct mlx5_dev_ctx_shared *sh = priv->sh;
2816         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
2817         struct mlx5dv_dr_domain *domain;
2818         uint32_t idx = 0;
2819
2820         /* Lookup a matching resource from cache. */
2821         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2822                       sh->push_vlan_action_list, idx, cache_resource, next) {
2823                 if (resource->vlan_tag == cache_resource->vlan_tag &&
2824                     resource->ft_type == cache_resource->ft_type) {
2825                         DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
2826                                 "refcnt %d++",
2827                                 (void *)cache_resource,
2828                                 rte_atomic32_read(&cache_resource->refcnt));
2829                         rte_atomic32_inc(&cache_resource->refcnt);
2830                         dev_flow->handle->dvh.rix_push_vlan = idx;
2831                         dev_flow->dv.push_vlan_res = cache_resource;
2832                         return 0;
2833                 }
2834         }
2835         /* Register new push_vlan action resource. */
2836         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2837                                        &dev_flow->handle->dvh.rix_push_vlan);
2838         if (!cache_resource)
2839                 return rte_flow_error_set(error, ENOMEM,
2840                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2841                                           "cannot allocate resource memory");
2842         *cache_resource = *resource;
2843         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2844                 domain = sh->fdb_domain;
2845         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2846                 domain = sh->rx_domain;
2847         else
2848                 domain = sh->tx_domain;
2849         cache_resource->action =
2850                 mlx5_glue->dr_create_flow_action_push_vlan(domain,
2851                                                            resource->vlan_tag);
2852         if (!cache_resource->action) {
2853                 rte_free(cache_resource);
2854                 return rte_flow_error_set(error, ENOMEM,
2855                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2856                                           NULL, "cannot create action");
2857         }
2858         rte_atomic32_init(&cache_resource->refcnt);
2859         rte_atomic32_inc(&cache_resource->refcnt);
2860         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2861                      &sh->push_vlan_action_list,
2862                      dev_flow->handle->dvh.rix_push_vlan,
2863                      cache_resource, next);
2864         dev_flow->dv.push_vlan_res = cache_resource;
2865         DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
2866                 (void *)cache_resource,
2867                 rte_atomic32_read(&cache_resource->refcnt));
2868         return 0;
2869 }
2870 /**
2871  * Get the size of specific rte_flow_item_type
2872  *
2873  * @param[in] item_type
2874  *   Tested rte_flow_item_type.
2875  *
2876  * @return
2877  *   sizeof struct item_type, 0 if void or irrelevant.
2878  */
2879 static size_t
2880 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
2881 {
2882         size_t retval;
2883
2884         switch (item_type) {
2885         case RTE_FLOW_ITEM_TYPE_ETH:
2886                 retval = sizeof(struct rte_flow_item_eth);
2887                 break;
2888         case RTE_FLOW_ITEM_TYPE_VLAN:
2889                 retval = sizeof(struct rte_flow_item_vlan);
2890                 break;
2891         case RTE_FLOW_ITEM_TYPE_IPV4:
2892                 retval = sizeof(struct rte_flow_item_ipv4);
2893                 break;
2894         case RTE_FLOW_ITEM_TYPE_IPV6:
2895                 retval = sizeof(struct rte_flow_item_ipv6);
2896                 break;
2897         case RTE_FLOW_ITEM_TYPE_UDP:
2898                 retval = sizeof(struct rte_flow_item_udp);
2899                 break;
2900         case RTE_FLOW_ITEM_TYPE_TCP:
2901                 retval = sizeof(struct rte_flow_item_tcp);
2902                 break;
2903         case RTE_FLOW_ITEM_TYPE_VXLAN:
2904                 retval = sizeof(struct rte_flow_item_vxlan);
2905                 break;
2906         case RTE_FLOW_ITEM_TYPE_GRE:
2907                 retval = sizeof(struct rte_flow_item_gre);
2908                 break;
2909         case RTE_FLOW_ITEM_TYPE_NVGRE:
2910                 retval = sizeof(struct rte_flow_item_nvgre);
2911                 break;
2912         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
2913                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
2914                 break;
2915         case RTE_FLOW_ITEM_TYPE_MPLS:
2916                 retval = sizeof(struct rte_flow_item_mpls);
2917                 break;
2918         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
2919         default:
2920                 retval = 0;
2921                 break;
2922         }
2923         return retval;
2924 }
2925
2926 #define MLX5_ENCAP_IPV4_VERSION         0x40
2927 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
2928 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
2929 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
2930 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
2931 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
2932 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
2933
2934 /**
2935  * Convert the encap action data from list of rte_flow_item to raw buffer
2936  *
2937  * @param[in] items
2938  *   Pointer to rte_flow_item objects list.
2939  * @param[out] buf
2940  *   Pointer to the output buffer.
2941  * @param[out] size
2942  *   Pointer to the output buffer size.
2943  * @param[out] error
2944  *   Pointer to the error structure.
2945  *
2946  * @return
2947  *   0 on success, a negative errno value otherwise and rte_errno is set.
2948  */
2949 static int
2950 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
2951                            size_t *size, struct rte_flow_error *error)
2952 {
2953         struct rte_ether_hdr *eth = NULL;
2954         struct rte_vlan_hdr *vlan = NULL;
2955         struct rte_ipv4_hdr *ipv4 = NULL;
2956         struct rte_ipv6_hdr *ipv6 = NULL;
2957         struct rte_udp_hdr *udp = NULL;
2958         struct rte_vxlan_hdr *vxlan = NULL;
2959         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
2960         struct rte_gre_hdr *gre = NULL;
2961         size_t len;
2962         size_t temp_size = 0;
2963
2964         if (!items)
2965                 return rte_flow_error_set(error, EINVAL,
2966                                           RTE_FLOW_ERROR_TYPE_ACTION,
2967                                           NULL, "invalid empty data");
2968         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2969                 len = flow_dv_get_item_len(items->type);
2970                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
2971                         return rte_flow_error_set(error, EINVAL,
2972                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2973                                                   (void *)items->type,
2974                                                   "items total size is too big"
2975                                                   " for encap action");
2976                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
2977                 switch (items->type) {
2978                 case RTE_FLOW_ITEM_TYPE_ETH:
2979                         eth = (struct rte_ether_hdr *)&buf[temp_size];
2980                         break;
2981                 case RTE_FLOW_ITEM_TYPE_VLAN:
2982                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
2983                         if (!eth)
2984                                 return rte_flow_error_set(error, EINVAL,
2985                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2986                                                 (void *)items->type,
2987                                                 "eth header not found");
2988                         if (!eth->ether_type)
2989                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
2990                         break;
2991                 case RTE_FLOW_ITEM_TYPE_IPV4:
2992                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
2993                         if (!vlan && !eth)
2994                                 return rte_flow_error_set(error, EINVAL,
2995                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2996                                                 (void *)items->type,
2997                                                 "neither eth nor vlan"
2998                                                 " header found");
2999                         if (vlan && !vlan->eth_proto)
3000                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3001                         else if (eth && !eth->ether_type)
3002                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3003                         if (!ipv4->version_ihl)
3004                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
3005                                                     MLX5_ENCAP_IPV4_IHL_MIN;
3006                         if (!ipv4->time_to_live)
3007                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
3008                         break;
3009                 case RTE_FLOW_ITEM_TYPE_IPV6:
3010                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
3011                         if (!vlan && !eth)
3012                                 return rte_flow_error_set(error, EINVAL,
3013                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3014                                                 (void *)items->type,
3015                                                 "neither eth nor vlan"
3016                                                 " header found");
3017                         if (vlan && !vlan->eth_proto)
3018                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3019                         else if (eth && !eth->ether_type)
3020                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3021                         if (!ipv6->vtc_flow)
3022                                 ipv6->vtc_flow =
3023                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
3024                         if (!ipv6->hop_limits)
3025                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
3026                         break;
3027                 case RTE_FLOW_ITEM_TYPE_UDP:
3028                         udp = (struct rte_udp_hdr *)&buf[temp_size];
3029                         if (!ipv4 && !ipv6)
3030                                 return rte_flow_error_set(error, EINVAL,
3031                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3032                                                 (void *)items->type,
3033                                                 "ip header not found");
3034                         if (ipv4 && !ipv4->next_proto_id)
3035                                 ipv4->next_proto_id = IPPROTO_UDP;
3036                         else if (ipv6 && !ipv6->proto)
3037                                 ipv6->proto = IPPROTO_UDP;
3038                         break;
3039                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3040                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
3041                         if (!udp)
3042                                 return rte_flow_error_set(error, EINVAL,
3043                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3044                                                 (void *)items->type,
3045                                                 "udp header not found");
3046                         if (!udp->dst_port)
3047                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
3048                         if (!vxlan->vx_flags)
3049                                 vxlan->vx_flags =
3050                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
3051                         break;
3052                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3053                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
3054                         if (!udp)
3055                                 return rte_flow_error_set(error, EINVAL,
3056                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3057                                                 (void *)items->type,
3058                                                 "udp header not found");
3059                         if (!vxlan_gpe->proto)
3060                                 return rte_flow_error_set(error, EINVAL,
3061                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3062                                                 (void *)items->type,
3063                                                 "next protocol not found");
3064                         if (!udp->dst_port)
3065                                 udp->dst_port =
3066                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
3067                         if (!vxlan_gpe->vx_flags)
3068                                 vxlan_gpe->vx_flags =
3069                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
3070                         break;
3071                 case RTE_FLOW_ITEM_TYPE_GRE:
3072                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3073                         gre = (struct rte_gre_hdr *)&buf[temp_size];
3074                         if (!gre->proto)
3075                                 return rte_flow_error_set(error, EINVAL,
3076                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3077                                                 (void *)items->type,
3078                                                 "next protocol not found");
3079                         if (!ipv4 && !ipv6)
3080                                 return rte_flow_error_set(error, EINVAL,
3081                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3082                                                 (void *)items->type,
3083                                                 "ip header not found");
3084                         if (ipv4 && !ipv4->next_proto_id)
3085                                 ipv4->next_proto_id = IPPROTO_GRE;
3086                         else if (ipv6 && !ipv6->proto)
3087                                 ipv6->proto = IPPROTO_GRE;
3088                         break;
3089                 case RTE_FLOW_ITEM_TYPE_VOID:
3090                         break;
3091                 default:
3092                         return rte_flow_error_set(error, EINVAL,
3093                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3094                                                   (void *)items->type,
3095                                                   "unsupported item type");
3096                         break;
3097                 }
3098                 temp_size += len;
3099         }
3100         *size = temp_size;
3101         return 0;
3102 }
3103
3104 static int
3105 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
3106 {
3107         struct rte_ether_hdr *eth = NULL;
3108         struct rte_vlan_hdr *vlan = NULL;
3109         struct rte_ipv6_hdr *ipv6 = NULL;
3110         struct rte_udp_hdr *udp = NULL;
3111         char *next_hdr;
3112         uint16_t proto;
3113
3114         eth = (struct rte_ether_hdr *)data;
3115         next_hdr = (char *)(eth + 1);
3116         proto = RTE_BE16(eth->ether_type);
3117
3118         /* VLAN skipping */
3119         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
3120                 vlan = (struct rte_vlan_hdr *)next_hdr;
3121                 proto = RTE_BE16(vlan->eth_proto);
3122                 next_hdr += sizeof(struct rte_vlan_hdr);
3123         }
3124
3125         /* HW calculates IPv4 csum. no need to proceed */
3126         if (proto == RTE_ETHER_TYPE_IPV4)
3127                 return 0;
3128
3129         /* non IPv4/IPv6 header. not supported */
3130         if (proto != RTE_ETHER_TYPE_IPV6) {
3131                 return rte_flow_error_set(error, ENOTSUP,
3132                                           RTE_FLOW_ERROR_TYPE_ACTION,
3133                                           NULL, "Cannot offload non IPv4/IPv6");
3134         }
3135
3136         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
3137
3138         /* ignore non UDP */
3139         if (ipv6->proto != IPPROTO_UDP)
3140                 return 0;
3141
3142         udp = (struct rte_udp_hdr *)(ipv6 + 1);
3143         udp->dgram_cksum = 0;
3144
3145         return 0;
3146 }
3147
3148 /**
3149  * Convert L2 encap action to DV specification.
3150  *
3151  * @param[in] dev
3152  *   Pointer to rte_eth_dev structure.
3153  * @param[in] action
3154  *   Pointer to action structure.
3155  * @param[in, out] dev_flow
3156  *   Pointer to the mlx5_flow.
3157  * @param[in] transfer
3158  *   Mark if the flow is E-Switch flow.
3159  * @param[out] error
3160  *   Pointer to the error structure.
3161  *
3162  * @return
3163  *   0 on success, a negative errno value otherwise and rte_errno is set.
3164  */
3165 static int
3166 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
3167                                const struct rte_flow_action *action,
3168                                struct mlx5_flow *dev_flow,
3169                                uint8_t transfer,
3170                                struct rte_flow_error *error)
3171 {
3172         const struct rte_flow_item *encap_data;
3173         const struct rte_flow_action_raw_encap *raw_encap_data;
3174         struct mlx5_flow_dv_encap_decap_resource res = {
3175                 .reformat_type =
3176                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
3177                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3178                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
3179         };
3180
3181         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3182                 raw_encap_data =
3183                         (const struct rte_flow_action_raw_encap *)action->conf;
3184                 res.size = raw_encap_data->size;
3185                 memcpy(res.buf, raw_encap_data->data, res.size);
3186         } else {
3187                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
3188                         encap_data =
3189                                 ((const struct rte_flow_action_vxlan_encap *)
3190                                                 action->conf)->definition;
3191                 else
3192                         encap_data =
3193                                 ((const struct rte_flow_action_nvgre_encap *)
3194                                                 action->conf)->definition;
3195                 if (flow_dv_convert_encap_data(encap_data, res.buf,
3196                                                &res.size, error))
3197                         return -rte_errno;
3198         }
3199         if (flow_dv_zero_encap_udp_csum(res.buf, error))
3200                 return -rte_errno;
3201         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3202                 return rte_flow_error_set(error, EINVAL,
3203                                           RTE_FLOW_ERROR_TYPE_ACTION,
3204                                           NULL, "can't create L2 encap action");
3205         return 0;
3206 }
3207
3208 /**
3209  * Convert L2 decap action to DV specification.
3210  *
3211  * @param[in] dev
3212  *   Pointer to rte_eth_dev structure.
3213  * @param[in, out] dev_flow
3214  *   Pointer to the mlx5_flow.
3215  * @param[in] transfer
3216  *   Mark if the flow is E-Switch flow.
3217  * @param[out] error
3218  *   Pointer to the error structure.
3219  *
3220  * @return
3221  *   0 on success, a negative errno value otherwise and rte_errno is set.
3222  */
3223 static int
3224 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
3225                                struct mlx5_flow *dev_flow,
3226                                uint8_t transfer,
3227                                struct rte_flow_error *error)
3228 {
3229         struct mlx5_flow_dv_encap_decap_resource res = {
3230                 .size = 0,
3231                 .reformat_type =
3232                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
3233                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3234                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
3235         };
3236
3237         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3238                 return rte_flow_error_set(error, EINVAL,
3239                                           RTE_FLOW_ERROR_TYPE_ACTION,
3240                                           NULL, "can't create L2 decap action");
3241         return 0;
3242 }
3243
3244 /**
3245  * Convert raw decap/encap (L3 tunnel) action to DV specification.
3246  *
3247  * @param[in] dev
3248  *   Pointer to rte_eth_dev structure.
3249  * @param[in] action
3250  *   Pointer to action structure.
3251  * @param[in, out] dev_flow
3252  *   Pointer to the mlx5_flow.
3253  * @param[in] attr
3254  *   Pointer to the flow attributes.
3255  * @param[out] error
3256  *   Pointer to the error structure.
3257  *
3258  * @return
3259  *   0 on success, a negative errno value otherwise and rte_errno is set.
3260  */
3261 static int
3262 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
3263                                 const struct rte_flow_action *action,
3264                                 struct mlx5_flow *dev_flow,
3265                                 const struct rte_flow_attr *attr,
3266                                 struct rte_flow_error *error)
3267 {
3268         const struct rte_flow_action_raw_encap *encap_data;
3269         struct mlx5_flow_dv_encap_decap_resource res;
3270
3271         memset(&res, 0, sizeof(res));
3272         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
3273         res.size = encap_data->size;
3274         memcpy(res.buf, encap_data->data, res.size);
3275         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
3276                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
3277                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3278         if (attr->transfer)
3279                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3280         else
3281                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3282                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3283         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3284                 return rte_flow_error_set(error, EINVAL,
3285                                           RTE_FLOW_ERROR_TYPE_ACTION,
3286                                           NULL, "can't create encap action");
3287         return 0;
3288 }
3289
3290 /**
3291  * Create action push VLAN.
3292  *
3293  * @param[in] dev
3294  *   Pointer to rte_eth_dev structure.
3295  * @param[in] attr
3296  *   Pointer to the flow attributes.
3297  * @param[in] vlan
3298  *   Pointer to the vlan to push to the Ethernet header.
3299  * @param[in, out] dev_flow
3300  *   Pointer to the mlx5_flow.
3301  * @param[out] error
3302  *   Pointer to the error structure.
3303  *
3304  * @return
3305  *   0 on success, a negative errno value otherwise and rte_errno is set.
3306  */
3307 static int
3308 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
3309                                 const struct rte_flow_attr *attr,
3310                                 const struct rte_vlan_hdr *vlan,
3311                                 struct mlx5_flow *dev_flow,
3312                                 struct rte_flow_error *error)
3313 {
3314         struct mlx5_flow_dv_push_vlan_action_resource res;
3315
3316         memset(&res, 0, sizeof(res));
3317         res.vlan_tag =
3318                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
3319                                  vlan->vlan_tci);
3320         if (attr->transfer)
3321                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3322         else
3323                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3324                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3325         return flow_dv_push_vlan_action_resource_register
3326                                             (dev, &res, dev_flow, error);
3327 }
3328
3329 /**
3330  * Validate the modify-header actions.
3331  *
3332  * @param[in] action_flags
3333  *   Holds the actions detected until now.
3334  * @param[in] action
3335  *   Pointer to the modify action.
3336  * @param[out] error
3337  *   Pointer to error structure.
3338  *
3339  * @return
3340  *   0 on success, a negative errno value otherwise and rte_errno is set.
3341  */
3342 static int
3343 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
3344                                    const struct rte_flow_action *action,
3345                                    struct rte_flow_error *error)
3346 {
3347         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
3348                 return rte_flow_error_set(error, EINVAL,
3349                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3350                                           NULL, "action configuration not set");
3351         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3352                 return rte_flow_error_set(error, EINVAL,
3353                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3354                                           "can't have encap action before"
3355                                           " modify action");
3356         return 0;
3357 }
3358
3359 /**
3360  * Validate the modify-header MAC address actions.
3361  *
3362  * @param[in] action_flags
3363  *   Holds the actions detected until now.
3364  * @param[in] action
3365  *   Pointer to the modify action.
3366  * @param[in] item_flags
3367  *   Holds the items detected.
3368  * @param[out] error
3369  *   Pointer to error structure.
3370  *
3371  * @return
3372  *   0 on success, a negative errno value otherwise and rte_errno is set.
3373  */
3374 static int
3375 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
3376                                    const struct rte_flow_action *action,
3377                                    const uint64_t item_flags,
3378                                    struct rte_flow_error *error)
3379 {
3380         int ret = 0;
3381
3382         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3383         if (!ret) {
3384                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
3385                         return rte_flow_error_set(error, EINVAL,
3386                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3387                                                   NULL,
3388                                                   "no L2 item in pattern");
3389         }
3390         return ret;
3391 }
3392
3393 /**
3394  * Validate the modify-header IPv4 address actions.
3395  *
3396  * @param[in] action_flags
3397  *   Holds the actions detected until now.
3398  * @param[in] action
3399  *   Pointer to the modify action.
3400  * @param[in] item_flags
3401  *   Holds the items detected.
3402  * @param[out] error
3403  *   Pointer to error structure.
3404  *
3405  * @return
3406  *   0 on success, a negative errno value otherwise and rte_errno is set.
3407  */
3408 static int
3409 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3410                                     const struct rte_flow_action *action,
3411                                     const uint64_t item_flags,
3412                                     struct rte_flow_error *error)
3413 {
3414         int ret = 0;
3415         uint64_t layer;
3416
3417         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3418         if (!ret) {
3419                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3420                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3421                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3422                 if (!(item_flags & layer))
3423                         return rte_flow_error_set(error, EINVAL,
3424                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3425                                                   NULL,
3426                                                   "no ipv4 item in pattern");
3427         }
3428         return ret;
3429 }
3430
3431 /**
3432  * Validate the modify-header IPv6 address actions.
3433  *
3434  * @param[in] action_flags
3435  *   Holds the actions detected until now.
3436  * @param[in] action
3437  *   Pointer to the modify action.
3438  * @param[in] item_flags
3439  *   Holds the items detected.
3440  * @param[out] error
3441  *   Pointer to error structure.
3442  *
3443  * @return
3444  *   0 on success, a negative errno value otherwise and rte_errno is set.
3445  */
3446 static int
3447 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3448                                     const struct rte_flow_action *action,
3449                                     const uint64_t item_flags,
3450                                     struct rte_flow_error *error)
3451 {
3452         int ret = 0;
3453         uint64_t layer;
3454
3455         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3456         if (!ret) {
3457                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3458                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3459                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3460                 if (!(item_flags & layer))
3461                         return rte_flow_error_set(error, EINVAL,
3462                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3463                                                   NULL,
3464                                                   "no ipv6 item in pattern");
3465         }
3466         return ret;
3467 }
3468
3469 /**
3470  * Validate the modify-header TP actions.
3471  *
3472  * @param[in] action_flags
3473  *   Holds the actions detected until now.
3474  * @param[in] action
3475  *   Pointer to the modify action.
3476  * @param[in] item_flags
3477  *   Holds the items detected.
3478  * @param[out] error
3479  *   Pointer to error structure.
3480  *
3481  * @return
3482  *   0 on success, a negative errno value otherwise and rte_errno is set.
3483  */
3484 static int
3485 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3486                                   const struct rte_flow_action *action,
3487                                   const uint64_t item_flags,
3488                                   struct rte_flow_error *error)
3489 {
3490         int ret = 0;
3491         uint64_t layer;
3492
3493         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3494         if (!ret) {
3495                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3496                                  MLX5_FLOW_LAYER_INNER_L4 :
3497                                  MLX5_FLOW_LAYER_OUTER_L4;
3498                 if (!(item_flags & layer))
3499                         return rte_flow_error_set(error, EINVAL,
3500                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3501                                                   NULL, "no transport layer "
3502                                                   "in pattern");
3503         }
3504         return ret;
3505 }
3506
3507 /**
3508  * Validate the modify-header actions of increment/decrement
3509  * TCP Sequence-number.
3510  *
3511  * @param[in] action_flags
3512  *   Holds the actions detected until now.
3513  * @param[in] action
3514  *   Pointer to the modify action.
3515  * @param[in] item_flags
3516  *   Holds the items detected.
3517  * @param[out] error
3518  *   Pointer to error structure.
3519  *
3520  * @return
3521  *   0 on success, a negative errno value otherwise and rte_errno is set.
3522  */
3523 static int
3524 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3525                                        const struct rte_flow_action *action,
3526                                        const uint64_t item_flags,
3527                                        struct rte_flow_error *error)
3528 {
3529         int ret = 0;
3530         uint64_t layer;
3531
3532         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3533         if (!ret) {
3534                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3535                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3536                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3537                 if (!(item_flags & layer))
3538                         return rte_flow_error_set(error, EINVAL,
3539                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3540                                                   NULL, "no TCP item in"
3541                                                   " pattern");
3542                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3543                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3544                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3545                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3546                         return rte_flow_error_set(error, EINVAL,
3547                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3548                                                   NULL,
3549                                                   "cannot decrease and increase"
3550                                                   " TCP sequence number"
3551                                                   " at the same time");
3552         }
3553         return ret;
3554 }
3555
3556 /**
3557  * Validate the modify-header actions of increment/decrement
3558  * TCP Acknowledgment number.
3559  *
3560  * @param[in] action_flags
3561  *   Holds the actions detected until now.
3562  * @param[in] action
3563  *   Pointer to the modify action.
3564  * @param[in] item_flags
3565  *   Holds the items detected.
3566  * @param[out] error
3567  *   Pointer to error structure.
3568  *
3569  * @return
3570  *   0 on success, a negative errno value otherwise and rte_errno is set.
3571  */
3572 static int
3573 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3574                                        const struct rte_flow_action *action,
3575                                        const uint64_t item_flags,
3576                                        struct rte_flow_error *error)
3577 {
3578         int ret = 0;
3579         uint64_t layer;
3580
3581         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3582         if (!ret) {
3583                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3584                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3585                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3586                 if (!(item_flags & layer))
3587                         return rte_flow_error_set(error, EINVAL,
3588                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3589                                                   NULL, "no TCP item in"
3590                                                   " pattern");
3591                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3592                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3593                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3594                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3595                         return rte_flow_error_set(error, EINVAL,
3596                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3597                                                   NULL,
3598                                                   "cannot decrease and increase"
3599                                                   " TCP acknowledgment number"
3600                                                   " at the same time");
3601         }
3602         return ret;
3603 }
3604
3605 /**
3606  * Validate the modify-header TTL actions.
3607  *
3608  * @param[in] action_flags
3609  *   Holds the actions detected until now.
3610  * @param[in] action
3611  *   Pointer to the modify action.
3612  * @param[in] item_flags
3613  *   Holds the items detected.
3614  * @param[out] error
3615  *   Pointer to error structure.
3616  *
3617  * @return
3618  *   0 on success, a negative errno value otherwise and rte_errno is set.
3619  */
3620 static int
3621 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3622                                    const struct rte_flow_action *action,
3623                                    const uint64_t item_flags,
3624                                    struct rte_flow_error *error)
3625 {
3626         int ret = 0;
3627         uint64_t layer;
3628
3629         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3630         if (!ret) {
3631                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3632                                  MLX5_FLOW_LAYER_INNER_L3 :
3633                                  MLX5_FLOW_LAYER_OUTER_L3;
3634                 if (!(item_flags & layer))
3635                         return rte_flow_error_set(error, EINVAL,
3636                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3637                                                   NULL,
3638                                                   "no IP protocol in pattern");
3639         }
3640         return ret;
3641 }
3642
3643 /**
3644  * Validate jump action.
3645  *
3646  * @param[in] action
3647  *   Pointer to the jump action.
3648  * @param[in] action_flags
3649  *   Holds the actions detected until now.
3650  * @param[in] attributes
3651  *   Pointer to flow attributes
3652  * @param[in] external
3653  *   Action belongs to flow rule created by request external to PMD.
3654  * @param[out] error
3655  *   Pointer to error structure.
3656  *
3657  * @return
3658  *   0 on success, a negative errno value otherwise and rte_errno is set.
3659  */
3660 static int
3661 flow_dv_validate_action_jump(const struct rte_flow_action *action,
3662                              uint64_t action_flags,
3663                              const struct rte_flow_attr *attributes,
3664                              bool external, struct rte_flow_error *error)
3665 {
3666         uint32_t target_group, table;
3667         int ret = 0;
3668
3669         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3670                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3671                 return rte_flow_error_set(error, EINVAL,
3672                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3673                                           "can't have 2 fate actions in"
3674                                           " same flow");
3675         if (action_flags & MLX5_FLOW_ACTION_METER)
3676                 return rte_flow_error_set(error, ENOTSUP,
3677                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3678                                           "jump with meter not support");
3679         if (!action->conf)
3680                 return rte_flow_error_set(error, EINVAL,
3681                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3682                                           NULL, "action configuration not set");
3683         target_group =
3684                 ((const struct rte_flow_action_jump *)action->conf)->group;
3685         ret = mlx5_flow_group_to_table(attributes, external, target_group,
3686                                        true, &table, error);
3687         if (ret)
3688                 return ret;
3689         if (attributes->group == target_group)
3690                 return rte_flow_error_set(error, EINVAL,
3691                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3692                                           "target group must be other than"
3693                                           " the current flow group");
3694         return 0;
3695 }
3696
3697 /*
3698  * Validate the port_id action.
3699  *
3700  * @param[in] dev
3701  *   Pointer to rte_eth_dev structure.
3702  * @param[in] action_flags
3703  *   Bit-fields that holds the actions detected until now.
3704  * @param[in] action
3705  *   Port_id RTE action structure.
3706  * @param[in] attr
3707  *   Attributes of flow that includes this action.
3708  * @param[out] error
3709  *   Pointer to error structure.
3710  *
3711  * @return
3712  *   0 on success, a negative errno value otherwise and rte_errno is set.
3713  */
3714 static int
3715 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
3716                                 uint64_t action_flags,
3717                                 const struct rte_flow_action *action,
3718                                 const struct rte_flow_attr *attr,
3719                                 struct rte_flow_error *error)
3720 {
3721         const struct rte_flow_action_port_id *port_id;
3722         struct mlx5_priv *act_priv;
3723         struct mlx5_priv *dev_priv;
3724         uint16_t port;
3725
3726         if (!attr->transfer)
3727                 return rte_flow_error_set(error, ENOTSUP,
3728                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3729                                           NULL,
3730                                           "port id action is valid in transfer"
3731                                           " mode only");
3732         if (!action || !action->conf)
3733                 return rte_flow_error_set(error, ENOTSUP,
3734                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3735                                           NULL,
3736                                           "port id action parameters must be"
3737                                           " specified");
3738         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3739                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3740                 return rte_flow_error_set(error, EINVAL,
3741                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3742                                           "can have only one fate actions in"
3743                                           " a flow");
3744         dev_priv = mlx5_dev_to_eswitch_info(dev);
3745         if (!dev_priv)
3746                 return rte_flow_error_set(error, rte_errno,
3747                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3748                                           NULL,
3749                                           "failed to obtain E-Switch info");
3750         port_id = action->conf;
3751         port = port_id->original ? dev->data->port_id : port_id->id;
3752         act_priv = mlx5_port_to_eswitch_info(port, false);
3753         if (!act_priv)
3754                 return rte_flow_error_set
3755                                 (error, rte_errno,
3756                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
3757                                  "failed to obtain E-Switch port id for port");
3758         if (act_priv->domain_id != dev_priv->domain_id)
3759                 return rte_flow_error_set
3760                                 (error, EINVAL,
3761                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3762                                  "port does not belong to"
3763                                  " E-Switch being configured");
3764         return 0;
3765 }
3766
3767 /**
3768  * Get the maximum number of modify header actions.
3769  *
3770  * @param dev
3771  *   Pointer to rte_eth_dev structure.
3772  * @param flags
3773  *   Flags bits to check if root level.
3774  *
3775  * @return
3776  *   Max number of modify header actions device can support.
3777  */
3778 static inline unsigned int
3779 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
3780                               uint64_t flags)
3781 {
3782         /*
3783          * There's no way to directly query the max capacity from FW.
3784          * The maximal value on root table should be assumed to be supported.
3785          */
3786         if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
3787                 return MLX5_MAX_MODIFY_NUM;
3788         else
3789                 return MLX5_ROOT_TBL_MODIFY_NUM;
3790 }
3791
3792 /**
3793  * Validate the meter action.
3794  *
3795  * @param[in] dev
3796  *   Pointer to rte_eth_dev structure.
3797  * @param[in] action_flags
3798  *   Bit-fields that holds the actions detected until now.
3799  * @param[in] action
3800  *   Pointer to the meter action.
3801  * @param[in] attr
3802  *   Attributes of flow that includes this action.
3803  * @param[out] error
3804  *   Pointer to error structure.
3805  *
3806  * @return
3807  *   0 on success, a negative errno value otherwise and rte_ernno is set.
3808  */
3809 static int
3810 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
3811                                 uint64_t action_flags,
3812                                 const struct rte_flow_action *action,
3813                                 const struct rte_flow_attr *attr,
3814                                 struct rte_flow_error *error)
3815 {
3816         struct mlx5_priv *priv = dev->data->dev_private;
3817         const struct rte_flow_action_meter *am = action->conf;
3818         struct mlx5_flow_meter *fm;
3819
3820         if (!am)
3821                 return rte_flow_error_set(error, EINVAL,
3822                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3823                                           "meter action conf is NULL");
3824
3825         if (action_flags & MLX5_FLOW_ACTION_METER)
3826                 return rte_flow_error_set(error, ENOTSUP,
3827                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3828                                           "meter chaining not support");
3829         if (action_flags & MLX5_FLOW_ACTION_JUMP)
3830                 return rte_flow_error_set(error, ENOTSUP,
3831                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3832                                           "meter with jump not support");
3833         if (!priv->mtr_en)
3834                 return rte_flow_error_set(error, ENOTSUP,
3835                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3836                                           NULL,
3837                                           "meter action not supported");
3838         fm = mlx5_flow_meter_find(priv, am->mtr_id);
3839         if (!fm)
3840                 return rte_flow_error_set(error, EINVAL,
3841                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3842                                           "Meter not found");
3843         if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
3844               (!fm->ingress && !attr->ingress && attr->egress) ||
3845               (!fm->egress && !attr->egress && attr->ingress))))
3846                 return rte_flow_error_set(error, EINVAL,
3847                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3848                                           "Flow attributes are either invalid "
3849                                           "or have a conflict with current "
3850                                           "meter attributes");
3851         return 0;
3852 }
3853
3854 /**
3855  * Validate the age action.
3856  *
3857  * @param[in] action_flags
3858  *   Holds the actions detected until now.
3859  * @param[in] action
3860  *   Pointer to the age action.
3861  * @param[in] dev
3862  *   Pointer to the Ethernet device structure.
3863  * @param[out] error
3864  *   Pointer to error structure.
3865  *
3866  * @return
3867  *   0 on success, a negative errno value otherwise and rte_errno is set.
3868  */
3869 static int
3870 flow_dv_validate_action_age(uint64_t action_flags,
3871                             const struct rte_flow_action *action,
3872                             struct rte_eth_dev *dev,
3873                             struct rte_flow_error *error)
3874 {
3875         struct mlx5_priv *priv = dev->data->dev_private;
3876         const struct rte_flow_action_age *age = action->conf;
3877
3878         if (!priv->config.devx || priv->counter_fallback)
3879                 return rte_flow_error_set(error, ENOTSUP,
3880                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3881                                           NULL,
3882                                           "age action not supported");
3883         if (!(action->conf))
3884                 return rte_flow_error_set(error, EINVAL,
3885                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3886                                           "configuration cannot be null");
3887         if (age->timeout >= UINT16_MAX / 2 / 10)
3888                 return rte_flow_error_set(error, ENOTSUP,
3889                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3890                                           "Max age time: 3275 seconds");
3891         if (action_flags & MLX5_FLOW_ACTION_AGE)
3892                 return rte_flow_error_set(error, EINVAL,
3893                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3894                                           "Duplicate age ctions set");
3895         return 0;
3896 }
3897
3898 /**
3899  * Validate the modify-header IPv4 DSCP actions.
3900  *
3901  * @param[in] action_flags
3902  *   Holds the actions detected until now.
3903  * @param[in] action
3904  *   Pointer to the modify action.
3905  * @param[in] item_flags
3906  *   Holds the items detected.
3907  * @param[out] error
3908  *   Pointer to error structure.
3909  *
3910  * @return
3911  *   0 on success, a negative errno value otherwise and rte_errno is set.
3912  */
3913 static int
3914 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
3915                                          const struct rte_flow_action *action,
3916                                          const uint64_t item_flags,
3917                                          struct rte_flow_error *error)
3918 {
3919         int ret = 0;
3920
3921         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3922         if (!ret) {
3923                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
3924                         return rte_flow_error_set(error, EINVAL,
3925                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3926                                                   NULL,
3927                                                   "no ipv4 item in pattern");
3928         }
3929         return ret;
3930 }
3931
3932 /**
3933  * Validate the modify-header IPv6 DSCP actions.
3934  *
3935  * @param[in] action_flags
3936  *   Holds the actions detected until now.
3937  * @param[in] action
3938  *   Pointer to the modify action.
3939  * @param[in] item_flags
3940  *   Holds the items detected.
3941  * @param[out] error
3942  *   Pointer to error structure.
3943  *
3944  * @return
3945  *   0 on success, a negative errno value otherwise and rte_errno is set.
3946  */
3947 static int
3948 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
3949                                          const struct rte_flow_action *action,
3950                                          const uint64_t item_flags,
3951                                          struct rte_flow_error *error)
3952 {
3953         int ret = 0;
3954
3955         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3956         if (!ret) {
3957                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
3958                         return rte_flow_error_set(error, EINVAL,
3959                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3960                                                   NULL,
3961                                                   "no ipv6 item in pattern");
3962         }
3963         return ret;
3964 }
3965
3966 /**
3967  * Find existing modify-header resource or create and register a new one.
3968  *
3969  * @param dev[in, out]
3970  *   Pointer to rte_eth_dev structure.
3971  * @param[in, out] resource
3972  *   Pointer to modify-header resource.
3973  * @parm[in, out] dev_flow
3974  *   Pointer to the dev_flow.
3975  * @param[out] error
3976  *   pointer to error structure.
3977  *
3978  * @return
3979  *   0 on success otherwise -errno and errno is set.
3980  */
3981 static int
3982 flow_dv_modify_hdr_resource_register
3983                         (struct rte_eth_dev *dev,
3984                          struct mlx5_flow_dv_modify_hdr_resource *resource,
3985                          struct mlx5_flow *dev_flow,
3986                          struct rte_flow_error *error)
3987 {
3988         struct mlx5_priv *priv = dev->data->dev_private;
3989         struct mlx5_dev_ctx_shared *sh = priv->sh;
3990         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
3991         struct mlx5dv_dr_domain *ns;
3992         uint32_t actions_len;
3993
3994         resource->flags = dev_flow->dv.group ? 0 :
3995                           MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
3996         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
3997                                     resource->flags))
3998                 return rte_flow_error_set(error, EOVERFLOW,
3999                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4000                                           "too many modify header items");
4001         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
4002                 ns = sh->fdb_domain;
4003         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
4004                 ns = sh->tx_domain;
4005         else
4006                 ns = sh->rx_domain;
4007         /* Lookup a matching resource from cache. */
4008         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4009         LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
4010                 if (resource->ft_type == cache_resource->ft_type &&
4011                     resource->actions_num == cache_resource->actions_num &&
4012                     resource->flags == cache_resource->flags &&
4013                     !memcmp((const void *)resource->actions,
4014                             (const void *)cache_resource->actions,
4015                             actions_len)) {
4016                         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
4017                                 (void *)cache_resource,
4018                                 rte_atomic32_read(&cache_resource->refcnt));
4019                         rte_atomic32_inc(&cache_resource->refcnt);
4020                         dev_flow->handle->dvh.modify_hdr = cache_resource;
4021                         return 0;
4022                 }
4023         }
4024         /* Register new modify-header resource. */
4025         cache_resource = rte_calloc(__func__, 1,
4026                                     sizeof(*cache_resource) + actions_len, 0);
4027         if (!cache_resource)
4028                 return rte_flow_error_set(error, ENOMEM,
4029                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4030                                           "cannot allocate resource memory");
4031         *cache_resource = *resource;
4032         rte_memcpy(cache_resource->actions, resource->actions, actions_len);
4033         cache_resource->verbs_action =
4034                 mlx5_glue->dv_create_flow_action_modify_header
4035                                         (sh->ctx, cache_resource->ft_type, ns,
4036                                          cache_resource->flags, actions_len,
4037                                          (uint64_t *)cache_resource->actions);
4038         if (!cache_resource->verbs_action) {
4039                 rte_free(cache_resource);
4040                 return rte_flow_error_set(error, ENOMEM,
4041                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4042                                           NULL, "cannot create action");
4043         }
4044         rte_atomic32_init(&cache_resource->refcnt);
4045         rte_atomic32_inc(&cache_resource->refcnt);
4046         LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
4047         dev_flow->handle->dvh.modify_hdr = cache_resource;
4048         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
4049                 (void *)cache_resource,
4050                 rte_atomic32_read(&cache_resource->refcnt));
4051         return 0;
4052 }
4053
4054 /**
4055  * Get DV flow counter by index.
4056  *
4057  * @param[in] dev
4058  *   Pointer to the Ethernet device structure.
4059  * @param[in] idx
4060  *   mlx5 flow counter index in the container.
4061  * @param[out] ppool
4062  *   mlx5 flow counter pool in the container,
4063  *
4064  * @return
4065  *   Pointer to the counter, NULL otherwise.
4066  */
4067 static struct mlx5_flow_counter *
4068 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
4069                            uint32_t idx,
4070                            struct mlx5_flow_counter_pool **ppool)
4071 {
4072         struct mlx5_priv *priv = dev->data->dev_private;
4073         struct mlx5_pools_container *cont;
4074         struct mlx5_flow_counter_pool *pool;
4075         uint32_t batch = 0, age = 0;
4076
4077         idx--;
4078         age = MLX_CNT_IS_AGE(idx);
4079         idx = age ? idx - MLX5_CNT_AGE_OFFSET : idx;
4080         if (idx >= MLX5_CNT_BATCH_OFFSET) {
4081                 idx -= MLX5_CNT_BATCH_OFFSET;
4082                 batch = 1;
4083         }
4084         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4085         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n);
4086         pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
4087         MLX5_ASSERT(pool);
4088         if (ppool)
4089                 *ppool = pool;
4090         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
4091 }
4092
4093 /**
4094  * Check the devx counter belongs to the pool.
4095  *
4096  * @param[in] pool
4097  *   Pointer to the counter pool.
4098  * @param[in] id
4099  *   The counter devx ID.
4100  *
4101  * @return
4102  *   True if counter belongs to the pool, false otherwise.
4103  */
4104 static bool
4105 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
4106 {
4107         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
4108                    MLX5_COUNTERS_PER_POOL;
4109
4110         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
4111                 return true;
4112         return false;
4113 }
4114
4115 /**
4116  * Get a pool by devx counter ID.
4117  *
4118  * @param[in] cont
4119  *   Pointer to the counter container.
4120  * @param[in] id
4121  *   The counter devx ID.
4122  *
4123  * @return
4124  *   The counter pool pointer if exists, NULL otherwise,
4125  */
4126 static struct mlx5_flow_counter_pool *
4127 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
4128 {
4129         uint32_t i;
4130
4131         /* Check last used pool. */
4132         if (cont->last_pool_idx != POOL_IDX_INVALID &&
4133             flow_dv_is_counter_in_pool(cont->pools[cont->last_pool_idx], id))
4134                 return cont->pools[cont->last_pool_idx];
4135         /* ID out of range means no suitable pool in the container. */
4136         if (id > cont->max_id || id < cont->min_id)
4137                 return NULL;
4138         /*
4139          * Find the pool from the end of the container, since mostly counter
4140          * ID is sequence increasing, and the last pool should be the needed
4141          * one.
4142          */
4143         i = rte_atomic16_read(&cont->n_valid);
4144         while (i--) {
4145                 struct mlx5_flow_counter_pool *pool = cont->pools[i];
4146
4147                 if (flow_dv_is_counter_in_pool(pool, id))
4148                         return pool;
4149         }
4150         return NULL;
4151 }
4152
4153 /**
4154  * Allocate a new memory for the counter values wrapped by all the needed
4155  * management.
4156  *
4157  * @param[in] dev
4158  *   Pointer to the Ethernet device structure.
4159  * @param[in] raws_n
4160  *   The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
4161  *
4162  * @return
4163  *   The new memory management pointer on success, otherwise NULL and rte_errno
4164  *   is set.
4165  */
4166 static struct mlx5_counter_stats_mem_mng *
4167 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
4168 {
4169         struct mlx5_priv *priv = dev->data->dev_private;
4170         struct mlx5_dev_ctx_shared *sh = priv->sh;
4171         struct mlx5_devx_mkey_attr mkey_attr;
4172         struct mlx5_counter_stats_mem_mng *mem_mng;
4173         volatile struct flow_counter_stats *raw_data;
4174         int size = (sizeof(struct flow_counter_stats) *
4175                         MLX5_COUNTERS_PER_POOL +
4176                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
4177                         sizeof(struct mlx5_counter_stats_mem_mng);
4178         uint8_t *mem = rte_calloc(__func__, 1, size, sysconf(_SC_PAGESIZE));
4179         int i;
4180
4181         if (!mem) {
4182                 rte_errno = ENOMEM;
4183                 return NULL;
4184         }
4185         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
4186         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
4187         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
4188                                                  IBV_ACCESS_LOCAL_WRITE);
4189         if (!mem_mng->umem) {
4190                 rte_errno = errno;
4191                 rte_free(mem);
4192                 return NULL;
4193         }
4194         mkey_attr.addr = (uintptr_t)mem;
4195         mkey_attr.size = size;
4196         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
4197         mkey_attr.pd = sh->pdn;
4198         mkey_attr.log_entity_size = 0;
4199         mkey_attr.pg_access = 0;
4200         mkey_attr.klm_array = NULL;
4201         mkey_attr.klm_num = 0;
4202         if (priv->config.hca_attr.relaxed_ordering_write &&
4203                 priv->config.hca_attr.relaxed_ordering_read  &&
4204                 !haswell_broadwell_cpu)
4205                 mkey_attr.relaxed_ordering = 1;
4206         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
4207         if (!mem_mng->dm) {
4208                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
4209                 rte_errno = errno;
4210                 rte_free(mem);
4211                 return NULL;
4212         }
4213         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
4214         raw_data = (volatile struct flow_counter_stats *)mem;
4215         for (i = 0; i < raws_n; ++i) {
4216                 mem_mng->raws[i].mem_mng = mem_mng;
4217                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
4218         }
4219         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
4220         return mem_mng;
4221 }
4222
4223 /**
4224  * Resize a counter container.
4225  *
4226  * @param[in] dev
4227  *   Pointer to the Ethernet device structure.
4228  * @param[in] batch
4229  *   Whether the pool is for counter that was allocated by batch command.
4230  * @param[in] age
4231  *   Whether the pool is for Aging counter.
4232  *
4233  * @return
4234  *   0 on success, otherwise negative errno value and rte_errno is set.
4235  */
4236 static int
4237 flow_dv_container_resize(struct rte_eth_dev *dev,
4238                                 uint32_t batch, uint32_t age)
4239 {
4240         struct mlx5_priv *priv = dev->data->dev_private;
4241         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4242                                                                age);
4243         struct mlx5_counter_stats_mem_mng *mem_mng = NULL;
4244         void *old_pools = cont->pools;
4245         uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
4246         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
4247         void *pools = rte_calloc(__func__, 1, mem_size, 0);
4248
4249         if (!pools) {
4250                 rte_errno = ENOMEM;
4251                 return -ENOMEM;
4252         }
4253         if (old_pools)
4254                 memcpy(pools, old_pools, cont->n *
4255                                        sizeof(struct mlx5_flow_counter_pool *));
4256         /*
4257          * Fallback mode query the counter directly, no background query
4258          * resources are needed.
4259          */
4260         if (!priv->counter_fallback) {
4261                 int i;
4262
4263                 mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
4264                           MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
4265                 if (!mem_mng) {
4266                         rte_free(pools);
4267                         return -ENOMEM;
4268                 }
4269                 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
4270                         LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
4271                                          mem_mng->raws +
4272                                          MLX5_CNT_CONTAINER_RESIZE +
4273                                          i, next);
4274         }
4275         rte_spinlock_lock(&cont->resize_sl);
4276         cont->n = resize;
4277         cont->mem_mng = mem_mng;
4278         cont->pools = pools;
4279         rte_spinlock_unlock(&cont->resize_sl);
4280         if (old_pools)
4281                 rte_free(old_pools);
4282         return 0;
4283 }
4284
4285 /**
4286  * Query a devx flow counter.
4287  *
4288  * @param[in] dev
4289  *   Pointer to the Ethernet device structure.
4290  * @param[in] cnt
4291  *   Index to the flow counter.
4292  * @param[out] pkts
4293  *   The statistics value of packets.
4294  * @param[out] bytes
4295  *   The statistics value of bytes.
4296  *
4297  * @return
4298  *   0 on success, otherwise a negative errno value and rte_errno is set.
4299  */
4300 static inline int
4301 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
4302                      uint64_t *bytes)
4303 {
4304         struct mlx5_priv *priv = dev->data->dev_private;
4305         struct mlx5_flow_counter_pool *pool = NULL;
4306         struct mlx5_flow_counter *cnt;
4307         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4308         int offset;
4309
4310         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4311         MLX5_ASSERT(pool);
4312         if (counter < MLX5_CNT_BATCH_OFFSET) {
4313                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4314                 if (priv->counter_fallback)
4315                         return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
4316                                         0, pkts, bytes, 0, NULL, NULL, 0);
4317         }
4318
4319         rte_spinlock_lock(&pool->sl);
4320         /*
4321          * The single counters allocation may allocate smaller ID than the
4322          * current allocated in parallel to the host reading.
4323          * In this case the new counter values must be reported as 0.
4324          */
4325         if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) {
4326                 *pkts = 0;
4327                 *bytes = 0;
4328         } else {
4329                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
4330                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
4331                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
4332         }
4333         rte_spinlock_unlock(&pool->sl);
4334         return 0;
4335 }
4336
4337 /**
4338  * Create and initialize a new counter pool.
4339  *
4340  * @param[in] dev
4341  *   Pointer to the Ethernet device structure.
4342  * @param[out] dcs
4343  *   The devX counter handle.
4344  * @param[in] batch
4345  *   Whether the pool is for counter that was allocated by batch command.
4346  * @param[in] age
4347  *   Whether the pool is for counter that was allocated for aging.
4348  * @param[in/out] cont_cur
4349  *   Pointer to the container pointer, it will be update in pool resize.
4350  *
4351  * @return
4352  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
4353  */
4354 static struct mlx5_flow_counter_pool *
4355 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
4356                     uint32_t batch, uint32_t age)
4357 {
4358         struct mlx5_priv *priv = dev->data->dev_private;
4359         struct mlx5_flow_counter_pool *pool;
4360         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4361                                                                age);
4362         int16_t n_valid = rte_atomic16_read(&cont->n_valid);
4363         uint32_t size = sizeof(*pool);
4364
4365         if (cont->n == n_valid && flow_dv_container_resize(dev, batch, age))
4366                 return NULL;
4367         size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
4368         size += (batch ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
4369         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE);
4370         pool = rte_calloc(__func__, 1, size, 0);
4371         if (!pool) {
4372                 rte_errno = ENOMEM;
4373                 return NULL;
4374         }
4375         pool->min_dcs = dcs;
4376         if (!priv->counter_fallback)
4377                 pool->raw = cont->mem_mng->raws + n_valid %
4378                                                       MLX5_CNT_CONTAINER_RESIZE;
4379         pool->raw_hw = NULL;
4380         pool->type = 0;
4381         pool->type |= (batch ? 0 :  CNT_POOL_TYPE_EXT);
4382         pool->type |= (!age ? 0 :  CNT_POOL_TYPE_AGE);
4383         pool->query_gen = 0;
4384         rte_spinlock_init(&pool->sl);
4385         TAILQ_INIT(&pool->counters[0]);
4386         TAILQ_INIT(&pool->counters[1]);
4387         TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
4388         pool->index = n_valid;
4389         cont->pools[n_valid] = pool;
4390         if (!batch) {
4391                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
4392
4393                 if (base < cont->min_id)
4394                         cont->min_id = base;
4395                 if (base > cont->max_id)
4396                         cont->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
4397                 cont->last_pool_idx = pool->index;
4398         }
4399         /* Pool initialization must be updated before host thread access. */
4400         rte_cio_wmb();
4401         rte_atomic16_add(&cont->n_valid, 1);
4402         return pool;
4403 }
4404
4405 /**
4406  * Update the minimum dcs-id for aged or no-aged counter pool.
4407  *
4408  * @param[in] dev
4409  *   Pointer to the Ethernet device structure.
4410  * @param[in] pool
4411  *   Current counter pool.
4412  * @param[in] batch
4413  *   Whether the pool is for counter that was allocated by batch command.
4414  * @param[in] age
4415  *   Whether the counter is for aging.
4416  */
4417 static void
4418 flow_dv_counter_update_min_dcs(struct rte_eth_dev *dev,
4419                         struct mlx5_flow_counter_pool *pool,
4420                         uint32_t batch, uint32_t age)
4421 {
4422         struct mlx5_priv *priv = dev->data->dev_private;
4423         struct mlx5_flow_counter_pool *other;
4424         struct mlx5_pools_container *cont;
4425
4426         cont = MLX5_CNT_CONTAINER(priv->sh, batch, (age ^ 0x1));
4427         other = flow_dv_find_pool_by_id(cont, pool->min_dcs->id);
4428         if (!other)
4429                 return;
4430         if (pool->min_dcs->id < other->min_dcs->id) {
4431                 rte_atomic64_set(&other->a64_dcs,
4432                         rte_atomic64_read(&pool->a64_dcs));
4433         } else {
4434                 rte_atomic64_set(&pool->a64_dcs,
4435                         rte_atomic64_read(&other->a64_dcs));
4436         }
4437 }
4438 /**
4439  * Prepare a new counter and/or a new counter pool.
4440  *
4441  * @param[in] dev
4442  *   Pointer to the Ethernet device structure.
4443  * @param[out] cnt_free
4444  *   Where to put the pointer of a new counter.
4445  * @param[in] batch
4446  *   Whether the pool is for counter that was allocated by batch command.
4447  * @param[in] age
4448  *   Whether the pool is for counter that was allocated for aging.
4449  *
4450  * @return
4451  *   The counter pool pointer and @p cnt_free is set on success,
4452  *   NULL otherwise and rte_errno is set.
4453  */
4454 static struct mlx5_flow_counter_pool *
4455 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
4456                              struct mlx5_flow_counter **cnt_free,
4457                              uint32_t batch, uint32_t age)
4458 {
4459         struct mlx5_priv *priv = dev->data->dev_private;
4460         struct mlx5_pools_container *cont;
4461         struct mlx5_flow_counter_pool *pool;
4462         struct mlx5_counters tmp_tq;
4463         struct mlx5_devx_obj *dcs = NULL;
4464         struct mlx5_flow_counter *cnt;
4465         uint32_t i;
4466
4467         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4468         if (!batch) {
4469                 /* bulk_bitmap must be 0 for single counter allocation. */
4470                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
4471                 if (!dcs)
4472                         return NULL;
4473                 pool = flow_dv_find_pool_by_id(cont, dcs->id);
4474                 if (!pool) {
4475                         pool = flow_dv_pool_create(dev, dcs, batch, age);
4476                         if (!pool) {
4477                                 mlx5_devx_cmd_destroy(dcs);
4478                                 return NULL;
4479                         }
4480                 } else if (dcs->id < pool->min_dcs->id) {
4481                         rte_atomic64_set(&pool->a64_dcs,
4482                                          (int64_t)(uintptr_t)dcs);
4483                 }
4484                 flow_dv_counter_update_min_dcs(dev,
4485                                                 pool, batch, age);
4486                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
4487                 cnt = MLX5_POOL_GET_CNT(pool, i);
4488                 cnt->pool = pool;
4489                 MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs;
4490                 *cnt_free = cnt;
4491                 return pool;
4492         }
4493         /* bulk_bitmap is in 128 counters units. */
4494         if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
4495                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
4496         if (!dcs) {
4497                 rte_errno = ENODATA;
4498                 return NULL;
4499         }
4500         pool = flow_dv_pool_create(dev, dcs, batch, age);
4501         if (!pool) {
4502                 mlx5_devx_cmd_destroy(dcs);
4503                 return NULL;
4504         }
4505         TAILQ_INIT(&tmp_tq);
4506         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
4507                 cnt = MLX5_POOL_GET_CNT(pool, i);
4508                 cnt->pool = pool;
4509                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
4510         }
4511         rte_spinlock_lock(&cont->csl);
4512         TAILQ_CONCAT(&cont->counters, &tmp_tq, next);
4513         rte_spinlock_unlock(&cont->csl);
4514         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
4515         (*cnt_free)->pool = pool;
4516         return pool;
4517 }
4518
4519 /**
4520  * Search for existed shared counter.
4521  *
4522  * @param[in] dev
4523  *   Pointer to the Ethernet device structure.
4524  * @param[in] id
4525  *   The shared counter ID to search.
4526  * @param[out] ppool
4527  *   mlx5 flow counter pool in the container,
4528  *
4529  * @return
4530  *   NULL if not existed, otherwise pointer to the shared extend counter.
4531  */
4532 static struct mlx5_flow_counter_ext *
4533 flow_dv_counter_shared_search(struct rte_eth_dev *dev, uint32_t id,
4534                               struct mlx5_flow_counter_pool **ppool)
4535 {
4536         struct mlx5_priv *priv = dev->data->dev_private;
4537         union mlx5_l3t_data data;
4538         uint32_t cnt_idx;
4539
4540         if (mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) || !data.dword)
4541                 return NULL;
4542         cnt_idx = data.dword;
4543         /*
4544          * Shared counters don't have age info. The counter extend is after
4545          * the counter datat structure.
4546          */
4547         return (struct mlx5_flow_counter_ext *)
4548                ((flow_dv_counter_get_by_idx(dev, cnt_idx, ppool)) + 1);
4549 }
4550
4551 /**
4552  * Allocate a flow counter.
4553  *
4554  * @param[in] dev
4555  *   Pointer to the Ethernet device structure.
4556  * @param[in] shared
4557  *   Indicate if this counter is shared with other flows.
4558  * @param[in] id
4559  *   Counter identifier.
4560  * @param[in] group
4561  *   Counter flow group.
4562  * @param[in] age
4563  *   Whether the counter was allocated for aging.
4564  *
4565  * @return
4566  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
4567  */
4568 static uint32_t
4569 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
4570                       uint16_t group, uint32_t age)
4571 {
4572         struct mlx5_priv *priv = dev->data->dev_private;
4573         struct mlx5_flow_counter_pool *pool = NULL;
4574         struct mlx5_flow_counter *cnt_free = NULL;
4575         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4576         /*
4577          * Currently group 0 flow counter cannot be assigned to a flow if it is
4578          * not the first one in the batch counter allocation, so it is better
4579          * to allocate counters one by one for these flows in a separate
4580          * container.
4581          * A counter can be shared between different groups so need to take
4582          * shared counters from the single container.
4583          */
4584         uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0;
4585         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4586                                                                age);
4587         uint32_t cnt_idx;
4588
4589         if (!priv->config.devx) {
4590                 rte_errno = ENOTSUP;
4591                 return 0;
4592         }
4593         if (shared) {
4594                 cnt_ext = flow_dv_counter_shared_search(dev, id, &pool);
4595                 if (cnt_ext) {
4596                         if (cnt_ext->ref_cnt + 1 == 0) {
4597                                 rte_errno = E2BIG;
4598                                 return 0;
4599                         }
4600                         cnt_ext->ref_cnt++;
4601                         cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL +
4602                                   (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL)
4603                                   + 1;
4604                         return cnt_idx;
4605                 }
4606         }
4607         /* Get free counters from container. */
4608         rte_spinlock_lock(&cont->csl);
4609         cnt_free = TAILQ_FIRST(&cont->counters);
4610         if (cnt_free)
4611                 TAILQ_REMOVE(&cont->counters, cnt_free, next);
4612         rte_spinlock_unlock(&cont->csl);
4613         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free,
4614                                                        batch, age))
4615                 goto err;
4616         pool = cnt_free->pool;
4617         if (!batch)
4618                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free);
4619         /* Create a DV counter action only in the first time usage. */
4620         if (!cnt_free->action) {
4621                 uint16_t offset;
4622                 struct mlx5_devx_obj *dcs;
4623
4624                 if (batch) {
4625                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
4626                         dcs = pool->min_dcs;
4627                 } else {
4628                         offset = 0;
4629                         dcs = cnt_ext->dcs;
4630                 }
4631                 cnt_free->action = mlx5_glue->dv_create_flow_action_counter
4632                                         (dcs->obj, offset);
4633                 if (!cnt_free->action) {
4634                         rte_errno = errno;
4635                         goto err;
4636                 }
4637         }
4638         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
4639                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
4640         cnt_idx += batch * MLX5_CNT_BATCH_OFFSET;
4641         cnt_idx += age * MLX5_CNT_AGE_OFFSET;
4642         /* Update the counter reset values. */
4643         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
4644                                  &cnt_free->bytes))
4645                 goto err;
4646         if (cnt_ext) {
4647                 cnt_ext->shared = shared;
4648                 cnt_ext->ref_cnt = 1;
4649                 cnt_ext->id = id;
4650                 if (shared) {
4651                         union mlx5_l3t_data data;
4652
4653                         data.dword = cnt_idx;
4654                         if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
4655                                 return 0;
4656                 }
4657         }
4658         if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on)
4659                 /* Start the asynchronous batch query by the host thread. */
4660                 mlx5_set_query_alarm(priv->sh);
4661         return cnt_idx;
4662 err:
4663         if (cnt_free) {
4664                 cnt_free->pool = pool;
4665                 rte_spinlock_lock(&cont->csl);
4666                 TAILQ_INSERT_TAIL(&cont->counters, cnt_free, next);
4667                 rte_spinlock_unlock(&cont->csl);
4668         }
4669         return 0;
4670 }
4671
4672 /**
4673  * Get age param from counter index.
4674  *
4675  * @param[in] dev
4676  *   Pointer to the Ethernet device structure.
4677  * @param[in] counter
4678  *   Index to the counter handler.
4679  *
4680  * @return
4681  *   The aging parameter specified for the counter index.
4682  */
4683 static struct mlx5_age_param*
4684 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
4685                                 uint32_t counter)
4686 {
4687         struct mlx5_flow_counter *cnt;
4688         struct mlx5_flow_counter_pool *pool = NULL;
4689
4690         flow_dv_counter_get_by_idx(dev, counter, &pool);
4691         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
4692         cnt = MLX5_POOL_GET_CNT(pool, counter);
4693         return MLX5_CNT_TO_AGE(cnt);
4694 }
4695
4696 /**
4697  * Remove a flow counter from aged counter list.
4698  *
4699  * @param[in] dev
4700  *   Pointer to the Ethernet device structure.
4701  * @param[in] counter
4702  *   Index to the counter handler.
4703  * @param[in] cnt
4704  *   Pointer to the counter handler.
4705  */
4706 static void
4707 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
4708                                 uint32_t counter, struct mlx5_flow_counter *cnt)
4709 {
4710         struct mlx5_age_info *age_info;
4711         struct mlx5_age_param *age_param;
4712         struct mlx5_priv *priv = dev->data->dev_private;
4713
4714         age_info = GET_PORT_AGE_INFO(priv);
4715         age_param = flow_dv_counter_idx_get_age(dev, counter);
4716         if (rte_atomic16_cmpset((volatile uint16_t *)
4717                         &age_param->state,
4718                         AGE_CANDIDATE, AGE_FREE)
4719                         != AGE_CANDIDATE) {
4720                 /**
4721                  * We need the lock even it is age timeout,
4722                  * since counter may still in process.
4723                  */
4724                 rte_spinlock_lock(&age_info->aged_sl);
4725                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
4726                 rte_spinlock_unlock(&age_info->aged_sl);
4727         }
4728         rte_atomic16_set(&age_param->state, AGE_FREE);
4729 }
4730 /**
4731  * Release a flow counter.
4732  *
4733  * @param[in] dev
4734  *   Pointer to the Ethernet device structure.
4735  * @param[in] counter
4736  *   Index to the counter handler.
4737  */
4738 static void
4739 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
4740 {
4741         struct mlx5_priv *priv = dev->data->dev_private;
4742         struct mlx5_flow_counter_pool *pool = NULL;
4743         struct mlx5_flow_counter *cnt;
4744         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4745
4746         if (!counter)
4747                 return;
4748         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4749         MLX5_ASSERT(pool);
4750         if (counter < MLX5_CNT_BATCH_OFFSET) {
4751                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4752                 if (cnt_ext) {
4753                         if (--cnt_ext->ref_cnt)
4754                                 return;
4755                         if (cnt_ext->shared)
4756                                 mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
4757                                                      cnt_ext->id);
4758                 }
4759         }
4760         if (IS_AGE_POOL(pool))
4761                 flow_dv_counter_remove_from_age(dev, counter, cnt);
4762         cnt->pool = pool;
4763         /*
4764          * Put the counter back to list to be updated in none fallback mode.
4765          * Currently, we are using two list alternately, while one is in query,
4766          * add the freed counter to the other list based on the pool query_gen
4767          * value. After query finishes, add counter the list to the global
4768          * container counter list. The list changes while query starts. In
4769          * this case, lock will not be needed as query callback and release
4770          * function both operate with the different list.
4771          *
4772          */
4773         if (!priv->counter_fallback)
4774                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
4775         else
4776                 TAILQ_INSERT_TAIL(&((MLX5_CNT_CONTAINER
4777                                   (priv->sh, 0, 0))->counters),
4778                                   cnt, next);
4779 }
4780
4781 /**
4782  * Verify the @p attributes will be correctly understood by the NIC and store
4783  * them in the @p flow if everything is correct.
4784  *
4785  * @param[in] dev
4786  *   Pointer to dev struct.
4787  * @param[in] attributes
4788  *   Pointer to flow attributes
4789  * @param[in] external
4790  *   This flow rule is created by request external to PMD.
4791  * @param[out] error
4792  *   Pointer to error structure.
4793  *
4794  * @return
4795  *   - 0 on success and non root table.
4796  *   - 1 on success and root table.
4797  *   - a negative errno value otherwise and rte_errno is set.
4798  */
4799 static int
4800 flow_dv_validate_attributes(struct rte_eth_dev *dev,
4801                             const struct rte_flow_attr *attributes,
4802                             bool external __rte_unused,
4803                             struct rte_flow_error *error)
4804 {
4805         struct mlx5_priv *priv = dev->data->dev_private;
4806         uint32_t priority_max = priv->config.flow_prio - 1;
4807         int ret = 0;
4808
4809 #ifndef HAVE_MLX5DV_DR
4810         if (attributes->group)
4811                 return rte_flow_error_set(error, ENOTSUP,
4812                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
4813                                           NULL,
4814                                           "groups are not supported");
4815 #else
4816         uint32_t table = 0;
4817
4818         ret = mlx5_flow_group_to_table(attributes, external,
4819                                        attributes->group, !!priv->fdb_def_rule,
4820                                        &table, error);
4821         if (ret)
4822                 return ret;
4823         if (!table)
4824                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4825 #endif
4826         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
4827             attributes->priority >= priority_max)
4828                 return rte_flow_error_set(error, ENOTSUP,
4829                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
4830                                           NULL,
4831                                           "priority out of range");
4832         if (attributes->transfer) {
4833                 if (!priv->config.dv_esw_en)
4834                         return rte_flow_error_set
4835                                 (error, ENOTSUP,
4836                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4837                                  "E-Switch dr is not supported");
4838                 if (!(priv->representor || priv->master))
4839                         return rte_flow_error_set
4840                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4841                                  NULL, "E-Switch configuration can only be"
4842                                  " done by a master or a representor device");
4843                 if (attributes->egress)
4844                         return rte_flow_error_set
4845                                 (error, ENOTSUP,
4846                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
4847                                  "egress is not supported");
4848         }
4849         if (!(attributes->egress ^ attributes->ingress))
4850                 return rte_flow_error_set(error, ENOTSUP,
4851                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
4852                                           "must specify exactly one of "
4853                                           "ingress or egress");
4854         return ret;
4855 }
4856
4857 /**
4858  * Internal validation function. For validating both actions and items.
4859  *
4860  * @param[in] dev
4861  *   Pointer to the rte_eth_dev structure.
4862  * @param[in] attr
4863  *   Pointer to the flow attributes.
4864  * @param[in] items
4865  *   Pointer to the list of items.
4866  * @param[in] actions
4867  *   Pointer to the list of actions.
4868  * @param[in] external
4869  *   This flow rule is created by request external to PMD.
4870  * @param[in] hairpin
4871  *   Number of hairpin TX actions, 0 means classic flow.
4872  * @param[out] error
4873  *   Pointer to the error structure.
4874  *
4875  * @return
4876  *   0 on success, a negative errno value otherwise and rte_errno is set.
4877  */
4878 static int
4879 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
4880                  const struct rte_flow_item items[],
4881                  const struct rte_flow_action actions[],
4882                  bool external, int hairpin, struct rte_flow_error *error)
4883 {
4884         int ret;
4885         uint64_t action_flags = 0;
4886         uint64_t item_flags = 0;
4887         uint64_t last_item = 0;
4888         uint8_t next_protocol = 0xff;
4889         uint16_t ether_type = 0;
4890         int actions_n = 0;
4891         uint8_t item_ipv6_proto = 0;
4892         const struct rte_flow_item *gre_item = NULL;
4893         const struct rte_flow_action_raw_decap *decap;
4894         const struct rte_flow_action_raw_encap *encap;
4895         const struct rte_flow_action_rss *rss;
4896         const struct rte_flow_item_tcp nic_tcp_mask = {
4897                 .hdr = {
4898                         .tcp_flags = 0xFF,
4899                         .src_port = RTE_BE16(UINT16_MAX),
4900                         .dst_port = RTE_BE16(UINT16_MAX),
4901                 }
4902         };
4903         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
4904                 .hdr = {
4905                         .src_addr = RTE_BE32(0xffffffff),
4906                         .dst_addr = RTE_BE32(0xffffffff),
4907                         .type_of_service = 0xff,
4908                         .next_proto_id = 0xff,
4909                         .time_to_live = 0xff,
4910                 },
4911         };
4912         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
4913                 .hdr = {
4914                         .src_addr =
4915                         "\xff\xff\xff\xff\xff\xff\xff\xff"
4916                         "\xff\xff\xff\xff\xff\xff\xff\xff",
4917                         .dst_addr =
4918                         "\xff\xff\xff\xff\xff\xff\xff\xff"
4919                         "\xff\xff\xff\xff\xff\xff\xff\xff",
4920                         .vtc_flow = RTE_BE32(0xffffffff),
4921                         .proto = 0xff,
4922                         .hop_limits = 0xff,
4923                 },
4924         };
4925         struct mlx5_priv *priv = dev->data->dev_private;
4926         struct mlx5_dev_config *dev_conf = &priv->config;
4927         uint16_t queue_index = 0xFFFF;
4928         const struct rte_flow_item_vlan *vlan_m = NULL;
4929         int16_t rw_act_num = 0;
4930         uint64_t is_root;
4931
4932         if (items == NULL)
4933                 return -1;
4934         ret = flow_dv_validate_attributes(dev, attr, external, error);
4935         if (ret < 0)
4936                 return ret;
4937         is_root = (uint64_t)ret;
4938         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4939                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
4940                 int type = items->type;
4941
4942                 switch (type) {
4943                 case RTE_FLOW_ITEM_TYPE_VOID:
4944                         break;
4945                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4946                         ret = flow_dv_validate_item_port_id
4947                                         (dev, items, attr, item_flags, error);
4948                         if (ret < 0)
4949                                 return ret;
4950                         last_item = MLX5_FLOW_ITEM_PORT_ID;
4951                         break;
4952                 case RTE_FLOW_ITEM_TYPE_ETH:
4953                         ret = mlx5_flow_validate_item_eth(items, item_flags,
4954                                                           error);
4955                         if (ret < 0)
4956                                 return ret;
4957                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
4958                                              MLX5_FLOW_LAYER_OUTER_L2;
4959                         if (items->mask != NULL && items->spec != NULL) {
4960                                 ether_type =
4961                                         ((const struct rte_flow_item_eth *)
4962                                          items->spec)->type;
4963                                 ether_type &=
4964                                         ((const struct rte_flow_item_eth *)
4965                                          items->mask)->type;
4966                                 ether_type = rte_be_to_cpu_16(ether_type);
4967                         } else {
4968                                 ether_type = 0;
4969                         }
4970                         break;
4971                 case RTE_FLOW_ITEM_TYPE_VLAN:
4972                         ret = flow_dv_validate_item_vlan(items, item_flags,
4973                                                          dev, error);
4974                         if (ret < 0)
4975                                 return ret;
4976                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
4977                                              MLX5_FLOW_LAYER_OUTER_VLAN;
4978                         if (items->mask != NULL && items->spec != NULL) {
4979                                 ether_type =
4980                                         ((const struct rte_flow_item_vlan *)
4981                                          items->spec)->inner_type;
4982                                 ether_type &=
4983                                         ((const struct rte_flow_item_vlan *)
4984                                          items->mask)->inner_type;
4985                                 ether_type = rte_be_to_cpu_16(ether_type);
4986                         } else {
4987                                 ether_type = 0;
4988                         }
4989                         /* Store outer VLAN mask for of_push_vlan action. */
4990                         if (!tunnel)
4991                                 vlan_m = items->mask;
4992                         break;
4993                 case RTE_FLOW_ITEM_TYPE_IPV4:
4994                         mlx5_flow_tunnel_ip_check(items, next_protocol,
4995                                                   &item_flags, &tunnel);
4996                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
4997                                                            last_item,
4998                                                            ether_type,
4999                                                            &nic_ipv4_mask,
5000                                                            error);
5001                         if (ret < 0)
5002                                 return ret;
5003                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5004                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5005                         if (items->mask != NULL &&
5006                             ((const struct rte_flow_item_ipv4 *)
5007                              items->mask)->hdr.next_proto_id) {
5008                                 next_protocol =
5009                                         ((const struct rte_flow_item_ipv4 *)
5010                                          (items->spec))->hdr.next_proto_id;
5011                                 next_protocol &=
5012                                         ((const struct rte_flow_item_ipv4 *)
5013                                          (items->mask))->hdr.next_proto_id;
5014                         } else {
5015                                 /* Reset for inner layer. */
5016                                 next_protocol = 0xff;
5017                         }
5018                         break;
5019                 case RTE_FLOW_ITEM_TYPE_IPV6:
5020                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5021                                                   &item_flags, &tunnel);
5022                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
5023                                                            last_item,
5024                                                            ether_type,
5025                                                            &nic_ipv6_mask,
5026                                                            error);
5027                         if (ret < 0)
5028                                 return ret;
5029                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5030                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5031                         if (items->mask != NULL &&
5032                             ((const struct rte_flow_item_ipv6 *)
5033                              items->mask)->hdr.proto) {
5034                                 item_ipv6_proto =
5035                                         ((const struct rte_flow_item_ipv6 *)
5036                                          items->spec)->hdr.proto;
5037                                 next_protocol =
5038                                         ((const struct rte_flow_item_ipv6 *)
5039                                          items->spec)->hdr.proto;
5040                                 next_protocol &=
5041                                         ((const struct rte_flow_item_ipv6 *)
5042                                          items->mask)->hdr.proto;
5043                         } else {
5044                                 /* Reset for inner layer. */
5045                                 next_protocol = 0xff;
5046                         }
5047                         break;
5048                 case RTE_FLOW_ITEM_TYPE_TCP:
5049                         ret = mlx5_flow_validate_item_tcp
5050                                                 (items, item_flags,
5051                                                  next_protocol,
5052                                                  &nic_tcp_mask,
5053                                                  error);
5054                         if (ret < 0)
5055                                 return ret;
5056                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5057                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5058                         break;
5059                 case RTE_FLOW_ITEM_TYPE_UDP:
5060                         ret = mlx5_flow_validate_item_udp(items, item_flags,
5061                                                           next_protocol,
5062                                                           error);
5063                         if (ret < 0)
5064                                 return ret;
5065                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5066                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5067                         break;
5068                 case RTE_FLOW_ITEM_TYPE_GRE:
5069                         ret = mlx5_flow_validate_item_gre(items, item_flags,
5070                                                           next_protocol, error);
5071                         if (ret < 0)
5072                                 return ret;
5073                         gre_item = items;
5074                         last_item = MLX5_FLOW_LAYER_GRE;
5075                         break;
5076                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5077                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
5078                                                             next_protocol,
5079                                                             error);
5080                         if (ret < 0)
5081                                 return ret;
5082                         last_item = MLX5_FLOW_LAYER_NVGRE;
5083                         break;
5084                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5085                         ret = mlx5_flow_validate_item_gre_key
5086                                 (items, item_flags, gre_item, error);
5087                         if (ret < 0)
5088                                 return ret;
5089                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5090                         break;
5091                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5092                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
5093                                                             error);
5094                         if (ret < 0)
5095                                 return ret;
5096                         last_item = MLX5_FLOW_LAYER_VXLAN;
5097                         break;
5098                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5099                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
5100                                                                 item_flags, dev,
5101                                                                 error);
5102                         if (ret < 0)
5103                                 return ret;
5104                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5105                         break;
5106                 case RTE_FLOW_ITEM_TYPE_GENEVE:
5107                         ret = mlx5_flow_validate_item_geneve(items,
5108                                                              item_flags, dev,
5109                                                              error);
5110                         if (ret < 0)
5111                                 return ret;
5112                         last_item = MLX5_FLOW_LAYER_GENEVE;
5113                         break;
5114                 case RTE_FLOW_ITEM_TYPE_MPLS:
5115                         ret = mlx5_flow_validate_item_mpls(dev, items,
5116                                                            item_flags,
5117                                                            last_item, error);
5118                         if (ret < 0)
5119                                 return ret;
5120                         last_item = MLX5_FLOW_LAYER_MPLS;
5121                         break;
5122
5123                 case RTE_FLOW_ITEM_TYPE_MARK:
5124                         ret = flow_dv_validate_item_mark(dev, items, attr,
5125                                                          error);
5126                         if (ret < 0)
5127                                 return ret;
5128                         last_item = MLX5_FLOW_ITEM_MARK;
5129                         break;
5130                 case RTE_FLOW_ITEM_TYPE_META:
5131                         ret = flow_dv_validate_item_meta(dev, items, attr,
5132                                                          error);
5133                         if (ret < 0)
5134                                 return ret;
5135                         last_item = MLX5_FLOW_ITEM_METADATA;
5136                         break;
5137                 case RTE_FLOW_ITEM_TYPE_ICMP:
5138                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
5139                                                            next_protocol,
5140                                                            error);
5141                         if (ret < 0)
5142                                 return ret;
5143                         last_item = MLX5_FLOW_LAYER_ICMP;
5144                         break;
5145                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5146                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
5147                                                             next_protocol,
5148                                                             error);
5149                         if (ret < 0)
5150                                 return ret;
5151                         item_ipv6_proto = IPPROTO_ICMPV6;
5152                         last_item = MLX5_FLOW_LAYER_ICMP6;
5153                         break;
5154                 case RTE_FLOW_ITEM_TYPE_TAG:
5155                         ret = flow_dv_validate_item_tag(dev, items,
5156                                                         attr, error);
5157                         if (ret < 0)
5158                                 return ret;
5159                         last_item = MLX5_FLOW_ITEM_TAG;
5160                         break;
5161                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
5162                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
5163                         break;
5164                 case RTE_FLOW_ITEM_TYPE_GTP:
5165                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
5166                                                         error);
5167                         if (ret < 0)
5168                                 return ret;
5169                         last_item = MLX5_FLOW_LAYER_GTP;
5170                         break;
5171                 default:
5172                         return rte_flow_error_set(error, ENOTSUP,
5173                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5174                                                   NULL, "item not supported");
5175                 }
5176                 item_flags |= last_item;
5177         }
5178         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5179                 int type = actions->type;
5180                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5181                         return rte_flow_error_set(error, ENOTSUP,
5182                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5183                                                   actions, "too many actions");
5184                 switch (type) {
5185                 case RTE_FLOW_ACTION_TYPE_VOID:
5186                         break;
5187                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5188                         ret = flow_dv_validate_action_port_id(dev,
5189                                                               action_flags,
5190                                                               actions,
5191                                                               attr,
5192                                                               error);
5193                         if (ret)
5194                                 return ret;
5195                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5196                         ++actions_n;
5197                         break;
5198                 case RTE_FLOW_ACTION_TYPE_FLAG:
5199                         ret = flow_dv_validate_action_flag(dev, action_flags,
5200                                                            attr, error);
5201                         if (ret < 0)
5202                                 return ret;
5203                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5204                                 /* Count all modify-header actions as one. */
5205                                 if (!(action_flags &
5206                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5207                                         ++actions_n;
5208                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
5209                                                 MLX5_FLOW_ACTION_MARK_EXT;
5210                         } else {
5211                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
5212                                 ++actions_n;
5213                         }
5214                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5215                         break;
5216                 case RTE_FLOW_ACTION_TYPE_MARK:
5217                         ret = flow_dv_validate_action_mark(dev, actions,
5218                                                            action_flags,
5219                                                            attr, error);
5220                         if (ret < 0)
5221                                 return ret;
5222                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5223                                 /* Count all modify-header actions as one. */
5224                                 if (!(action_flags &
5225                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5226                                         ++actions_n;
5227                                 action_flags |= MLX5_FLOW_ACTION_MARK |
5228                                                 MLX5_FLOW_ACTION_MARK_EXT;
5229                         } else {
5230                                 action_flags |= MLX5_FLOW_ACTION_MARK;
5231                                 ++actions_n;
5232                         }
5233                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5234                         break;
5235                 case RTE_FLOW_ACTION_TYPE_SET_META:
5236                         ret = flow_dv_validate_action_set_meta(dev, actions,
5237                                                                action_flags,
5238                                                                attr, error);
5239                         if (ret < 0)
5240                                 return ret;
5241                         /* Count all modify-header actions as one action. */
5242                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5243                                 ++actions_n;
5244                         action_flags |= MLX5_FLOW_ACTION_SET_META;
5245                         rw_act_num += MLX5_ACT_NUM_SET_META;
5246                         break;
5247                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5248                         ret = flow_dv_validate_action_set_tag(dev, actions,
5249                                                               action_flags,
5250                                                               attr, error);
5251                         if (ret < 0)
5252                                 return ret;
5253                         /* Count all modify-header actions as one action. */
5254                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5255                                 ++actions_n;
5256                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
5257                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5258                         break;
5259                 case RTE_FLOW_ACTION_TYPE_DROP:
5260                         ret = mlx5_flow_validate_action_drop(action_flags,
5261                                                              attr, error);
5262                         if (ret < 0)
5263                                 return ret;
5264                         action_flags |= MLX5_FLOW_ACTION_DROP;
5265                         ++actions_n;
5266                         break;
5267                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5268                         ret = mlx5_flow_validate_action_queue(actions,
5269                                                               action_flags, dev,
5270                                                               attr, error);
5271                         if (ret < 0)
5272                                 return ret;
5273                         queue_index = ((const struct rte_flow_action_queue *)
5274                                                         (actions->conf))->index;
5275                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
5276                         ++actions_n;
5277                         break;
5278                 case RTE_FLOW_ACTION_TYPE_RSS:
5279                         rss = actions->conf;
5280                         ret = mlx5_flow_validate_action_rss(actions,
5281                                                             action_flags, dev,
5282                                                             attr, item_flags,
5283                                                             error);
5284                         if (ret < 0)
5285                                 return ret;
5286                         if (rss != NULL && rss->queue_num)
5287                                 queue_index = rss->queue[0];
5288                         action_flags |= MLX5_FLOW_ACTION_RSS;
5289                         ++actions_n;
5290                         break;
5291                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
5292                         ret =
5293                         mlx5_flow_validate_action_default_miss(action_flags,
5294                                         attr, error);
5295                         if (ret < 0)
5296                                 return ret;
5297                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
5298                         ++actions_n;
5299                         break;
5300                 case RTE_FLOW_ACTION_TYPE_COUNT:
5301                         ret = flow_dv_validate_action_count(dev, error);
5302                         if (ret < 0)
5303                                 return ret;
5304                         action_flags |= MLX5_FLOW_ACTION_COUNT;
5305                         ++actions_n;
5306                         break;
5307                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5308                         if (flow_dv_validate_action_pop_vlan(dev,
5309                                                              action_flags,
5310                                                              actions,
5311                                                              item_flags, attr,
5312                                                              error))
5313                                 return -rte_errno;
5314                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5315                         ++actions_n;
5316                         break;
5317                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5318                         ret = flow_dv_validate_action_push_vlan(dev,
5319                                                                 action_flags,
5320                                                                 vlan_m,
5321                                                                 actions, attr,
5322                                                                 error);
5323                         if (ret < 0)
5324                                 return ret;
5325                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5326                         ++actions_n;
5327                         break;
5328                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5329                         ret = flow_dv_validate_action_set_vlan_pcp
5330                                                 (action_flags, actions, error);
5331                         if (ret < 0)
5332                                 return ret;
5333                         /* Count PCP with push_vlan command. */
5334                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
5335                         break;
5336                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5337                         ret = flow_dv_validate_action_set_vlan_vid
5338                                                 (item_flags, action_flags,
5339                                                  actions, error);
5340                         if (ret < 0)
5341                                 return ret;
5342                         /* Count VID with push_vlan command. */
5343                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
5344                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
5345                         break;
5346                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5347                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5348                         ret = flow_dv_validate_action_l2_encap(dev,
5349                                                                action_flags,
5350                                                                actions, attr,
5351                                                                error);
5352                         if (ret < 0)
5353                                 return ret;
5354                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
5355                         ++actions_n;
5356                         break;
5357                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5358                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5359                         ret = flow_dv_validate_action_decap(dev, action_flags,
5360                                                             attr, error);
5361                         if (ret < 0)
5362                                 return ret;
5363                         action_flags |= MLX5_FLOW_ACTION_DECAP;
5364                         ++actions_n;
5365                         break;
5366                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5367                         ret = flow_dv_validate_action_raw_encap_decap
5368                                 (dev, NULL, actions->conf, attr, &action_flags,
5369                                  &actions_n, error);
5370                         if (ret < 0)
5371                                 return ret;
5372                         break;
5373                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5374                         decap = actions->conf;
5375                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5376                                 ;
5377                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5378                                 encap = NULL;
5379                                 actions--;
5380                         } else {
5381                                 encap = actions->conf;
5382                         }
5383                         ret = flow_dv_validate_action_raw_encap_decap
5384                                            (dev,
5385                                             decap ? decap : &empty_decap, encap,
5386                                             attr, &action_flags, &actions_n,
5387                                             error);
5388                         if (ret < 0)
5389                                 return ret;
5390                         break;
5391                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5392                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5393                         ret = flow_dv_validate_action_modify_mac(action_flags,
5394                                                                  actions,
5395                                                                  item_flags,
5396                                                                  error);
5397                         if (ret < 0)
5398                                 return ret;
5399                         /* Count all modify-header actions as one action. */
5400                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5401                                 ++actions_n;
5402                         action_flags |= actions->type ==
5403                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5404                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
5405                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
5406                         /*
5407                          * Even if the source and destination MAC addresses have
5408                          * overlap in the header with 4B alignment, the convert
5409                          * function will handle them separately and 4 SW actions
5410                          * will be created. And 2 actions will be added each
5411                          * time no matter how many bytes of address will be set.
5412                          */
5413                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
5414                         break;
5415                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5416                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5417                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
5418                                                                   actions,
5419                                                                   item_flags,
5420                                                                   error);
5421                         if (ret < 0)
5422                                 return ret;
5423                         /* Count all modify-header actions as one action. */
5424                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5425                                 ++actions_n;
5426                         action_flags |= actions->type ==
5427                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5428                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
5429                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
5430                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
5431                         break;
5432                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5433                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5434                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
5435                                                                   actions,
5436                                                                   item_flags,
5437                                                                   error);
5438                         if (ret < 0)
5439                                 return ret;
5440                         if (item_ipv6_proto == IPPROTO_ICMPV6)
5441                                 return rte_flow_error_set(error, ENOTSUP,
5442                                         RTE_FLOW_ERROR_TYPE_ACTION,
5443                                         actions,
5444                                         "Can't change header "
5445                                         "with ICMPv6 proto");
5446                         /* Count all modify-header actions as one action. */
5447                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5448                                 ++actions_n;
5449                         action_flags |= actions->type ==
5450                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5451                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
5452                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
5453                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
5454                         break;
5455                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5456                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5457                         ret = flow_dv_validate_action_modify_tp(action_flags,
5458                                                                 actions,
5459                                                                 item_flags,
5460                                                                 error);
5461                         if (ret < 0)
5462                                 return ret;
5463                         /* Count all modify-header actions as one action. */
5464                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5465                                 ++actions_n;
5466                         action_flags |= actions->type ==
5467                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5468                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
5469                                                 MLX5_FLOW_ACTION_SET_TP_DST;
5470                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
5471                         break;
5472                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5473                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5474                         ret = flow_dv_validate_action_modify_ttl(action_flags,
5475                                                                  actions,
5476                                                                  item_flags,
5477                                                                  error);
5478                         if (ret < 0)
5479                                 return ret;
5480                         /* Count all modify-header actions as one action. */
5481                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5482                                 ++actions_n;
5483                         action_flags |= actions->type ==
5484                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
5485                                                 MLX5_FLOW_ACTION_SET_TTL :
5486                                                 MLX5_FLOW_ACTION_DEC_TTL;
5487                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
5488                         break;
5489                 case RTE_FLOW_ACTION_TYPE_JUMP:
5490                         ret = flow_dv_validate_action_jump(actions,
5491                                                            action_flags,
5492                                                            attr, external,
5493                                                            error);
5494                         if (ret)
5495                                 return ret;
5496                         ++actions_n;
5497                         action_flags |= MLX5_FLOW_ACTION_JUMP;
5498                         break;
5499                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5500                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5501                         ret = flow_dv_validate_action_modify_tcp_seq
5502                                                                 (action_flags,
5503                                                                  actions,
5504                                                                  item_flags,
5505                                                                  error);
5506                         if (ret < 0)
5507                                 return ret;
5508                         /* Count all modify-header actions as one action. */
5509                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5510                                 ++actions_n;
5511                         action_flags |= actions->type ==
5512                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5513                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
5514                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5515                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
5516                         break;
5517                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5518                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5519                         ret = flow_dv_validate_action_modify_tcp_ack
5520                                                                 (action_flags,
5521                                                                  actions,
5522                                                                  item_flags,
5523                                                                  error);
5524                         if (ret < 0)
5525                                 return ret;
5526                         /* Count all modify-header actions as one action. */
5527                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5528                                 ++actions_n;
5529                         action_flags |= actions->type ==
5530                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
5531                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
5532                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
5533                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
5534                         break;
5535                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
5536                         break;
5537                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
5538                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
5539                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5540                         break;
5541                 case RTE_FLOW_ACTION_TYPE_METER:
5542                         ret = mlx5_flow_validate_action_meter(dev,
5543                                                               action_flags,
5544                                                               actions, attr,
5545                                                               error);
5546                         if (ret < 0)
5547                                 return ret;
5548                         action_flags |= MLX5_FLOW_ACTION_METER;
5549                         ++actions_n;
5550                         /* Meter action will add one more TAG action. */
5551                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5552                         break;
5553                 case RTE_FLOW_ACTION_TYPE_AGE:
5554                         ret = flow_dv_validate_action_age(action_flags,
5555                                                           actions, dev,
5556                                                           error);
5557                         if (ret < 0)
5558                                 return ret;
5559                         action_flags |= MLX5_FLOW_ACTION_AGE;
5560                         ++actions_n;
5561                         break;
5562                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5563                         ret = flow_dv_validate_action_modify_ipv4_dscp
5564                                                          (action_flags,
5565                                                           actions,
5566                                                           item_flags,
5567                                                           error);
5568                         if (ret < 0)
5569                                 return ret;
5570                         /* Count all modify-header actions as one action. */
5571                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5572                                 ++actions_n;
5573                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
5574                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5575                         break;
5576                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5577                         ret = flow_dv_validate_action_modify_ipv6_dscp
5578                                                                 (action_flags,
5579                                                                  actions,
5580                                                                  item_flags,
5581                                                                  error);
5582                         if (ret < 0)
5583                                 return ret;
5584                         /* Count all modify-header actions as one action. */
5585                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5586                                 ++actions_n;
5587                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
5588                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5589                         break;
5590                 default:
5591                         return rte_flow_error_set(error, ENOTSUP,
5592                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5593                                                   actions,
5594                                                   "action not supported");
5595                 }
5596         }
5597         /*
5598          * Validate the drop action mutual exclusion with other actions.
5599          * Drop action is mutually-exclusive with any other action, except for
5600          * Count action.
5601          */
5602         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
5603             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
5604                 return rte_flow_error_set(error, EINVAL,
5605                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5606                                           "Drop action is mutually-exclusive "
5607                                           "with any other action, except for "
5608                                           "Count action");
5609         /* Eswitch has few restrictions on using items and actions */
5610         if (attr->transfer) {
5611                 if (!mlx5_flow_ext_mreg_supported(dev) &&
5612                     action_flags & MLX5_FLOW_ACTION_FLAG)
5613                         return rte_flow_error_set(error, ENOTSUP,
5614                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5615                                                   NULL,
5616                                                   "unsupported action FLAG");
5617                 if (!mlx5_flow_ext_mreg_supported(dev) &&
5618                     action_flags & MLX5_FLOW_ACTION_MARK)
5619                         return rte_flow_error_set(error, ENOTSUP,
5620                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5621                                                   NULL,
5622                                                   "unsupported action MARK");
5623                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
5624                         return rte_flow_error_set(error, ENOTSUP,
5625                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5626                                                   NULL,
5627                                                   "unsupported action QUEUE");
5628                 if (action_flags & MLX5_FLOW_ACTION_RSS)
5629                         return rte_flow_error_set(error, ENOTSUP,
5630                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5631                                                   NULL,
5632                                                   "unsupported action RSS");
5633                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5634                         return rte_flow_error_set(error, EINVAL,
5635                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5636                                                   actions,
5637                                                   "no fate action is found");
5638         } else {
5639                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
5640                         return rte_flow_error_set(error, EINVAL,
5641                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5642                                                   actions,
5643                                                   "no fate action is found");
5644         }
5645         /* Continue validation for Xcap actions.*/
5646         if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) && (queue_index == 0xFFFF ||
5647             mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
5648                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
5649                     MLX5_FLOW_XCAP_ACTIONS)
5650                         return rte_flow_error_set(error, ENOTSUP,
5651                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5652                                                   NULL, "encap and decap "
5653                                                   "combination aren't supported");
5654                 if (!attr->transfer && attr->ingress && (action_flags &
5655                                                         MLX5_FLOW_ACTION_ENCAP))
5656                         return rte_flow_error_set(error, ENOTSUP,
5657                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5658                                                   NULL, "encap is not supported"
5659                                                   " for ingress traffic");
5660         }
5661         /* Hairpin flow will add one more TAG action. */
5662         if (hairpin > 0)
5663                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5664         /* extra metadata enabled: one more TAG action will be add. */
5665         if (dev_conf->dv_flow_en &&
5666             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
5667             mlx5_flow_ext_mreg_supported(dev))
5668                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5669         if ((uint32_t)rw_act_num >
5670                         flow_dv_modify_hdr_action_max(dev, is_root)) {
5671                 return rte_flow_error_set(error, ENOTSUP,
5672                                           RTE_FLOW_ERROR_TYPE_ACTION,
5673                                           NULL, "too many header modify"
5674                                           " actions to support");
5675         }
5676         return 0;
5677 }
5678
5679 /**
5680  * Internal preparation function. Allocates the DV flow size,
5681  * this size is constant.
5682  *
5683  * @param[in] dev
5684  *   Pointer to the rte_eth_dev structure.
5685  * @param[in] attr
5686  *   Pointer to the flow attributes.
5687  * @param[in] items
5688  *   Pointer to the list of items.
5689  * @param[in] actions
5690  *   Pointer to the list of actions.
5691  * @param[out] error
5692  *   Pointer to the error structure.
5693  *
5694  * @return
5695  *   Pointer to mlx5_flow object on success,
5696  *   otherwise NULL and rte_errno is set.
5697  */
5698 static struct mlx5_flow *
5699 flow_dv_prepare(struct rte_eth_dev *dev,
5700                 const struct rte_flow_attr *attr __rte_unused,
5701                 const struct rte_flow_item items[] __rte_unused,
5702                 const struct rte_flow_action actions[] __rte_unused,
5703                 struct rte_flow_error *error)
5704 {
5705         uint32_t handle_idx = 0;
5706         struct mlx5_flow *dev_flow;
5707         struct mlx5_flow_handle *dev_handle;
5708         struct mlx5_priv *priv = dev->data->dev_private;
5709
5710         /* In case of corrupting the memory. */
5711         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
5712                 rte_flow_error_set(error, ENOSPC,
5713                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5714                                    "not free temporary device flow");
5715                 return NULL;
5716         }
5717         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
5718                                    &handle_idx);
5719         if (!dev_handle) {
5720                 rte_flow_error_set(error, ENOMEM,
5721                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5722                                    "not enough memory to create flow handle");
5723                 return NULL;
5724         }
5725         /* No multi-thread supporting. */
5726         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
5727         dev_flow->handle = dev_handle;
5728         dev_flow->handle_idx = handle_idx;
5729         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
5730         /*
5731          * The matching value needs to be cleared to 0 before using. In the
5732          * past, it will be automatically cleared when using rte_*alloc
5733          * API. The time consumption will be almost the same as before.
5734          */
5735         memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
5736         dev_flow->ingress = attr->ingress;
5737         dev_flow->dv.transfer = attr->transfer;
5738         return dev_flow;
5739 }
5740
5741 #ifdef RTE_LIBRTE_MLX5_DEBUG
5742 /**
5743  * Sanity check for match mask and value. Similar to check_valid_spec() in
5744  * kernel driver. If unmasked bit is present in value, it returns failure.
5745  *
5746  * @param match_mask
5747  *   pointer to match mask buffer.
5748  * @param match_value
5749  *   pointer to match value buffer.
5750  *
5751  * @return
5752  *   0 if valid, -EINVAL otherwise.
5753  */
5754 static int
5755 flow_dv_check_valid_spec(void *match_mask, void *match_value)
5756 {
5757         uint8_t *m = match_mask;
5758         uint8_t *v = match_value;
5759         unsigned int i;
5760
5761         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
5762                 if (v[i] & ~m[i]) {
5763                         DRV_LOG(ERR,
5764                                 "match_value differs from match_criteria"
5765                                 " %p[%u] != %p[%u]",
5766                                 match_value, i, match_mask, i);
5767                         return -EINVAL;
5768                 }
5769         }
5770         return 0;
5771 }
5772 #endif
5773
5774 /**
5775  * Add match of ip_version.
5776  *
5777  * @param[in] group
5778  *   Flow group.
5779  * @param[in] headers_v
5780  *   Values header pointer.
5781  * @param[in] headers_m
5782  *   Masks header pointer.
5783  * @param[in] ip_version
5784  *   The IP version to set.
5785  */
5786 static inline void
5787 flow_dv_set_match_ip_version(uint32_t group,
5788                              void *headers_v,
5789                              void *headers_m,
5790                              uint8_t ip_version)
5791 {
5792         if (group == 0)
5793                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
5794         else
5795                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
5796                          ip_version);
5797         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
5798         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
5799         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
5800 }
5801
5802 /**
5803  * Add Ethernet item to matcher and to the value.
5804  *
5805  * @param[in, out] matcher
5806  *   Flow matcher.
5807  * @param[in, out] key
5808  *   Flow matcher value.
5809  * @param[in] item
5810  *   Flow pattern to translate.
5811  * @param[in] inner
5812  *   Item is inner pattern.
5813  */
5814 static void
5815 flow_dv_translate_item_eth(void *matcher, void *key,
5816                            const struct rte_flow_item *item, int inner,
5817                            uint32_t group)
5818 {
5819         const struct rte_flow_item_eth *eth_m = item->mask;
5820         const struct rte_flow_item_eth *eth_v = item->spec;
5821         const struct rte_flow_item_eth nic_mask = {
5822                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
5823                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
5824                 .type = RTE_BE16(0xffff),
5825         };
5826         void *headers_m;
5827         void *headers_v;
5828         char *l24_v;
5829         unsigned int i;
5830
5831         if (!eth_v)
5832                 return;
5833         if (!eth_m)
5834                 eth_m = &nic_mask;
5835         if (inner) {
5836                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5837                                          inner_headers);
5838                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5839         } else {
5840                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5841                                          outer_headers);
5842                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5843         }
5844         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
5845                &eth_m->dst, sizeof(eth_m->dst));
5846         /* The value must be in the range of the mask. */
5847         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
5848         for (i = 0; i < sizeof(eth_m->dst); ++i)
5849                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
5850         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
5851                &eth_m->src, sizeof(eth_m->src));
5852         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
5853         /* The value must be in the range of the mask. */
5854         for (i = 0; i < sizeof(eth_m->dst); ++i)
5855                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
5856         if (eth_v->type) {
5857                 /* When ethertype is present set mask for tagged VLAN. */
5858                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
5859                 /* Set value for tagged VLAN if ethertype is 802.1Q. */
5860                 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
5861                     eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
5862                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag,
5863                                  1);
5864                         /* Return here to avoid setting match on ethertype. */
5865                         return;
5866                 }
5867         }
5868         /*
5869          * HW supports match on one Ethertype, the Ethertype following the last
5870          * VLAN tag of the packet (see PRM).
5871          * Set match on ethertype only if ETH header is not followed by VLAN.
5872          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
5873          * ethertype, and use ip_version field instead.
5874          */
5875         if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
5876             eth_m->type == 0xFFFF) {
5877                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
5878         } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
5879                    eth_m->type == 0xFFFF) {
5880                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
5881         } else {
5882                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
5883                          rte_be_to_cpu_16(eth_m->type));
5884                 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
5885                                      ethertype);
5886                 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
5887         }
5888 }
5889
5890 /**
5891  * Add VLAN item to matcher and to the value.
5892  *
5893  * @param[in, out] dev_flow
5894  *   Flow descriptor.
5895  * @param[in, out] matcher
5896  *   Flow matcher.
5897  * @param[in, out] key
5898  *   Flow matcher value.
5899  * @param[in] item
5900  *   Flow pattern to translate.
5901  * @param[in] inner
5902  *   Item is inner pattern.
5903  */
5904 static void
5905 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
5906                             void *matcher, void *key,
5907                             const struct rte_flow_item *item,
5908                             int inner, uint32_t group)
5909 {
5910         const struct rte_flow_item_vlan *vlan_m = item->mask;
5911         const struct rte_flow_item_vlan *vlan_v = item->spec;
5912         void *headers_m;
5913         void *headers_v;
5914         uint16_t tci_m;
5915         uint16_t tci_v;
5916
5917         if (inner) {
5918                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5919                                          inner_headers);
5920                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5921         } else {
5922                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5923                                          outer_headers);
5924                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5925                 /*
5926                  * This is workaround, masks are not supported,
5927                  * and pre-validated.
5928                  */
5929                 if (vlan_v)
5930                         dev_flow->handle->vf_vlan.tag =
5931                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
5932         }
5933         /*
5934          * When VLAN item exists in flow, mark packet as tagged,
5935          * even if TCI is not specified.
5936          */
5937         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
5938         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
5939         if (!vlan_v)
5940                 return;
5941         if (!vlan_m)
5942                 vlan_m = &rte_flow_item_vlan_mask;
5943         tci_m = rte_be_to_cpu_16(vlan_m->tci);
5944         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
5945         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
5946         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
5947         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
5948         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
5949         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
5950         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
5951         /*
5952          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
5953          * ethertype, and use ip_version field instead.
5954          */
5955         if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
5956             vlan_m->inner_type == 0xFFFF) {
5957                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
5958         } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
5959                    vlan_m->inner_type == 0xFFFF) {
5960                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
5961         } else {
5962                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
5963                          rte_be_to_cpu_16(vlan_m->inner_type));
5964                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
5965                          rte_be_to_cpu_16(vlan_m->inner_type &
5966                                           vlan_v->inner_type));
5967         }
5968 }
5969
5970 /**
5971  * Add IPV4 item to matcher and to the value.
5972  *
5973  * @param[in, out] matcher
5974  *   Flow matcher.
5975  * @param[in, out] key
5976  *   Flow matcher value.
5977  * @param[in] item
5978  *   Flow pattern to translate.
5979  * @param[in] item_flags
5980  *   Bit-fields that holds the items detected until now.
5981  * @param[in] inner
5982  *   Item is inner pattern.
5983  * @param[in] group
5984  *   The group to insert the rule.
5985  */
5986 static void
5987 flow_dv_translate_item_ipv4(void *matcher, void *key,
5988                             const struct rte_flow_item *item,
5989                             const uint64_t item_flags,
5990                             int inner, uint32_t group)
5991 {
5992         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
5993         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
5994         const struct rte_flow_item_ipv4 nic_mask = {
5995                 .hdr = {
5996                         .src_addr = RTE_BE32(0xffffffff),
5997                         .dst_addr = RTE_BE32(0xffffffff),
5998                         .type_of_service = 0xff,
5999                         .next_proto_id = 0xff,
6000                         .time_to_live = 0xff,
6001                 },
6002         };
6003         void *headers_m;
6004         void *headers_v;
6005         char *l24_m;
6006         char *l24_v;
6007         uint8_t tos;
6008
6009         if (inner) {
6010                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6011                                          inner_headers);
6012                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6013         } else {
6014                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6015                                          outer_headers);
6016                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6017         }
6018         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6019         /*
6020          * On outer header (which must contains L2), or inner header with L2,
6021          * set cvlan_tag mask bit to mark this packet as untagged.
6022          * This should be done even if item->spec is empty.
6023          */
6024         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6025                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6026         if (!ipv4_v)
6027                 return;
6028         if (!ipv4_m)
6029                 ipv4_m = &nic_mask;
6030         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6031                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6032         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6033                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6034         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
6035         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
6036         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6037                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6038         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6039                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6040         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
6041         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
6042         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
6043         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
6044                  ipv4_m->hdr.type_of_service);
6045         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
6046         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
6047                  ipv4_m->hdr.type_of_service >> 2);
6048         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
6049         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6050                  ipv4_m->hdr.next_proto_id);
6051         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6052                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
6053         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6054                  ipv4_m->hdr.time_to_live);
6055         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6056                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
6057 }
6058
6059 /**
6060  * Add IPV6 item to matcher and to the value.
6061  *
6062  * @param[in, out] matcher
6063  *   Flow matcher.
6064  * @param[in, out] key
6065  *   Flow matcher value.
6066  * @param[in] item
6067  *   Flow pattern to translate.
6068  * @param[in] item_flags
6069  *   Bit-fields that holds the items detected until now.
6070  * @param[in] inner
6071  *   Item is inner pattern.
6072  * @param[in] group
6073  *   The group to insert the rule.
6074  */
6075 static void
6076 flow_dv_translate_item_ipv6(void *matcher, void *key,
6077                             const struct rte_flow_item *item,
6078                             const uint64_t item_flags,
6079                             int inner, uint32_t group)
6080 {
6081         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
6082         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
6083         const struct rte_flow_item_ipv6 nic_mask = {
6084                 .hdr = {
6085                         .src_addr =
6086                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6087                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6088                         .dst_addr =
6089                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6090                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6091                         .vtc_flow = RTE_BE32(0xffffffff),
6092                         .proto = 0xff,
6093                         .hop_limits = 0xff,
6094                 },
6095         };
6096         void *headers_m;
6097         void *headers_v;
6098         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6099         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6100         char *l24_m;
6101         char *l24_v;
6102         uint32_t vtc_m;
6103         uint32_t vtc_v;
6104         int i;
6105         int size;
6106
6107         if (inner) {
6108                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6109                                          inner_headers);
6110                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6111         } else {
6112                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6113                                          outer_headers);
6114                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6115         }
6116         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6117         /*
6118          * On outer header (which must contains L2), or inner header with L2,
6119          * set cvlan_tag mask bit to mark this packet as untagged.
6120          * This should be done even if item->spec is empty.
6121          */
6122         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6123                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6124         if (!ipv6_v)
6125                 return;
6126         if (!ipv6_m)
6127                 ipv6_m = &nic_mask;
6128         size = sizeof(ipv6_m->hdr.dst_addr);
6129         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6130                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6131         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6132                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6133         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
6134         for (i = 0; i < size; ++i)
6135                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
6136         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6137                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6138         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6139                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6140         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
6141         for (i = 0; i < size; ++i)
6142                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
6143         /* TOS. */
6144         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
6145         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
6146         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
6147         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
6148         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
6149         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
6150         /* Label. */
6151         if (inner) {
6152                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
6153                          vtc_m);
6154                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
6155                          vtc_v);
6156         } else {
6157                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
6158                          vtc_m);
6159                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
6160                          vtc_v);
6161         }
6162         /* Protocol. */
6163         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6164                  ipv6_m->hdr.proto);
6165         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6166                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
6167         /* Hop limit. */
6168         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6169                  ipv6_m->hdr.hop_limits);
6170         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6171                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
6172 }
6173
6174 /**
6175  * Add TCP item to matcher and to the value.
6176  *
6177  * @param[in, out] matcher
6178  *   Flow matcher.
6179  * @param[in, out] key
6180  *   Flow matcher value.
6181  * @param[in] item
6182  *   Flow pattern to translate.
6183  * @param[in] inner
6184  *   Item is inner pattern.
6185  */
6186 static void
6187 flow_dv_translate_item_tcp(void *matcher, void *key,
6188                            const struct rte_flow_item *item,
6189                            int inner)
6190 {
6191         const struct rte_flow_item_tcp *tcp_m = item->mask;
6192         const struct rte_flow_item_tcp *tcp_v = item->spec;
6193         void *headers_m;
6194         void *headers_v;
6195
6196         if (inner) {
6197                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6198                                          inner_headers);
6199                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6200         } else {
6201                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6202                                          outer_headers);
6203                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6204         }
6205         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6206         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
6207         if (!tcp_v)
6208                 return;
6209         if (!tcp_m)
6210                 tcp_m = &rte_flow_item_tcp_mask;
6211         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
6212                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
6213         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
6214                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
6215         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
6216                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
6217         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
6218                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
6219         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
6220                  tcp_m->hdr.tcp_flags);
6221         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
6222                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
6223 }
6224
6225 /**
6226  * Add UDP item to matcher and to the value.
6227  *
6228  * @param[in, out] matcher
6229  *   Flow matcher.
6230  * @param[in, out] key
6231  *   Flow matcher value.
6232  * @param[in] item
6233  *   Flow pattern to translate.
6234  * @param[in] inner
6235  *   Item is inner pattern.
6236  */
6237 static void
6238 flow_dv_translate_item_udp(void *matcher, void *key,
6239                            const struct rte_flow_item *item,
6240                            int inner)
6241 {
6242         const struct rte_flow_item_udp *udp_m = item->mask;
6243         const struct rte_flow_item_udp *udp_v = item->spec;
6244         void *headers_m;
6245         void *headers_v;
6246
6247         if (inner) {
6248                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6249                                          inner_headers);
6250                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6251         } else {
6252                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6253                                          outer_headers);
6254                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6255         }
6256         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6257         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
6258         if (!udp_v)
6259                 return;
6260         if (!udp_m)
6261                 udp_m = &rte_flow_item_udp_mask;
6262         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
6263                  rte_be_to_cpu_16(udp_m->hdr.src_port));
6264         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
6265                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
6266         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
6267                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
6268         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6269                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
6270 }
6271
6272 /**
6273  * Add GRE optional Key item to matcher and to the value.
6274  *
6275  * @param[in, out] matcher
6276  *   Flow matcher.
6277  * @param[in, out] key
6278  *   Flow matcher value.
6279  * @param[in] item
6280  *   Flow pattern to translate.
6281  * @param[in] inner
6282  *   Item is inner pattern.
6283  */
6284 static void
6285 flow_dv_translate_item_gre_key(void *matcher, void *key,
6286                                    const struct rte_flow_item *item)
6287 {
6288         const rte_be32_t *key_m = item->mask;
6289         const rte_be32_t *key_v = item->spec;
6290         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6291         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6292         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
6293
6294         /* GRE K bit must be on and should already be validated */
6295         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
6296         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
6297         if (!key_v)
6298                 return;
6299         if (!key_m)
6300                 key_m = &gre_key_default_mask;
6301         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
6302                  rte_be_to_cpu_32(*key_m) >> 8);
6303         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
6304                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
6305         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
6306                  rte_be_to_cpu_32(*key_m) & 0xFF);
6307         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
6308                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
6309 }
6310
6311 /**
6312  * Add GRE item to matcher and to the value.
6313  *
6314  * @param[in, out] matcher
6315  *   Flow matcher.
6316  * @param[in, out] key
6317  *   Flow matcher value.
6318  * @param[in] item
6319  *   Flow pattern to translate.
6320  * @param[in] inner
6321  *   Item is inner pattern.
6322  */
6323 static void
6324 flow_dv_translate_item_gre(void *matcher, void *key,
6325                            const struct rte_flow_item *item,
6326                            int inner)
6327 {
6328         const struct rte_flow_item_gre *gre_m = item->mask;
6329         const struct rte_flow_item_gre *gre_v = item->spec;
6330         void *headers_m;
6331         void *headers_v;
6332         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6333         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6334         struct {
6335                 union {
6336                         __extension__
6337                         struct {
6338                                 uint16_t version:3;
6339                                 uint16_t rsvd0:9;
6340                                 uint16_t s_present:1;
6341                                 uint16_t k_present:1;
6342                                 uint16_t rsvd_bit1:1;
6343                                 uint16_t c_present:1;
6344                         };
6345                         uint16_t value;
6346                 };
6347         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
6348
6349         if (inner) {
6350                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6351                                          inner_headers);
6352                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6353         } else {
6354                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6355                                          outer_headers);
6356                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6357         }
6358         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6359         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
6360         if (!gre_v)
6361                 return;
6362         if (!gre_m)
6363                 gre_m = &rte_flow_item_gre_mask;
6364         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
6365                  rte_be_to_cpu_16(gre_m->protocol));
6366         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6367                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
6368         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
6369         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
6370         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
6371                  gre_crks_rsvd0_ver_m.c_present);
6372         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
6373                  gre_crks_rsvd0_ver_v.c_present &
6374                  gre_crks_rsvd0_ver_m.c_present);
6375         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
6376                  gre_crks_rsvd0_ver_m.k_present);
6377         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
6378                  gre_crks_rsvd0_ver_v.k_present &
6379                  gre_crks_rsvd0_ver_m.k_present);
6380         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
6381                  gre_crks_rsvd0_ver_m.s_present);
6382         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
6383                  gre_crks_rsvd0_ver_v.s_present &
6384                  gre_crks_rsvd0_ver_m.s_present);
6385 }
6386
6387 /**
6388  * Add NVGRE item to matcher and to the value.
6389  *
6390  * @param[in, out] matcher
6391  *   Flow matcher.
6392  * @param[in, out] key
6393  *   Flow matcher value.
6394  * @param[in] item
6395  *   Flow pattern to translate.
6396  * @param[in] inner
6397  *   Item is inner pattern.
6398  */
6399 static void
6400 flow_dv_translate_item_nvgre(void *matcher, void *key,
6401                              const struct rte_flow_item *item,
6402                              int inner)
6403 {
6404         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
6405         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
6406         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6407         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6408         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
6409         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
6410         char *gre_key_m;
6411         char *gre_key_v;
6412         int size;
6413         int i;
6414
6415         /* For NVGRE, GRE header fields must be set with defined values. */
6416         const struct rte_flow_item_gre gre_spec = {
6417                 .c_rsvd0_ver = RTE_BE16(0x2000),
6418                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
6419         };
6420         const struct rte_flow_item_gre gre_mask = {
6421                 .c_rsvd0_ver = RTE_BE16(0xB000),
6422                 .protocol = RTE_BE16(UINT16_MAX),
6423         };
6424         const struct rte_flow_item gre_item = {
6425                 .spec = &gre_spec,
6426                 .mask = &gre_mask,
6427                 .last = NULL,
6428         };
6429         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
6430         if (!nvgre_v)
6431                 return;
6432         if (!nvgre_m)
6433                 nvgre_m = &rte_flow_item_nvgre_mask;
6434         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
6435         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
6436         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
6437         memcpy(gre_key_m, tni_flow_id_m, size);
6438         for (i = 0; i < size; ++i)
6439                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
6440 }
6441
6442 /**
6443  * Add VXLAN item to matcher and to the value.
6444  *
6445  * @param[in, out] matcher
6446  *   Flow matcher.
6447  * @param[in, out] key
6448  *   Flow matcher value.
6449  * @param[in] item
6450  *   Flow pattern to translate.
6451  * @param[in] inner
6452  *   Item is inner pattern.
6453  */
6454 static void
6455 flow_dv_translate_item_vxlan(void *matcher, void *key,
6456                              const struct rte_flow_item *item,
6457                              int inner)
6458 {
6459         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
6460         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
6461         void *headers_m;
6462         void *headers_v;
6463         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6464         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6465         char *vni_m;
6466         char *vni_v;
6467         uint16_t dport;
6468         int size;
6469         int i;
6470
6471         if (inner) {
6472                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6473                                          inner_headers);
6474                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6475         } else {
6476                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6477                                          outer_headers);
6478                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6479         }
6480         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6481                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6482         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6483                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6484                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6485         }
6486         if (!vxlan_v)
6487                 return;
6488         if (!vxlan_m)
6489                 vxlan_m = &rte_flow_item_vxlan_mask;
6490         size = sizeof(vxlan_m->vni);
6491         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
6492         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
6493         memcpy(vni_m, vxlan_m->vni, size);
6494         for (i = 0; i < size; ++i)
6495                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6496 }
6497
6498 /**
6499  * Add VXLAN-GPE item to matcher and to the value.
6500  *
6501  * @param[in, out] matcher
6502  *   Flow matcher.
6503  * @param[in, out] key
6504  *   Flow matcher value.
6505  * @param[in] item
6506  *   Flow pattern to translate.
6507  * @param[in] inner
6508  *   Item is inner pattern.
6509  */
6510
6511 static void
6512 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
6513                                  const struct rte_flow_item *item, int inner)
6514 {
6515         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
6516         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
6517         void *headers_m;
6518         void *headers_v;
6519         void *misc_m =
6520                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
6521         void *misc_v =
6522                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
6523         char *vni_m;
6524         char *vni_v;
6525         uint16_t dport;
6526         int size;
6527         int i;
6528         uint8_t flags_m = 0xff;
6529         uint8_t flags_v = 0xc;
6530
6531         if (inner) {
6532                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6533                                          inner_headers);
6534                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6535         } else {
6536                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6537                                          outer_headers);
6538                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6539         }
6540         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6541                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6542         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6543                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6544                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6545         }
6546         if (!vxlan_v)
6547                 return;
6548         if (!vxlan_m)
6549                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
6550         size = sizeof(vxlan_m->vni);
6551         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
6552         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
6553         memcpy(vni_m, vxlan_m->vni, size);
6554         for (i = 0; i < size; ++i)
6555                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6556         if (vxlan_m->flags) {
6557                 flags_m = vxlan_m->flags;
6558                 flags_v = vxlan_v->flags;
6559         }
6560         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
6561         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
6562         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
6563                  vxlan_m->protocol);
6564         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
6565                  vxlan_v->protocol);
6566 }
6567
6568 /**
6569  * Add Geneve item to matcher and to the value.
6570  *
6571  * @param[in, out] matcher
6572  *   Flow matcher.
6573  * @param[in, out] key
6574  *   Flow matcher value.
6575  * @param[in] item
6576  *   Flow pattern to translate.
6577  * @param[in] inner
6578  *   Item is inner pattern.
6579  */
6580
6581 static void
6582 flow_dv_translate_item_geneve(void *matcher, void *key,
6583                               const struct rte_flow_item *item, int inner)
6584 {
6585         const struct rte_flow_item_geneve *geneve_m = item->mask;
6586         const struct rte_flow_item_geneve *geneve_v = item->spec;
6587         void *headers_m;
6588         void *headers_v;
6589         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6590         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6591         uint16_t dport;
6592         uint16_t gbhdr_m;
6593         uint16_t gbhdr_v;
6594         char *vni_m;
6595         char *vni_v;
6596         size_t size, i;
6597
6598         if (inner) {
6599                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6600                                          inner_headers);
6601                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6602         } else {
6603                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6604                                          outer_headers);
6605                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6606         }
6607         dport = MLX5_UDP_PORT_GENEVE;
6608         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6609                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6610                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6611         }
6612         if (!geneve_v)
6613                 return;
6614         if (!geneve_m)
6615                 geneve_m = &rte_flow_item_geneve_mask;
6616         size = sizeof(geneve_m->vni);
6617         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
6618         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
6619         memcpy(vni_m, geneve_m->vni, size);
6620         for (i = 0; i < size; ++i)
6621                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
6622         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
6623                  rte_be_to_cpu_16(geneve_m->protocol));
6624         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
6625                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
6626         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
6627         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
6628         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
6629                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
6630         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
6631                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
6632         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
6633                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
6634         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
6635                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
6636                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
6637 }
6638
6639 /**
6640  * Add MPLS item to matcher and to the value.
6641  *
6642  * @param[in, out] matcher
6643  *   Flow matcher.
6644  * @param[in, out] key
6645  *   Flow matcher value.
6646  * @param[in] item
6647  *   Flow pattern to translate.
6648  * @param[in] prev_layer
6649  *   The protocol layer indicated in previous item.
6650  * @param[in] inner
6651  *   Item is inner pattern.
6652  */
6653 static void
6654 flow_dv_translate_item_mpls(void *matcher, void *key,
6655                             const struct rte_flow_item *item,
6656                             uint64_t prev_layer,
6657                             int inner)
6658 {
6659         const uint32_t *in_mpls_m = item->mask;
6660         const uint32_t *in_mpls_v = item->spec;
6661         uint32_t *out_mpls_m = 0;
6662         uint32_t *out_mpls_v = 0;
6663         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6664         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6665         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
6666                                      misc_parameters_2);
6667         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
6668         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
6669         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6670
6671         switch (prev_layer) {
6672         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
6673                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
6674                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6675                          MLX5_UDP_PORT_MPLS);
6676                 break;
6677         case MLX5_FLOW_LAYER_GRE:
6678                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
6679                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6680                          RTE_ETHER_TYPE_MPLS);
6681                 break;
6682         default:
6683                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6684                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6685                          IPPROTO_MPLS);
6686                 break;
6687         }
6688         if (!in_mpls_v)
6689                 return;
6690         if (!in_mpls_m)
6691                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
6692         switch (prev_layer) {
6693         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
6694                 out_mpls_m =
6695                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
6696                                                  outer_first_mpls_over_udp);
6697                 out_mpls_v =
6698                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
6699                                                  outer_first_mpls_over_udp);
6700                 break;
6701         case MLX5_FLOW_LAYER_GRE:
6702                 out_mpls_m =
6703                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
6704                                                  outer_first_mpls_over_gre);
6705                 out_mpls_v =
6706                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
6707                                                  outer_first_mpls_over_gre);
6708                 break;
6709         default:
6710                 /* Inner MPLS not over GRE is not supported. */
6711                 if (!inner) {
6712                         out_mpls_m =
6713                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
6714                                                          misc2_m,
6715                                                          outer_first_mpls);
6716                         out_mpls_v =
6717                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
6718                                                          misc2_v,
6719                                                          outer_first_mpls);
6720                 }
6721                 break;
6722         }
6723         if (out_mpls_m && out_mpls_v) {
6724                 *out_mpls_m = *in_mpls_m;
6725                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
6726         }
6727 }
6728
6729 /**
6730  * Add metadata register item to matcher
6731  *
6732  * @param[in, out] matcher
6733  *   Flow matcher.
6734  * @param[in, out] key
6735  *   Flow matcher value.
6736  * @param[in] reg_type
6737  *   Type of device metadata register
6738  * @param[in] value
6739  *   Register value
6740  * @param[in] mask
6741  *   Register mask
6742  */
6743 static void
6744 flow_dv_match_meta_reg(void *matcher, void *key,
6745                        enum modify_reg reg_type,
6746                        uint32_t data, uint32_t mask)
6747 {
6748         void *misc2_m =
6749                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
6750         void *misc2_v =
6751                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
6752         uint32_t temp;
6753
6754         data &= mask;
6755         switch (reg_type) {
6756         case REG_A:
6757                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
6758                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
6759                 break;
6760         case REG_B:
6761                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
6762                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
6763                 break;
6764         case REG_C_0:
6765                 /*
6766                  * The metadata register C0 field might be divided into
6767                  * source vport index and META item value, we should set
6768                  * this field according to specified mask, not as whole one.
6769                  */
6770                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
6771                 temp |= mask;
6772                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
6773                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
6774                 temp &= ~mask;
6775                 temp |= data;
6776                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
6777                 break;
6778         case REG_C_1:
6779                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
6780                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
6781                 break;
6782         case REG_C_2:
6783                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
6784                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
6785                 break;
6786         case REG_C_3:
6787                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
6788                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
6789                 break;
6790         case REG_C_4:
6791                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
6792                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
6793                 break;
6794         case REG_C_5:
6795                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
6796                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
6797                 break;
6798         case REG_C_6:
6799                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
6800                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
6801                 break;
6802         case REG_C_7:
6803                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
6804                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
6805                 break;
6806         default:
6807                 MLX5_ASSERT(false);
6808                 break;
6809         }
6810 }
6811
6812 /**
6813  * Add MARK item to matcher
6814  *
6815  * @param[in] dev
6816  *   The device to configure through.
6817  * @param[in, out] matcher
6818  *   Flow matcher.
6819  * @param[in, out] key
6820  *   Flow matcher value.
6821  * @param[in] item
6822  *   Flow pattern to translate.
6823  */
6824 static void
6825 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
6826                             void *matcher, void *key,
6827                             const struct rte_flow_item *item)
6828 {
6829         struct mlx5_priv *priv = dev->data->dev_private;
6830         const struct rte_flow_item_mark *mark;
6831         uint32_t value;
6832         uint32_t mask;
6833
6834         mark = item->mask ? (const void *)item->mask :
6835                             &rte_flow_item_mark_mask;
6836         mask = mark->id & priv->sh->dv_mark_mask;
6837         mark = (const void *)item->spec;
6838         MLX5_ASSERT(mark);
6839         value = mark->id & priv->sh->dv_mark_mask & mask;
6840         if (mask) {
6841                 enum modify_reg reg;
6842
6843                 /* Get the metadata register index for the mark. */
6844                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
6845                 MLX5_ASSERT(reg > 0);
6846                 if (reg == REG_C_0) {
6847                         struct mlx5_priv *priv = dev->data->dev_private;
6848                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6849                         uint32_t shl_c0 = rte_bsf32(msk_c0);
6850
6851                         mask &= msk_c0;
6852                         mask <<= shl_c0;
6853                         value <<= shl_c0;
6854                 }
6855                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
6856         }
6857 }
6858
6859 /**
6860  * Add META item to matcher
6861  *
6862  * @param[in] dev
6863  *   The devich to configure through.
6864  * @param[in, out] matcher
6865  *   Flow matcher.
6866  * @param[in, out] key
6867  *   Flow matcher value.
6868  * @param[in] attr
6869  *   Attributes of flow that includes this item.
6870  * @param[in] item
6871  *   Flow pattern to translate.
6872  */
6873 static void
6874 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
6875                             void *matcher, void *key,
6876                             const struct rte_flow_attr *attr,
6877                             const struct rte_flow_item *item)
6878 {
6879         const struct rte_flow_item_meta *meta_m;
6880         const struct rte_flow_item_meta *meta_v;
6881
6882         meta_m = (const void *)item->mask;
6883         if (!meta_m)
6884                 meta_m = &rte_flow_item_meta_mask;
6885         meta_v = (const void *)item->spec;
6886         if (meta_v) {
6887                 int reg;
6888                 uint32_t value = meta_v->data;
6889                 uint32_t mask = meta_m->data;
6890
6891                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
6892                 if (reg < 0)
6893                         return;
6894                 /*
6895                  * In datapath code there is no endianness
6896                  * coversions for perfromance reasons, all
6897                  * pattern conversions are done in rte_flow.
6898                  */
6899                 value = rte_cpu_to_be_32(value);
6900                 mask = rte_cpu_to_be_32(mask);
6901                 if (reg == REG_C_0) {
6902                         struct mlx5_priv *priv = dev->data->dev_private;
6903                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6904                         uint32_t shl_c0 = rte_bsf32(msk_c0);
6905 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6906                         uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
6907
6908                         value >>= shr_c0;
6909                         mask >>= shr_c0;
6910 #endif
6911                         value <<= shl_c0;
6912                         mask <<= shl_c0;
6913                         MLX5_ASSERT(msk_c0);
6914                         MLX5_ASSERT(!(~msk_c0 & mask));
6915                 }
6916                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
6917         }
6918 }
6919
6920 /**
6921  * Add vport metadata Reg C0 item to matcher
6922  *
6923  * @param[in, out] matcher
6924  *   Flow matcher.
6925  * @param[in, out] key
6926  *   Flow matcher value.
6927  * @param[in] reg
6928  *   Flow pattern to translate.
6929  */
6930 static void
6931 flow_dv_translate_item_meta_vport(void *matcher, void *key,
6932                                   uint32_t value, uint32_t mask)
6933 {
6934         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
6935 }
6936
6937 /**
6938  * Add tag item to matcher
6939  *
6940  * @param[in] dev
6941  *   The devich to configure through.
6942  * @param[in, out] matcher
6943  *   Flow matcher.
6944  * @param[in, out] key
6945  *   Flow matcher value.
6946  * @param[in] item
6947  *   Flow pattern to translate.
6948  */
6949 static void
6950 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
6951                                 void *matcher, void *key,
6952                                 const struct rte_flow_item *item)
6953 {
6954         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
6955         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
6956         uint32_t mask, value;
6957
6958         MLX5_ASSERT(tag_v);
6959         value = tag_v->data;
6960         mask = tag_m ? tag_m->data : UINT32_MAX;
6961         if (tag_v->id == REG_C_0) {
6962                 struct mlx5_priv *priv = dev->data->dev_private;
6963                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6964                 uint32_t shl_c0 = rte_bsf32(msk_c0);
6965
6966                 mask &= msk_c0;
6967                 mask <<= shl_c0;
6968                 value <<= shl_c0;
6969         }
6970         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
6971 }
6972
6973 /**
6974  * Add TAG item to matcher
6975  *
6976  * @param[in] dev
6977  *   The devich to configure through.
6978  * @param[in, out] matcher
6979  *   Flow matcher.
6980  * @param[in, out] key
6981  *   Flow matcher value.
6982  * @param[in] item
6983  *   Flow pattern to translate.
6984  */
6985 static void
6986 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
6987                            void *matcher, void *key,
6988                            const struct rte_flow_item *item)
6989 {
6990         const struct rte_flow_item_tag *tag_v = item->spec;
6991         const struct rte_flow_item_tag *tag_m = item->mask;
6992         enum modify_reg reg;
6993
6994         MLX5_ASSERT(tag_v);
6995         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
6996         /* Get the metadata register index for the tag. */
6997         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
6998         MLX5_ASSERT(reg > 0);
6999         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
7000 }
7001
7002 /**
7003  * Add source vport match to the specified matcher.
7004  *
7005  * @param[in, out] matcher
7006  *   Flow matcher.
7007  * @param[in, out] key
7008  *   Flow matcher value.
7009  * @param[in] port
7010  *   Source vport value to match
7011  * @param[in] mask
7012  *   Mask
7013  */
7014 static void
7015 flow_dv_translate_item_source_vport(void *matcher, void *key,
7016                                     int16_t port, uint16_t mask)
7017 {
7018         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7019         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7020
7021         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
7022         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
7023 }
7024
7025 /**
7026  * Translate port-id item to eswitch match on  port-id.
7027  *
7028  * @param[in] dev
7029  *   The devich to configure through.
7030  * @param[in, out] matcher
7031  *   Flow matcher.
7032  * @param[in, out] key
7033  *   Flow matcher value.
7034  * @param[in] item
7035  *   Flow pattern to translate.
7036  *
7037  * @return
7038  *   0 on success, a negative errno value otherwise.
7039  */
7040 static int
7041 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
7042                                void *key, const struct rte_flow_item *item)
7043 {
7044         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
7045         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
7046         struct mlx5_priv *priv;
7047         uint16_t mask, id;
7048
7049         mask = pid_m ? pid_m->id : 0xffff;
7050         id = pid_v ? pid_v->id : dev->data->port_id;
7051         priv = mlx5_port_to_eswitch_info(id, item == NULL);
7052         if (!priv)
7053                 return -rte_errno;
7054         /* Translate to vport field or to metadata, depending on mode. */
7055         if (priv->vport_meta_mask)
7056                 flow_dv_translate_item_meta_vport(matcher, key,
7057                                                   priv->vport_meta_tag,
7058                                                   priv->vport_meta_mask);
7059         else
7060                 flow_dv_translate_item_source_vport(matcher, key,
7061                                                     priv->vport_id, mask);
7062         return 0;
7063 }
7064
7065 /**
7066  * Add ICMP6 item to matcher and to the value.
7067  *
7068  * @param[in, out] matcher
7069  *   Flow matcher.
7070  * @param[in, out] key
7071  *   Flow matcher value.
7072  * @param[in] item
7073  *   Flow pattern to translate.
7074  * @param[in] inner
7075  *   Item is inner pattern.
7076  */
7077 static void
7078 flow_dv_translate_item_icmp6(void *matcher, void *key,
7079                               const struct rte_flow_item *item,
7080                               int inner)
7081 {
7082         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
7083         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
7084         void *headers_m;
7085         void *headers_v;
7086         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7087                                      misc_parameters_3);
7088         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7089         if (inner) {
7090                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7091                                          inner_headers);
7092                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7093         } else {
7094                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7095                                          outer_headers);
7096                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7097         }
7098         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7099         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
7100         if (!icmp6_v)
7101                 return;
7102         if (!icmp6_m)
7103                 icmp6_m = &rte_flow_item_icmp6_mask;
7104         /*
7105          * Force flow only to match the non-fragmented IPv6 ICMPv6 packets.
7106          * If only the protocol is specified, no need to match the frag.
7107          */
7108         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
7109         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
7110         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
7111         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
7112                  icmp6_v->type & icmp6_m->type);
7113         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
7114         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
7115                  icmp6_v->code & icmp6_m->code);
7116 }
7117
7118 /**
7119  * Add ICMP item to matcher and to the value.
7120  *
7121  * @param[in, out] matcher
7122  *   Flow matcher.
7123  * @param[in, out] key
7124  *   Flow matcher value.
7125  * @param[in] item
7126  *   Flow pattern to translate.
7127  * @param[in] inner
7128  *   Item is inner pattern.
7129  */
7130 static void
7131 flow_dv_translate_item_icmp(void *matcher, void *key,
7132                             const struct rte_flow_item *item,
7133                             int inner)
7134 {
7135         const struct rte_flow_item_icmp *icmp_m = item->mask;
7136         const struct rte_flow_item_icmp *icmp_v = item->spec;
7137         void *headers_m;
7138         void *headers_v;
7139         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7140                                      misc_parameters_3);
7141         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7142         if (inner) {
7143                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7144                                          inner_headers);
7145                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7146         } else {
7147                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7148                                          outer_headers);
7149                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7150         }
7151         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7152         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
7153         if (!icmp_v)
7154                 return;
7155         if (!icmp_m)
7156                 icmp_m = &rte_flow_item_icmp_mask;
7157         /*
7158          * Force flow only to match the non-fragmented IPv4 ICMP packets.
7159          * If only the protocol is specified, no need to match the frag.
7160          */
7161         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
7162         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
7163         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
7164                  icmp_m->hdr.icmp_type);
7165         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
7166                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
7167         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
7168                  icmp_m->hdr.icmp_code);
7169         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
7170                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
7171 }
7172
7173 /**
7174  * Add GTP item to matcher and to the value.
7175  *
7176  * @param[in, out] matcher
7177  *   Flow matcher.
7178  * @param[in, out] key
7179  *   Flow matcher value.
7180  * @param[in] item
7181  *   Flow pattern to translate.
7182  * @param[in] inner
7183  *   Item is inner pattern.
7184  */
7185 static void
7186 flow_dv_translate_item_gtp(void *matcher, void *key,
7187                            const struct rte_flow_item *item, int inner)
7188 {
7189         const struct rte_flow_item_gtp *gtp_m = item->mask;
7190         const struct rte_flow_item_gtp *gtp_v = item->spec;
7191         void *headers_m;
7192         void *headers_v;
7193         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7194                                      misc_parameters_3);
7195         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7196         uint16_t dport = RTE_GTPU_UDP_PORT;
7197
7198         if (inner) {
7199                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7200                                          inner_headers);
7201                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7202         } else {
7203                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7204                                          outer_headers);
7205                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7206         }
7207         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7208                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7209                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7210         }
7211         if (!gtp_v)
7212                 return;
7213         if (!gtp_m)
7214                 gtp_m = &rte_flow_item_gtp_mask;
7215         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
7216                  gtp_m->v_pt_rsv_flags);
7217         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
7218                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
7219         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
7220         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
7221                  gtp_v->msg_type & gtp_m->msg_type);
7222         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
7223                  rte_be_to_cpu_32(gtp_m->teid));
7224         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
7225                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
7226 }
7227
7228 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
7229
7230 #define HEADER_IS_ZERO(match_criteria, headers)                              \
7231         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
7232                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
7233
7234 /**
7235  * Calculate flow matcher enable bitmap.
7236  *
7237  * @param match_criteria
7238  *   Pointer to flow matcher criteria.
7239  *
7240  * @return
7241  *   Bitmap of enabled fields.
7242  */
7243 static uint8_t
7244 flow_dv_matcher_enable(uint32_t *match_criteria)
7245 {
7246         uint8_t match_criteria_enable;
7247
7248         match_criteria_enable =
7249                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
7250                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
7251         match_criteria_enable |=
7252                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
7253                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
7254         match_criteria_enable |=
7255                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
7256                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
7257         match_criteria_enable |=
7258                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
7259                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
7260         match_criteria_enable |=
7261                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
7262                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
7263         return match_criteria_enable;
7264 }
7265
7266
7267 /**
7268  * Get a flow table.
7269  *
7270  * @param[in, out] dev
7271  *   Pointer to rte_eth_dev structure.
7272  * @param[in] table_id
7273  *   Table id to use.
7274  * @param[in] egress
7275  *   Direction of the table.
7276  * @param[in] transfer
7277  *   E-Switch or NIC flow.
7278  * @param[out] error
7279  *   pointer to error structure.
7280  *
7281  * @return
7282  *   Returns tables resource based on the index, NULL in case of failed.
7283  */
7284 static struct mlx5_flow_tbl_resource *
7285 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
7286                          uint32_t table_id, uint8_t egress,
7287                          uint8_t transfer,
7288                          struct rte_flow_error *error)
7289 {
7290         struct mlx5_priv *priv = dev->data->dev_private;
7291         struct mlx5_dev_ctx_shared *sh = priv->sh;
7292         struct mlx5_flow_tbl_resource *tbl;
7293         union mlx5_flow_tbl_key table_key = {
7294                 {
7295                         .table_id = table_id,
7296                         .reserved = 0,
7297                         .domain = !!transfer,
7298                         .direction = !!egress,
7299                 }
7300         };
7301         struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
7302                                                          table_key.v64);
7303         struct mlx5_flow_tbl_data_entry *tbl_data;
7304         uint32_t idx = 0;
7305         int ret;
7306         void *domain;
7307
7308         if (pos) {
7309                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
7310                                         entry);
7311                 tbl = &tbl_data->tbl;
7312                 rte_atomic32_inc(&tbl->refcnt);
7313                 return tbl;
7314         }
7315         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
7316         if (!tbl_data) {
7317                 rte_flow_error_set(error, ENOMEM,
7318                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7319                                    NULL,
7320                                    "cannot allocate flow table data entry");
7321                 return NULL;
7322         }
7323         tbl_data->idx = idx;
7324         tbl = &tbl_data->tbl;
7325         pos = &tbl_data->entry;
7326         if (transfer)
7327                 domain = sh->fdb_domain;
7328         else if (egress)
7329                 domain = sh->tx_domain;
7330         else
7331                 domain = sh->rx_domain;
7332         tbl->obj = mlx5_glue->dr_create_flow_tbl(domain, table_id);
7333         if (!tbl->obj) {
7334                 rte_flow_error_set(error, ENOMEM,
7335                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7336                                    NULL, "cannot create flow table object");
7337                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7338                 return NULL;
7339         }
7340         /*
7341          * No multi-threads now, but still better to initialize the reference
7342          * count before insert it into the hash list.
7343          */
7344         rte_atomic32_init(&tbl->refcnt);
7345         /* Jump action reference count is initialized here. */
7346         rte_atomic32_init(&tbl_data->jump.refcnt);
7347         pos->key = table_key.v64;
7348         ret = mlx5_hlist_insert(sh->flow_tbls, pos);
7349         if (ret < 0) {
7350                 rte_flow_error_set(error, -ret,
7351                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7352                                    "cannot insert flow table data entry");
7353                 mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
7354                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7355         }
7356         rte_atomic32_inc(&tbl->refcnt);
7357         return tbl;
7358 }
7359
7360 /**
7361  * Release a flow table.
7362  *
7363  * @param[in] dev
7364  *   Pointer to rte_eth_dev structure.
7365  * @param[in] tbl
7366  *   Table resource to be released.
7367  *
7368  * @return
7369  *   Returns 0 if table was released, else return 1;
7370  */
7371 static int
7372 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
7373                              struct mlx5_flow_tbl_resource *tbl)
7374 {
7375         struct mlx5_priv *priv = dev->data->dev_private;
7376         struct mlx5_dev_ctx_shared *sh = priv->sh;
7377         struct mlx5_flow_tbl_data_entry *tbl_data =
7378                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7379
7380         if (!tbl)
7381                 return 0;
7382         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
7383                 struct mlx5_hlist_entry *pos = &tbl_data->entry;
7384
7385                 mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
7386                 tbl->obj = NULL;
7387                 /* remove the entry from the hash list and free memory. */
7388                 mlx5_hlist_remove(sh->flow_tbls, pos);
7389                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
7390                                 tbl_data->idx);
7391                 return 0;
7392         }
7393         return 1;
7394 }
7395
7396 /**
7397  * Register the flow matcher.
7398  *
7399  * @param[in, out] dev
7400  *   Pointer to rte_eth_dev structure.
7401  * @param[in, out] matcher
7402  *   Pointer to flow matcher.
7403  * @param[in, out] key
7404  *   Pointer to flow table key.
7405  * @parm[in, out] dev_flow
7406  *   Pointer to the dev_flow.
7407  * @param[out] error
7408  *   pointer to error structure.
7409  *
7410  * @return
7411  *   0 on success otherwise -errno and errno is set.
7412  */
7413 static int
7414 flow_dv_matcher_register(struct rte_eth_dev *dev,
7415                          struct mlx5_flow_dv_matcher *matcher,
7416                          union mlx5_flow_tbl_key *key,
7417                          struct mlx5_flow *dev_flow,
7418                          struct rte_flow_error *error)
7419 {
7420         struct mlx5_priv *priv = dev->data->dev_private;
7421         struct mlx5_dev_ctx_shared *sh = priv->sh;
7422         struct mlx5_flow_dv_matcher *cache_matcher;
7423         struct mlx5dv_flow_matcher_attr dv_attr = {
7424                 .type = IBV_FLOW_ATTR_NORMAL,
7425                 .match_mask = (void *)&matcher->mask,
7426         };
7427         struct mlx5_flow_tbl_resource *tbl;
7428         struct mlx5_flow_tbl_data_entry *tbl_data;
7429
7430         tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
7431                                        key->domain, error);
7432         if (!tbl)
7433                 return -rte_errno;      /* No need to refill the error info */
7434         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7435         /* Lookup from cache. */
7436         LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
7437                 if (matcher->crc == cache_matcher->crc &&
7438                     matcher->priority == cache_matcher->priority &&
7439                     !memcmp((const void *)matcher->mask.buf,
7440                             (const void *)cache_matcher->mask.buf,
7441                             cache_matcher->mask.size)) {
7442                         DRV_LOG(DEBUG,
7443                                 "%s group %u priority %hd use %s "
7444                                 "matcher %p: refcnt %d++",
7445                                 key->domain ? "FDB" : "NIC", key->table_id,
7446                                 cache_matcher->priority,
7447                                 key->direction ? "tx" : "rx",
7448                                 (void *)cache_matcher,
7449                                 rte_atomic32_read(&cache_matcher->refcnt));
7450                         rte_atomic32_inc(&cache_matcher->refcnt);
7451                         dev_flow->handle->dvh.matcher = cache_matcher;
7452                         /* old matcher should not make the table ref++. */
7453                         flow_dv_tbl_resource_release(dev, tbl);
7454                         return 0;
7455                 }
7456         }
7457         /* Register new matcher. */
7458         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
7459         if (!cache_matcher) {
7460                 flow_dv_tbl_resource_release(dev, tbl);
7461                 return rte_flow_error_set(error, ENOMEM,
7462                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7463                                           "cannot allocate matcher memory");
7464         }
7465         *cache_matcher = *matcher;
7466         dv_attr.match_criteria_enable =
7467                 flow_dv_matcher_enable(cache_matcher->mask.buf);
7468         dv_attr.priority = matcher->priority;
7469         if (key->direction)
7470                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
7471         cache_matcher->matcher_object =
7472                 mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj);
7473         if (!cache_matcher->matcher_object) {
7474                 rte_free(cache_matcher);
7475 #ifdef HAVE_MLX5DV_DR
7476                 flow_dv_tbl_resource_release(dev, tbl);
7477 #endif
7478                 return rte_flow_error_set(error, ENOMEM,
7479                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7480                                           NULL, "cannot create matcher");
7481         }
7482         /* Save the table information */
7483         cache_matcher->tbl = tbl;
7484         rte_atomic32_init(&cache_matcher->refcnt);
7485         /* only matcher ref++, table ref++ already done above in get API. */
7486         rte_atomic32_inc(&cache_matcher->refcnt);
7487         LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
7488         dev_flow->handle->dvh.matcher = cache_matcher;
7489         DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
7490                 key->domain ? "FDB" : "NIC", key->table_id,
7491                 cache_matcher->priority,
7492                 key->direction ? "tx" : "rx", (void *)cache_matcher,
7493                 rte_atomic32_read(&cache_matcher->refcnt));
7494         return 0;
7495 }
7496
7497 /**
7498  * Find existing tag resource or create and register a new one.
7499  *
7500  * @param dev[in, out]
7501  *   Pointer to rte_eth_dev structure.
7502  * @param[in, out] tag_be24
7503  *   Tag value in big endian then R-shift 8.
7504  * @parm[in, out] dev_flow
7505  *   Pointer to the dev_flow.
7506  * @param[out] error
7507  *   pointer to error structure.
7508  *
7509  * @return
7510  *   0 on success otherwise -errno and errno is set.
7511  */
7512 static int
7513 flow_dv_tag_resource_register
7514                         (struct rte_eth_dev *dev,
7515                          uint32_t tag_be24,
7516                          struct mlx5_flow *dev_flow,
7517                          struct rte_flow_error *error)
7518 {
7519         struct mlx5_priv *priv = dev->data->dev_private;
7520         struct mlx5_dev_ctx_shared *sh = priv->sh;
7521         struct mlx5_flow_dv_tag_resource *cache_resource;
7522         struct mlx5_hlist_entry *entry;
7523
7524         /* Lookup a matching resource from cache. */
7525         entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
7526         if (entry) {
7527                 cache_resource = container_of
7528                         (entry, struct mlx5_flow_dv_tag_resource, entry);
7529                 rte_atomic32_inc(&cache_resource->refcnt);
7530                 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
7531                 dev_flow->dv.tag_resource = cache_resource;
7532                 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
7533                         (void *)cache_resource,
7534                         rte_atomic32_read(&cache_resource->refcnt));
7535                 return 0;
7536         }
7537         /* Register new resource. */
7538         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG],
7539                                        &dev_flow->handle->dvh.rix_tag);
7540         if (!cache_resource)
7541                 return rte_flow_error_set(error, ENOMEM,
7542                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7543                                           "cannot allocate resource memory");
7544         cache_resource->entry.key = (uint64_t)tag_be24;
7545         cache_resource->action = mlx5_glue->dv_create_flow_action_tag(tag_be24);
7546         if (!cache_resource->action) {
7547                 rte_free(cache_resource);
7548                 return rte_flow_error_set(error, ENOMEM,
7549                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7550                                           NULL, "cannot create action");
7551         }
7552         rte_atomic32_init(&cache_resource->refcnt);
7553         rte_atomic32_inc(&cache_resource->refcnt);
7554         if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
7555                 mlx5_glue->destroy_flow_action(cache_resource->action);
7556                 rte_free(cache_resource);
7557                 return rte_flow_error_set(error, EEXIST,
7558                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7559                                           NULL, "cannot insert tag");
7560         }
7561         dev_flow->dv.tag_resource = cache_resource;
7562         DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
7563                 (void *)cache_resource,
7564                 rte_atomic32_read(&cache_resource->refcnt));
7565         return 0;
7566 }
7567
7568 /**
7569  * Release the tag.
7570  *
7571  * @param dev
7572  *   Pointer to Ethernet device.
7573  * @param tag_idx
7574  *   Tag index.
7575  *
7576  * @return
7577  *   1 while a reference on it exists, 0 when freed.
7578  */
7579 static int
7580 flow_dv_tag_release(struct rte_eth_dev *dev,
7581                     uint32_t tag_idx)
7582 {
7583         struct mlx5_priv *priv = dev->data->dev_private;
7584         struct mlx5_dev_ctx_shared *sh = priv->sh;
7585         struct mlx5_flow_dv_tag_resource *tag;
7586
7587         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
7588         if (!tag)
7589                 return 0;
7590         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
7591                 dev->data->port_id, (void *)tag,
7592                 rte_atomic32_read(&tag->refcnt));
7593         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
7594                 claim_zero(mlx5_glue->destroy_flow_action(tag->action));
7595                 mlx5_hlist_remove(sh->tag_table, &tag->entry);
7596                 DRV_LOG(DEBUG, "port %u tag %p: removed",
7597                         dev->data->port_id, (void *)tag);
7598                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
7599                 return 0;
7600         }
7601         return 1;
7602 }
7603
7604 /**
7605  * Translate port ID action to vport.
7606  *
7607  * @param[in] dev
7608  *   Pointer to rte_eth_dev structure.
7609  * @param[in] action
7610  *   Pointer to the port ID action.
7611  * @param[out] dst_port_id
7612  *   The target port ID.
7613  * @param[out] error
7614  *   Pointer to the error structure.
7615  *
7616  * @return
7617  *   0 on success, a negative errno value otherwise and rte_errno is set.
7618  */
7619 static int
7620 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
7621                                  const struct rte_flow_action *action,
7622                                  uint32_t *dst_port_id,
7623                                  struct rte_flow_error *error)
7624 {
7625         uint32_t port;
7626         struct mlx5_priv *priv;
7627         const struct rte_flow_action_port_id *conf =
7628                         (const struct rte_flow_action_port_id *)action->conf;
7629
7630         port = conf->original ? dev->data->port_id : conf->id;
7631         priv = mlx5_port_to_eswitch_info(port, false);
7632         if (!priv)
7633                 return rte_flow_error_set(error, -rte_errno,
7634                                           RTE_FLOW_ERROR_TYPE_ACTION,
7635                                           NULL,
7636                                           "No eswitch info was found for port");
7637 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
7638         /*
7639          * This parameter is transferred to
7640          * mlx5dv_dr_action_create_dest_ib_port().
7641          */
7642         *dst_port_id = priv->dev_port;
7643 #else
7644         /*
7645          * Legacy mode, no LAG configurations is supported.
7646          * This parameter is transferred to
7647          * mlx5dv_dr_action_create_dest_vport().
7648          */
7649         *dst_port_id = priv->vport_id;
7650 #endif
7651         return 0;
7652 }
7653
7654 /**
7655  * Create a counter with aging configuration.
7656  *
7657  * @param[in] dev
7658  *   Pointer to rte_eth_dev structure.
7659  * @param[out] count
7660  *   Pointer to the counter action configuration.
7661  * @param[in] age
7662  *   Pointer to the aging action configuration.
7663  *
7664  * @return
7665  *   Index to flow counter on success, 0 otherwise.
7666  */
7667 static uint32_t
7668 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
7669                                 struct mlx5_flow *dev_flow,
7670                                 const struct rte_flow_action_count *count,
7671                                 const struct rte_flow_action_age *age)
7672 {
7673         uint32_t counter;
7674         struct mlx5_age_param *age_param;
7675
7676         counter = flow_dv_counter_alloc(dev,
7677                                 count ? count->shared : 0,
7678                                 count ? count->id : 0,
7679                                 dev_flow->dv.group, !!age);
7680         if (!counter || age == NULL)
7681                 return counter;
7682         age_param  = flow_dv_counter_idx_get_age(dev, counter);
7683         /*
7684          * The counter age accuracy may have a bit delay. Have 3/4
7685          * second bias on the timeount in order to let it age in time.
7686          */
7687         age_param->context = age->context ? age->context :
7688                 (void *)(uintptr_t)(dev_flow->flow_idx);
7689         /*
7690          * The counter age accuracy may have a bit delay. Have 3/4
7691          * second bias on the timeount in order to let it age in time.
7692          */
7693         age_param->timeout = age->timeout * 10 - MLX5_AGING_TIME_DELAY;
7694         /* Set expire time in unit of 0.1 sec. */
7695         age_param->port_id = dev->data->port_id;
7696         age_param->expire = age_param->timeout +
7697                         rte_rdtsc() / (rte_get_tsc_hz() / 10);
7698         rte_atomic16_set(&age_param->state, AGE_CANDIDATE);
7699         return counter;
7700 }
7701 /**
7702  * Add Tx queue matcher
7703  *
7704  * @param[in] dev
7705  *   Pointer to the dev struct.
7706  * @param[in, out] matcher
7707  *   Flow matcher.
7708  * @param[in, out] key
7709  *   Flow matcher value.
7710  * @param[in] item
7711  *   Flow pattern to translate.
7712  * @param[in] inner
7713  *   Item is inner pattern.
7714  */
7715 static void
7716 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
7717                                 void *matcher, void *key,
7718                                 const struct rte_flow_item *item)
7719 {
7720         const struct mlx5_rte_flow_item_tx_queue *queue_m;
7721         const struct mlx5_rte_flow_item_tx_queue *queue_v;
7722         void *misc_m =
7723                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7724         void *misc_v =
7725                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7726         struct mlx5_txq_ctrl *txq;
7727         uint32_t queue;
7728
7729
7730         queue_m = (const void *)item->mask;
7731         if (!queue_m)
7732                 return;
7733         queue_v = (const void *)item->spec;
7734         if (!queue_v)
7735                 return;
7736         txq = mlx5_txq_get(dev, queue_v->queue);
7737         if (!txq)
7738                 return;
7739         queue = txq->obj->sq->id;
7740         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
7741         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
7742                  queue & queue_m->queue);
7743         mlx5_txq_release(dev, queue_v->queue);
7744 }
7745
7746 /**
7747  * Set the hash fields according to the @p flow information.
7748  *
7749  * @param[in] dev_flow
7750  *   Pointer to the mlx5_flow.
7751  * @param[in] rss_desc
7752  *   Pointer to the mlx5_flow_rss_desc.
7753  */
7754 static void
7755 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
7756                        struct mlx5_flow_rss_desc *rss_desc)
7757 {
7758         uint64_t items = dev_flow->handle->layers;
7759         int rss_inner = 0;
7760         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
7761
7762         dev_flow->hash_fields = 0;
7763 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
7764         if (rss_desc->level >= 2) {
7765                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
7766                 rss_inner = 1;
7767         }
7768 #endif
7769         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
7770             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
7771                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
7772                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
7773                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
7774                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
7775                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
7776                         else
7777                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
7778                 }
7779         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
7780                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
7781                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
7782                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
7783                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
7784                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
7785                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
7786                         else
7787                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
7788                 }
7789         }
7790         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
7791             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
7792                 if (rss_types & ETH_RSS_UDP) {
7793                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
7794                                 dev_flow->hash_fields |=
7795                                                 IBV_RX_HASH_SRC_PORT_UDP;
7796                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
7797                                 dev_flow->hash_fields |=
7798                                                 IBV_RX_HASH_DST_PORT_UDP;
7799                         else
7800                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
7801                 }
7802         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
7803                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
7804                 if (rss_types & ETH_RSS_TCP) {
7805                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
7806                                 dev_flow->hash_fields |=
7807                                                 IBV_RX_HASH_SRC_PORT_TCP;
7808                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
7809                                 dev_flow->hash_fields |=
7810                                                 IBV_RX_HASH_DST_PORT_TCP;
7811                         else
7812                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
7813                 }
7814         }
7815 }
7816
7817 /**
7818  * Fill the flow with DV spec, lock free
7819  * (mutex should be acquired by caller).
7820  *
7821  * @param[in] dev
7822  *   Pointer to rte_eth_dev structure.
7823  * @param[in, out] dev_flow
7824  *   Pointer to the sub flow.
7825  * @param[in] attr
7826  *   Pointer to the flow attributes.
7827  * @param[in] items
7828  *   Pointer to the list of items.
7829  * @param[in] actions
7830  *   Pointer to the list of actions.
7831  * @param[out] error
7832  *   Pointer to the error structure.
7833  *
7834  * @return
7835  *   0 on success, a negative errno value otherwise and rte_errno is set.
7836  */
7837 static int
7838 __flow_dv_translate(struct rte_eth_dev *dev,
7839                     struct mlx5_flow *dev_flow,
7840                     const struct rte_flow_attr *attr,
7841                     const struct rte_flow_item items[],
7842                     const struct rte_flow_action actions[],
7843                     struct rte_flow_error *error)
7844 {
7845         struct mlx5_priv *priv = dev->data->dev_private;
7846         struct mlx5_dev_config *dev_conf = &priv->config;
7847         struct rte_flow *flow = dev_flow->flow;
7848         struct mlx5_flow_handle *handle = dev_flow->handle;
7849         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
7850                                               priv->rss_desc)
7851                                               [!!priv->flow_nested_idx];
7852         uint64_t item_flags = 0;
7853         uint64_t last_item = 0;
7854         uint64_t action_flags = 0;
7855         uint64_t priority = attr->priority;
7856         struct mlx5_flow_dv_matcher matcher = {
7857                 .mask = {
7858                         .size = sizeof(matcher.mask.buf),
7859                 },
7860         };
7861         int actions_n = 0;
7862         bool actions_end = false;
7863         union {
7864                 struct mlx5_flow_dv_modify_hdr_resource res;
7865                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
7866                             sizeof(struct mlx5_modification_cmd) *
7867                             (MLX5_MAX_MODIFY_NUM + 1)];
7868         } mhdr_dummy;
7869         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
7870         const struct rte_flow_action_count *count = NULL;
7871         const struct rte_flow_action_age *age = NULL;
7872         union flow_dv_attr flow_attr = { .attr = 0 };
7873         uint32_t tag_be;
7874         union mlx5_flow_tbl_key tbl_key;
7875         uint32_t modify_action_position = UINT32_MAX;
7876         void *match_mask = matcher.mask.buf;
7877         void *match_value = dev_flow->dv.value.buf;
7878         uint8_t next_protocol = 0xff;
7879         struct rte_vlan_hdr vlan = { 0 };
7880         uint32_t table;
7881         int ret = 0;
7882
7883         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
7884                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
7885         ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
7886                                        !!priv->fdb_def_rule, &table, error);
7887         if (ret)
7888                 return ret;
7889         dev_flow->dv.group = table;
7890         if (attr->transfer)
7891                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
7892         if (priority == MLX5_FLOW_PRIO_RSVD)
7893                 priority = dev_conf->flow_prio - 1;
7894         /* number of actions must be set to 0 in case of dirty stack. */
7895         mhdr_res->actions_num = 0;
7896         for (; !actions_end ; actions++) {
7897                 const struct rte_flow_action_queue *queue;
7898                 const struct rte_flow_action_rss *rss;
7899                 const struct rte_flow_action *action = actions;
7900                 const uint8_t *rss_key;
7901                 const struct rte_flow_action_jump *jump_data;
7902                 const struct rte_flow_action_meter *mtr;
7903                 struct mlx5_flow_tbl_resource *tbl;
7904                 uint32_t port_id = 0;
7905                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
7906                 int action_type = actions->type;
7907                 const struct rte_flow_action *found_action = NULL;
7908                 struct mlx5_flow_meter *fm = NULL;
7909
7910                 switch (action_type) {
7911                 case RTE_FLOW_ACTION_TYPE_VOID:
7912                         break;
7913                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
7914                         if (flow_dv_translate_action_port_id(dev, action,
7915                                                              &port_id, error))
7916                                 return -rte_errno;
7917                         port_id_resource.port_id = port_id;
7918                         MLX5_ASSERT(!handle->rix_port_id_action);
7919                         if (flow_dv_port_id_action_resource_register
7920                             (dev, &port_id_resource, dev_flow, error))
7921                                 return -rte_errno;
7922                         dev_flow->dv.actions[actions_n++] =
7923                                         dev_flow->dv.port_id_action->action;
7924                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
7925                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
7926                         break;
7927                 case RTE_FLOW_ACTION_TYPE_FLAG:
7928                         action_flags |= MLX5_FLOW_ACTION_FLAG;
7929                         dev_flow->handle->mark = 1;
7930                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7931                                 struct rte_flow_action_mark mark = {
7932                                         .id = MLX5_FLOW_MARK_DEFAULT,
7933                                 };
7934
7935                                 if (flow_dv_convert_action_mark(dev, &mark,
7936                                                                 mhdr_res,
7937                                                                 error))
7938                                         return -rte_errno;
7939                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
7940                                 break;
7941                         }
7942                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
7943                         /*
7944                          * Only one FLAG or MARK is supported per device flow
7945                          * right now. So the pointer to the tag resource must be
7946                          * zero before the register process.
7947                          */
7948                         MLX5_ASSERT(!handle->dvh.rix_tag);
7949                         if (flow_dv_tag_resource_register(dev, tag_be,
7950                                                           dev_flow, error))
7951                                 return -rte_errno;
7952                         MLX5_ASSERT(dev_flow->dv.tag_resource);
7953                         dev_flow->dv.actions[actions_n++] =
7954                                         dev_flow->dv.tag_resource->action;
7955                         break;
7956                 case RTE_FLOW_ACTION_TYPE_MARK:
7957                         action_flags |= MLX5_FLOW_ACTION_MARK;
7958                         dev_flow->handle->mark = 1;
7959                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7960                                 const struct rte_flow_action_mark *mark =
7961                                         (const struct rte_flow_action_mark *)
7962                                                 actions->conf;
7963
7964                                 if (flow_dv_convert_action_mark(dev, mark,
7965                                                                 mhdr_res,
7966                                                                 error))
7967                                         return -rte_errno;
7968                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
7969                                 break;
7970                         }
7971                         /* Fall-through */
7972                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
7973                         /* Legacy (non-extensive) MARK action. */
7974                         tag_be = mlx5_flow_mark_set
7975                               (((const struct rte_flow_action_mark *)
7976                                (actions->conf))->id);
7977                         MLX5_ASSERT(!handle->dvh.rix_tag);
7978                         if (flow_dv_tag_resource_register(dev, tag_be,
7979                                                           dev_flow, error))
7980                                 return -rte_errno;
7981                         MLX5_ASSERT(dev_flow->dv.tag_resource);
7982                         dev_flow->dv.actions[actions_n++] =
7983                                         dev_flow->dv.tag_resource->action;
7984                         break;
7985                 case RTE_FLOW_ACTION_TYPE_SET_META:
7986                         if (flow_dv_convert_action_set_meta
7987                                 (dev, mhdr_res, attr,
7988                                  (const struct rte_flow_action_set_meta *)
7989                                   actions->conf, error))
7990                                 return -rte_errno;
7991                         action_flags |= MLX5_FLOW_ACTION_SET_META;
7992                         break;
7993                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
7994                         if (flow_dv_convert_action_set_tag
7995                                 (dev, mhdr_res,
7996                                  (const struct rte_flow_action_set_tag *)
7997                                   actions->conf, error))
7998                                 return -rte_errno;
7999                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8000                         break;
8001                 case RTE_FLOW_ACTION_TYPE_DROP:
8002                         action_flags |= MLX5_FLOW_ACTION_DROP;
8003                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
8004                         break;
8005                 case RTE_FLOW_ACTION_TYPE_QUEUE:
8006                         queue = actions->conf;
8007                         rss_desc->queue_num = 1;
8008                         rss_desc->queue[0] = queue->index;
8009                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
8010                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
8011                         break;
8012                 case RTE_FLOW_ACTION_TYPE_RSS:
8013                         rss = actions->conf;
8014                         memcpy(rss_desc->queue, rss->queue,
8015                                rss->queue_num * sizeof(uint16_t));
8016                         rss_desc->queue_num = rss->queue_num;
8017                         /* NULL RSS key indicates default RSS key. */
8018                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
8019                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
8020                         /*
8021                          * rss->level and rss.types should be set in advance
8022                          * when expanding items for RSS.
8023                          */
8024                         action_flags |= MLX5_FLOW_ACTION_RSS;
8025                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
8026                         break;
8027                 case RTE_FLOW_ACTION_TYPE_AGE:
8028                 case RTE_FLOW_ACTION_TYPE_COUNT:
8029                         if (!dev_conf->devx) {
8030                                 return rte_flow_error_set
8031                                               (error, ENOTSUP,
8032                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8033                                                NULL,
8034                                                "count action not supported");
8035                         }
8036                         /* Save information first, will apply later. */
8037                         if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT)
8038                                 count = action->conf;
8039                         else
8040                                 age = action->conf;
8041                         action_flags |= MLX5_FLOW_ACTION_COUNT;
8042                         break;
8043                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
8044                         dev_flow->dv.actions[actions_n++] =
8045                                                 priv->sh->pop_vlan_action;
8046                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
8047                         break;
8048                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
8049                         if (!(action_flags &
8050                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
8051                                 flow_dev_get_vlan_info_from_items(items, &vlan);
8052                         vlan.eth_proto = rte_be_to_cpu_16
8053                              ((((const struct rte_flow_action_of_push_vlan *)
8054                                                    actions->conf)->ethertype));
8055                         found_action = mlx5_flow_find_action
8056                                         (actions + 1,
8057                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
8058                         if (found_action)
8059                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
8060                         found_action = mlx5_flow_find_action
8061                                         (actions + 1,
8062                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
8063                         if (found_action)
8064                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
8065                         if (flow_dv_create_action_push_vlan
8066                                             (dev, attr, &vlan, dev_flow, error))
8067                                 return -rte_errno;
8068                         dev_flow->dv.actions[actions_n++] =
8069                                         dev_flow->dv.push_vlan_res->action;
8070                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
8071                         break;
8072                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
8073                         /* of_vlan_push action handled this action */
8074                         MLX5_ASSERT(action_flags &
8075                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
8076                         break;
8077                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
8078                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
8079                                 break;
8080                         flow_dev_get_vlan_info_from_items(items, &vlan);
8081                         mlx5_update_vlan_vid_pcp(actions, &vlan);
8082                         /* If no VLAN push - this is a modify header action */
8083                         if (flow_dv_convert_action_modify_vlan_vid
8084                                                 (mhdr_res, actions, error))
8085                                 return -rte_errno;
8086                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
8087                         break;
8088                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
8089                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
8090                         if (flow_dv_create_action_l2_encap(dev, actions,
8091                                                            dev_flow,
8092                                                            attr->transfer,
8093                                                            error))
8094                                 return -rte_errno;
8095                         dev_flow->dv.actions[actions_n++] =
8096                                         dev_flow->dv.encap_decap->verbs_action;
8097                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
8098                         break;
8099                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
8100                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
8101                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
8102                                                            attr->transfer,
8103                                                            error))
8104                                 return -rte_errno;
8105                         dev_flow->dv.actions[actions_n++] =
8106                                         dev_flow->dv.encap_decap->verbs_action;
8107                         action_flags |= MLX5_FLOW_ACTION_DECAP;
8108                         break;
8109                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
8110                         /* Handle encap with preceding decap. */
8111                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
8112                                 if (flow_dv_create_action_raw_encap
8113                                         (dev, actions, dev_flow, attr, error))
8114                                         return -rte_errno;
8115                                 dev_flow->dv.actions[actions_n++] =
8116                                         dev_flow->dv.encap_decap->verbs_action;
8117                         } else {
8118                                 /* Handle encap without preceding decap. */
8119                                 if (flow_dv_create_action_l2_encap
8120                                     (dev, actions, dev_flow, attr->transfer,
8121                                      error))
8122                                         return -rte_errno;
8123                                 dev_flow->dv.actions[actions_n++] =
8124                                         dev_flow->dv.encap_decap->verbs_action;
8125                         }
8126                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
8127                         break;
8128                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
8129                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
8130                                 ;
8131                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
8132                                 if (flow_dv_create_action_l2_decap
8133                                     (dev, dev_flow, attr->transfer, error))
8134                                         return -rte_errno;
8135                                 dev_flow->dv.actions[actions_n++] =
8136                                         dev_flow->dv.encap_decap->verbs_action;
8137                         }
8138                         /* If decap is followed by encap, handle it at encap. */
8139                         action_flags |= MLX5_FLOW_ACTION_DECAP;
8140                         break;
8141                 case RTE_FLOW_ACTION_TYPE_JUMP:
8142                         jump_data = action->conf;
8143                         ret = mlx5_flow_group_to_table(attr, dev_flow->external,
8144                                                        jump_data->group,
8145                                                        !!priv->fdb_def_rule,
8146                                                        &table, error);
8147                         if (ret)
8148                                 return ret;
8149                         tbl = flow_dv_tbl_resource_get(dev, table,
8150                                                        attr->egress,
8151                                                        attr->transfer, error);
8152                         if (!tbl)
8153                                 return rte_flow_error_set
8154                                                 (error, errno,
8155                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8156                                                  NULL,
8157                                                  "cannot create jump action.");
8158                         if (flow_dv_jump_tbl_resource_register
8159                             (dev, tbl, dev_flow, error)) {
8160                                 flow_dv_tbl_resource_release(dev, tbl);
8161                                 return rte_flow_error_set
8162                                                 (error, errno,
8163                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8164                                                  NULL,
8165                                                  "cannot create jump action.");
8166                         }
8167                         dev_flow->dv.actions[actions_n++] =
8168                                         dev_flow->dv.jump->action;
8169                         action_flags |= MLX5_FLOW_ACTION_JUMP;
8170                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
8171                         break;
8172                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
8173                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
8174                         if (flow_dv_convert_action_modify_mac
8175                                         (mhdr_res, actions, error))
8176                                 return -rte_errno;
8177                         action_flags |= actions->type ==
8178                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
8179                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
8180                                         MLX5_FLOW_ACTION_SET_MAC_DST;
8181                         break;
8182                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
8183                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
8184                         if (flow_dv_convert_action_modify_ipv4
8185                                         (mhdr_res, actions, error))
8186                                 return -rte_errno;
8187                         action_flags |= actions->type ==
8188                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
8189                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
8190                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
8191                         break;
8192                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
8193                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
8194                         if (flow_dv_convert_action_modify_ipv6
8195                                         (mhdr_res, actions, error))
8196                                 return -rte_errno;
8197                         action_flags |= actions->type ==
8198                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
8199                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
8200                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
8201                         break;
8202                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
8203                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
8204                         if (flow_dv_convert_action_modify_tp
8205                                         (mhdr_res, actions, items,
8206                                          &flow_attr, dev_flow, !!(action_flags &
8207                                          MLX5_FLOW_ACTION_DECAP), error))
8208                                 return -rte_errno;
8209                         action_flags |= actions->type ==
8210                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
8211                                         MLX5_FLOW_ACTION_SET_TP_SRC :
8212                                         MLX5_FLOW_ACTION_SET_TP_DST;
8213                         break;
8214                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
8215                         if (flow_dv_convert_action_modify_dec_ttl
8216                                         (mhdr_res, items, &flow_attr, dev_flow,
8217                                          !!(action_flags &
8218                                          MLX5_FLOW_ACTION_DECAP), error))
8219                                 return -rte_errno;
8220                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
8221                         break;
8222                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
8223                         if (flow_dv_convert_action_modify_ttl
8224                                         (mhdr_res, actions, items, &flow_attr,
8225                                          dev_flow, !!(action_flags &
8226                                          MLX5_FLOW_ACTION_DECAP), error))
8227                                 return -rte_errno;
8228                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
8229                         break;
8230                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
8231                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
8232                         if (flow_dv_convert_action_modify_tcp_seq
8233                                         (mhdr_res, actions, error))
8234                                 return -rte_errno;
8235                         action_flags |= actions->type ==
8236                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
8237                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
8238                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
8239                         break;
8240
8241                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
8242                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
8243                         if (flow_dv_convert_action_modify_tcp_ack
8244                                         (mhdr_res, actions, error))
8245                                 return -rte_errno;
8246                         action_flags |= actions->type ==
8247                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
8248                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
8249                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
8250                         break;
8251                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
8252                         if (flow_dv_convert_action_set_reg
8253                                         (mhdr_res, actions, error))
8254                                 return -rte_errno;
8255                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8256                         break;
8257                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
8258                         if (flow_dv_convert_action_copy_mreg
8259                                         (dev, mhdr_res, actions, error))
8260                                 return -rte_errno;
8261                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8262                         break;
8263                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
8264                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
8265                         dev_flow->handle->fate_action =
8266                                         MLX5_FLOW_FATE_DEFAULT_MISS;
8267                         break;
8268                 case RTE_FLOW_ACTION_TYPE_METER:
8269                         mtr = actions->conf;
8270                         if (!flow->meter) {
8271                                 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
8272                                                             attr, error);
8273                                 if (!fm)
8274                                         return rte_flow_error_set(error,
8275                                                 rte_errno,
8276                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8277                                                 NULL,
8278                                                 "meter not found "
8279                                                 "or invalid parameters");
8280                                 flow->meter = fm->idx;
8281                         }
8282                         /* Set the meter action. */
8283                         if (!fm) {
8284                                 fm = mlx5_ipool_get(priv->sh->ipool
8285                                                 [MLX5_IPOOL_MTR], flow->meter);
8286                                 if (!fm)
8287                                         return rte_flow_error_set(error,
8288                                                 rte_errno,
8289                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8290                                                 NULL,
8291                                                 "meter not found "
8292                                                 "or invalid parameters");
8293                         }
8294                         dev_flow->dv.actions[actions_n++] =
8295                                 fm->mfts->meter_action;
8296                         action_flags |= MLX5_FLOW_ACTION_METER;
8297                         break;
8298                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
8299                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
8300                                                               actions, error))
8301                                 return -rte_errno;
8302                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
8303                         break;
8304                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
8305                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
8306                                                               actions, error))
8307                                 return -rte_errno;
8308                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
8309                         break;
8310                 case RTE_FLOW_ACTION_TYPE_END:
8311                         actions_end = true;
8312                         if (mhdr_res->actions_num) {
8313                                 /* create modify action if needed. */
8314                                 if (flow_dv_modify_hdr_resource_register
8315                                         (dev, mhdr_res, dev_flow, error))
8316                                         return -rte_errno;
8317                                 dev_flow->dv.actions[modify_action_position] =
8318                                         handle->dvh.modify_hdr->verbs_action;
8319                         }
8320                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
8321                                 flow->counter =
8322                                         flow_dv_translate_create_counter(dev,
8323                                                 dev_flow, count, age);
8324
8325                                 if (!flow->counter)
8326                                         return rte_flow_error_set
8327                                                 (error, rte_errno,
8328                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8329                                                 NULL,
8330                                                 "cannot create counter"
8331                                                 " object.");
8332                                 dev_flow->dv.actions[actions_n++] =
8333                                           (flow_dv_counter_get_by_idx(dev,
8334                                           flow->counter, NULL))->action;
8335                         }
8336                         break;
8337                 default:
8338                         break;
8339                 }
8340                 if (mhdr_res->actions_num &&
8341                     modify_action_position == UINT32_MAX)
8342                         modify_action_position = actions_n++;
8343         }
8344         dev_flow->dv.actions_n = actions_n;
8345         dev_flow->act_flags = action_flags;
8346         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
8347                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
8348                 int item_type = items->type;
8349
8350                 switch (item_type) {
8351                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
8352                         flow_dv_translate_item_port_id(dev, match_mask,
8353                                                        match_value, items);
8354                         last_item = MLX5_FLOW_ITEM_PORT_ID;
8355                         break;
8356                 case RTE_FLOW_ITEM_TYPE_ETH:
8357                         flow_dv_translate_item_eth(match_mask, match_value,
8358                                                    items, tunnel,
8359                                                    dev_flow->dv.group);
8360                         matcher.priority = action_flags &
8361                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
8362                                         !dev_flow->external ?
8363                                         MLX5_PRIORITY_MAP_L3 :
8364                                         MLX5_PRIORITY_MAP_L2;
8365                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
8366                                              MLX5_FLOW_LAYER_OUTER_L2;
8367                         break;
8368                 case RTE_FLOW_ITEM_TYPE_VLAN:
8369                         flow_dv_translate_item_vlan(dev_flow,
8370                                                     match_mask, match_value,
8371                                                     items, tunnel,
8372                                                     dev_flow->dv.group);
8373                         matcher.priority = MLX5_PRIORITY_MAP_L2;
8374                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
8375                                               MLX5_FLOW_LAYER_INNER_VLAN) :
8376                                              (MLX5_FLOW_LAYER_OUTER_L2 |
8377                                               MLX5_FLOW_LAYER_OUTER_VLAN);
8378                         break;
8379                 case RTE_FLOW_ITEM_TYPE_IPV4:
8380                         mlx5_flow_tunnel_ip_check(items, next_protocol,
8381                                                   &item_flags, &tunnel);
8382                         flow_dv_translate_item_ipv4(match_mask, match_value,
8383                                                     items, item_flags, tunnel,
8384                                                     dev_flow->dv.group);
8385                         matcher.priority = MLX5_PRIORITY_MAP_L3;
8386                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
8387                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
8388                         if (items->mask != NULL &&
8389                             ((const struct rte_flow_item_ipv4 *)
8390                              items->mask)->hdr.next_proto_id) {
8391                                 next_protocol =
8392                                         ((const struct rte_flow_item_ipv4 *)
8393                                          (items->spec))->hdr.next_proto_id;
8394                                 next_protocol &=
8395                                         ((const struct rte_flow_item_ipv4 *)
8396                                          (items->mask))->hdr.next_proto_id;
8397                         } else {
8398                                 /* Reset for inner layer. */
8399                                 next_protocol = 0xff;
8400                         }
8401                         break;
8402                 case RTE_FLOW_ITEM_TYPE_IPV6:
8403                         mlx5_flow_tunnel_ip_check(items, next_protocol,
8404                                                   &item_flags, &tunnel);
8405                         flow_dv_translate_item_ipv6(match_mask, match_value,
8406                                                     items, item_flags, tunnel,
8407                                                     dev_flow->dv.group);
8408                         matcher.priority = MLX5_PRIORITY_MAP_L3;
8409                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
8410                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
8411                         if (items->mask != NULL &&
8412                             ((const struct rte_flow_item_ipv6 *)
8413                              items->mask)->hdr.proto) {
8414                                 next_protocol =
8415                                         ((const struct rte_flow_item_ipv6 *)
8416                                          items->spec)->hdr.proto;
8417                                 next_protocol &=
8418                                         ((const struct rte_flow_item_ipv6 *)
8419                                          items->mask)->hdr.proto;
8420                         } else {
8421                                 /* Reset for inner layer. */
8422                                 next_protocol = 0xff;
8423                         }
8424                         break;
8425                 case RTE_FLOW_ITEM_TYPE_TCP:
8426                         flow_dv_translate_item_tcp(match_mask, match_value,
8427                                                    items, tunnel);
8428                         matcher.priority = MLX5_PRIORITY_MAP_L4;
8429                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
8430                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
8431                         break;
8432                 case RTE_FLOW_ITEM_TYPE_UDP:
8433                         flow_dv_translate_item_udp(match_mask, match_value,
8434                                                    items, tunnel);
8435                         matcher.priority = MLX5_PRIORITY_MAP_L4;
8436                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
8437                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
8438                         break;
8439                 case RTE_FLOW_ITEM_TYPE_GRE:
8440                         flow_dv_translate_item_gre(match_mask, match_value,
8441                                                    items, tunnel);
8442                         matcher.priority = rss_desc->level >= 2 ?
8443                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8444                         last_item = MLX5_FLOW_LAYER_GRE;
8445                         break;
8446                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
8447                         flow_dv_translate_item_gre_key(match_mask,
8448                                                        match_value, items);
8449                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
8450                         break;
8451                 case RTE_FLOW_ITEM_TYPE_NVGRE:
8452                         flow_dv_translate_item_nvgre(match_mask, match_value,
8453                                                      items, tunnel);
8454                         matcher.priority = rss_desc->level >= 2 ?
8455                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8456                         last_item = MLX5_FLOW_LAYER_GRE;
8457                         break;
8458                 case RTE_FLOW_ITEM_TYPE_VXLAN:
8459                         flow_dv_translate_item_vxlan(match_mask, match_value,
8460                                                      items, tunnel);
8461                         matcher.priority = rss_desc->level >= 2 ?
8462                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8463                         last_item = MLX5_FLOW_LAYER_VXLAN;
8464                         break;
8465                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
8466                         flow_dv_translate_item_vxlan_gpe(match_mask,
8467                                                          match_value, items,
8468                                                          tunnel);
8469                         matcher.priority = rss_desc->level >= 2 ?
8470                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8471                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
8472                         break;
8473                 case RTE_FLOW_ITEM_TYPE_GENEVE:
8474                         flow_dv_translate_item_geneve(match_mask, match_value,
8475                                                       items, tunnel);
8476                         matcher.priority = rss_desc->level >= 2 ?
8477                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8478                         last_item = MLX5_FLOW_LAYER_GENEVE;
8479                         break;
8480                 case RTE_FLOW_ITEM_TYPE_MPLS:
8481                         flow_dv_translate_item_mpls(match_mask, match_value,
8482                                                     items, last_item, tunnel);
8483                         matcher.priority = rss_desc->level >= 2 ?
8484                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8485                         last_item = MLX5_FLOW_LAYER_MPLS;
8486                         break;
8487                 case RTE_FLOW_ITEM_TYPE_MARK:
8488                         flow_dv_translate_item_mark(dev, match_mask,
8489                                                     match_value, items);
8490                         last_item = MLX5_FLOW_ITEM_MARK;
8491                         break;
8492                 case RTE_FLOW_ITEM_TYPE_META:
8493                         flow_dv_translate_item_meta(dev, match_mask,
8494                                                     match_value, attr, items);
8495                         last_item = MLX5_FLOW_ITEM_METADATA;
8496                         break;
8497                 case RTE_FLOW_ITEM_TYPE_ICMP:
8498                         flow_dv_translate_item_icmp(match_mask, match_value,
8499                                                     items, tunnel);
8500                         last_item = MLX5_FLOW_LAYER_ICMP;
8501                         break;
8502                 case RTE_FLOW_ITEM_TYPE_ICMP6:
8503                         flow_dv_translate_item_icmp6(match_mask, match_value,
8504                                                       items, tunnel);
8505                         last_item = MLX5_FLOW_LAYER_ICMP6;
8506                         break;
8507                 case RTE_FLOW_ITEM_TYPE_TAG:
8508                         flow_dv_translate_item_tag(dev, match_mask,
8509                                                    match_value, items);
8510                         last_item = MLX5_FLOW_ITEM_TAG;
8511                         break;
8512                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
8513                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
8514                                                         match_value, items);
8515                         last_item = MLX5_FLOW_ITEM_TAG;
8516                         break;
8517                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
8518                         flow_dv_translate_item_tx_queue(dev, match_mask,
8519                                                         match_value,
8520                                                         items);
8521                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
8522                         break;
8523                 case RTE_FLOW_ITEM_TYPE_GTP:
8524                         flow_dv_translate_item_gtp(match_mask, match_value,
8525                                                    items, tunnel);
8526                         matcher.priority = rss_desc->level >= 2 ?
8527                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8528                         last_item = MLX5_FLOW_LAYER_GTP;
8529                         break;
8530                 default:
8531                         break;
8532                 }
8533                 item_flags |= last_item;
8534         }
8535         /*
8536          * When E-Switch mode is enabled, we have two cases where we need to
8537          * set the source port manually.
8538          * The first one, is in case of Nic steering rule, and the second is
8539          * E-Switch rule where no port_id item was found. In both cases
8540          * the source port is set according the current port in use.
8541          */
8542         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
8543             (priv->representor || priv->master)) {
8544                 if (flow_dv_translate_item_port_id(dev, match_mask,
8545                                                    match_value, NULL))
8546                         return -rte_errno;
8547         }
8548 #ifdef RTE_LIBRTE_MLX5_DEBUG
8549         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
8550                                               dev_flow->dv.value.buf));
8551 #endif
8552         /*
8553          * Layers may be already initialized from prefix flow if this dev_flow
8554          * is the suffix flow.
8555          */
8556         handle->layers |= item_flags;
8557         if (action_flags & MLX5_FLOW_ACTION_RSS)
8558                 flow_dv_hashfields_set(dev_flow, rss_desc);
8559         /* Register matcher. */
8560         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
8561                                     matcher.mask.size);
8562         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
8563                                                      matcher.priority);
8564         /* reserved field no needs to be set to 0 here. */
8565         tbl_key.domain = attr->transfer;
8566         tbl_key.direction = attr->egress;
8567         tbl_key.table_id = dev_flow->dv.group;
8568         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
8569                 return -rte_errno;
8570         return 0;
8571 }
8572
8573 /**
8574  * Apply the flow to the NIC, lock free,
8575  * (mutex should be acquired by caller).
8576  *
8577  * @param[in] dev
8578  *   Pointer to the Ethernet device structure.
8579  * @param[in, out] flow
8580  *   Pointer to flow structure.
8581  * @param[out] error
8582  *   Pointer to error structure.
8583  *
8584  * @return
8585  *   0 on success, a negative errno value otherwise and rte_errno is set.
8586  */
8587 static int
8588 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
8589                 struct rte_flow_error *error)
8590 {
8591         struct mlx5_flow_dv_workspace *dv;
8592         struct mlx5_flow_handle *dh;
8593         struct mlx5_flow_handle_dv *dv_h;
8594         struct mlx5_flow *dev_flow;
8595         struct mlx5_priv *priv = dev->data->dev_private;
8596         uint32_t handle_idx;
8597         int n;
8598         int err;
8599         int idx;
8600
8601         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
8602                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
8603                 dv = &dev_flow->dv;
8604                 dh = dev_flow->handle;
8605                 dv_h = &dh->dvh;
8606                 n = dv->actions_n;
8607                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
8608                         if (dv->transfer) {
8609                                 dv->actions[n++] = priv->sh->esw_drop_action;
8610                         } else {
8611                                 struct mlx5_hrxq *drop_hrxq;
8612                                 drop_hrxq = mlx5_hrxq_drop_new(dev);
8613                                 if (!drop_hrxq) {
8614                                         rte_flow_error_set
8615                                                 (error, errno,
8616                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8617                                                  NULL,
8618                                                  "cannot get drop hash queue");
8619                                         goto error;
8620                                 }
8621                                 /*
8622                                  * Drop queues will be released by the specify
8623                                  * mlx5_hrxq_drop_release() function. Assign
8624                                  * the special index to hrxq to mark the queue
8625                                  * has been allocated.
8626                                  */
8627                                 dh->rix_hrxq = UINT32_MAX;
8628                                 dv->actions[n++] = drop_hrxq->action;
8629                         }
8630                 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
8631                         struct mlx5_hrxq *hrxq;
8632                         uint32_t hrxq_idx;
8633                         struct mlx5_flow_rss_desc *rss_desc =
8634                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
8635                                 [!!priv->flow_nested_idx];
8636
8637                         MLX5_ASSERT(rss_desc->queue_num);
8638                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
8639                                                  MLX5_RSS_HASH_KEY_LEN,
8640                                                  dev_flow->hash_fields,
8641                                                  rss_desc->queue,
8642                                                  rss_desc->queue_num);
8643                         if (!hrxq_idx) {
8644                                 hrxq_idx = mlx5_hrxq_new
8645                                                 (dev, rss_desc->key,
8646                                                 MLX5_RSS_HASH_KEY_LEN,
8647                                                 dev_flow->hash_fields,
8648                                                 rss_desc->queue,
8649                                                 rss_desc->queue_num,
8650                                                 !!(dh->layers &
8651                                                 MLX5_FLOW_LAYER_TUNNEL));
8652                         }
8653                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
8654                                               hrxq_idx);
8655                         if (!hrxq) {
8656                                 rte_flow_error_set
8657                                         (error, rte_errno,
8658                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8659                                          "cannot get hash queue");
8660                                 goto error;
8661                         }
8662                         dh->rix_hrxq = hrxq_idx;
8663                         dv->actions[n++] = hrxq->action;
8664                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
8665                         if (flow_dv_default_miss_resource_register
8666                                         (dev, error)) {
8667                                 rte_flow_error_set
8668                                         (error, rte_errno,
8669                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8670                                          "cannot create default miss resource");
8671                                 goto error_default_miss;
8672                         }
8673                         dh->rix_default_fate =  MLX5_FLOW_FATE_DEFAULT_MISS;
8674                         dv->actions[n++] = priv->sh->default_miss.action;
8675                 }
8676
8677                 dh->ib_flow =
8678                         mlx5_glue->dv_create_flow(dv_h->matcher->matcher_object,
8679                                                   (void *)&dv->value, n,
8680                                                   dv->actions);
8681                 if (!dh->ib_flow) {
8682                         rte_flow_error_set(error, errno,
8683                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8684                                            NULL,
8685                                            "hardware refuses to create flow");
8686                         goto error;
8687                 }
8688                 if (priv->vmwa_context &&
8689                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
8690                         /*
8691                          * The rule contains the VLAN pattern.
8692                          * For VF we are going to create VLAN
8693                          * interface to make hypervisor set correct
8694                          * e-Switch vport context.
8695                          */
8696                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
8697                 }
8698         }
8699         return 0;
8700 error:
8701         if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
8702                 flow_dv_default_miss_resource_release(dev);
8703 error_default_miss:
8704         err = rte_errno; /* Save rte_errno before cleanup. */
8705         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
8706                        handle_idx, dh, next) {
8707                 /* hrxq is union, don't clear it if the flag is not set. */
8708                 if (dh->rix_hrxq) {
8709                         if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
8710                                 mlx5_hrxq_drop_release(dev);
8711                                 dh->rix_hrxq = 0;
8712                         } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
8713                                 mlx5_hrxq_release(dev, dh->rix_hrxq);
8714                                 dh->rix_hrxq = 0;
8715                         }
8716                 }
8717                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
8718                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
8719         }
8720         rte_errno = err; /* Restore rte_errno. */
8721         return -rte_errno;
8722 }
8723
8724 /**
8725  * Release the flow matcher.
8726  *
8727  * @param dev
8728  *   Pointer to Ethernet device.
8729  * @param handle
8730  *   Pointer to mlx5_flow_handle.
8731  *
8732  * @return
8733  *   1 while a reference on it exists, 0 when freed.
8734  */
8735 static int
8736 flow_dv_matcher_release(struct rte_eth_dev *dev,
8737                         struct mlx5_flow_handle *handle)
8738 {
8739         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
8740
8741         MLX5_ASSERT(matcher->matcher_object);
8742         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
8743                 dev->data->port_id, (void *)matcher,
8744                 rte_atomic32_read(&matcher->refcnt));
8745         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
8746                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
8747                            (matcher->matcher_object));
8748                 LIST_REMOVE(matcher, next);
8749                 /* table ref-- in release interface. */
8750                 flow_dv_tbl_resource_release(dev, matcher->tbl);
8751                 rte_free(matcher);
8752                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
8753                         dev->data->port_id, (void *)matcher);
8754                 return 0;
8755         }
8756         return 1;
8757 }
8758
8759 /**
8760  * Release an encap/decap resource.
8761  *
8762  * @param dev
8763  *   Pointer to Ethernet device.
8764  * @param handle
8765  *   Pointer to mlx5_flow_handle.
8766  *
8767  * @return
8768  *   1 while a reference on it exists, 0 when freed.
8769  */
8770 static int
8771 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
8772                                      struct mlx5_flow_handle *handle)
8773 {
8774         struct mlx5_priv *priv = dev->data->dev_private;
8775         uint32_t idx = handle->dvh.rix_encap_decap;
8776         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
8777
8778         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
8779                          idx);
8780         if (!cache_resource)
8781                 return 0;
8782         MLX5_ASSERT(cache_resource->verbs_action);
8783         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
8784                 (void *)cache_resource,
8785                 rte_atomic32_read(&cache_resource->refcnt));
8786         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8787                 claim_zero(mlx5_glue->destroy_flow_action
8788                                 (cache_resource->verbs_action));
8789                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
8790                              &priv->sh->encaps_decaps, idx,
8791                              cache_resource, next);
8792                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
8793                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
8794                         (void *)cache_resource);
8795                 return 0;
8796         }
8797         return 1;
8798 }
8799
8800 /**
8801  * Release an jump to table action resource.
8802  *
8803  * @param dev
8804  *   Pointer to Ethernet device.
8805  * @param handle
8806  *   Pointer to mlx5_flow_handle.
8807  *
8808  * @return
8809  *   1 while a reference on it exists, 0 when freed.
8810  */
8811 static int
8812 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
8813                                   struct mlx5_flow_handle *handle)
8814 {
8815         struct mlx5_priv *priv = dev->data->dev_private;
8816         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
8817         struct mlx5_flow_tbl_data_entry *tbl_data;
8818
8819         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
8820                              handle->rix_jump);
8821         if (!tbl_data)
8822                 return 0;
8823         cache_resource = &tbl_data->jump;
8824         MLX5_ASSERT(cache_resource->action);
8825         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
8826                 (void *)cache_resource,
8827                 rte_atomic32_read(&cache_resource->refcnt));
8828         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8829                 claim_zero(mlx5_glue->destroy_flow_action
8830                                 (cache_resource->action));
8831                 /* jump action memory free is inside the table release. */
8832                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
8833                 DRV_LOG(DEBUG, "jump table resource %p: removed",
8834                         (void *)cache_resource);
8835                 return 0;
8836         }
8837         return 1;
8838 }
8839
8840 /**
8841  * Release a default miss resource.
8842  *
8843  * @param dev
8844  *   Pointer to Ethernet device.
8845  * @return
8846  *   1 while a reference on it exists, 0 when freed.
8847  */
8848 static int
8849 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
8850 {
8851         struct mlx5_priv *priv = dev->data->dev_private;
8852         struct mlx5_dev_ctx_shared *sh = priv->sh;
8853         struct mlx5_flow_default_miss_resource *cache_resource =
8854                         &sh->default_miss;
8855
8856         MLX5_ASSERT(cache_resource->action);
8857         DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--",
8858                         (void *)cache_resource->action,
8859                         rte_atomic32_read(&cache_resource->refcnt));
8860         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8861                 claim_zero(mlx5_glue->destroy_flow_action
8862                                 (cache_resource->action));
8863                 DRV_LOG(DEBUG, "default miss resource %p: removed",
8864                                 (void *)cache_resource->action);
8865                 return 0;
8866         }
8867         return 1;
8868 }
8869
8870 /**
8871  * Release a modify-header resource.
8872  *
8873  * @param handle
8874  *   Pointer to mlx5_flow_handle.
8875  *
8876  * @return
8877  *   1 while a reference on it exists, 0 when freed.
8878  */
8879 static int
8880 flow_dv_modify_hdr_resource_release(struct mlx5_flow_handle *handle)
8881 {
8882         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
8883                                                         handle->dvh.modify_hdr;
8884
8885         MLX5_ASSERT(cache_resource->verbs_action);
8886         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
8887                 (void *)cache_resource,
8888                 rte_atomic32_read(&cache_resource->refcnt));
8889         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8890                 claim_zero(mlx5_glue->destroy_flow_action
8891                                 (cache_resource->verbs_action));
8892                 LIST_REMOVE(cache_resource, next);
8893                 rte_free(cache_resource);
8894                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
8895                         (void *)cache_resource);
8896                 return 0;
8897         }
8898         return 1;
8899 }
8900
8901 /**
8902  * Release port ID action resource.
8903  *
8904  * @param dev
8905  *   Pointer to Ethernet device.
8906  * @param handle
8907  *   Pointer to mlx5_flow_handle.
8908  *
8909  * @return
8910  *   1 while a reference on it exists, 0 when freed.
8911  */
8912 static int
8913 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
8914                                         struct mlx5_flow_handle *handle)
8915 {
8916         struct mlx5_priv *priv = dev->data->dev_private;
8917         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
8918         uint32_t idx = handle->rix_port_id_action;
8919
8920         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
8921                                         idx);
8922         if (!cache_resource)
8923                 return 0;
8924         MLX5_ASSERT(cache_resource->action);
8925         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
8926                 (void *)cache_resource,
8927                 rte_atomic32_read(&cache_resource->refcnt));
8928         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8929                 claim_zero(mlx5_glue->destroy_flow_action
8930                                 (cache_resource->action));
8931                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
8932                              &priv->sh->port_id_action_list, idx,
8933                              cache_resource, next);
8934                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx);
8935                 DRV_LOG(DEBUG, "port id action resource %p: removed",
8936                         (void *)cache_resource);
8937                 return 0;
8938         }
8939         return 1;
8940 }
8941
8942 /**
8943  * Release push vlan action resource.
8944  *
8945  * @param dev
8946  *   Pointer to Ethernet device.
8947  * @param handle
8948  *   Pointer to mlx5_flow_handle.
8949  *
8950  * @return
8951  *   1 while a reference on it exists, 0 when freed.
8952  */
8953 static int
8954 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
8955                                           struct mlx5_flow_handle *handle)
8956 {
8957         struct mlx5_priv *priv = dev->data->dev_private;
8958         uint32_t idx = handle->dvh.rix_push_vlan;
8959         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
8960
8961         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
8962                                         idx);
8963         if (!cache_resource)
8964                 return 0;
8965         MLX5_ASSERT(cache_resource->action);
8966         DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
8967                 (void *)cache_resource,
8968                 rte_atomic32_read(&cache_resource->refcnt));
8969         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8970                 claim_zero(mlx5_glue->destroy_flow_action
8971                                 (cache_resource->action));
8972                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
8973                              &priv->sh->push_vlan_action_list, idx,
8974                              cache_resource, next);
8975                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
8976                 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
8977                         (void *)cache_resource);
8978                 return 0;
8979         }
8980         return 1;
8981 }
8982
8983 /**
8984  * Release the fate resource.
8985  *
8986  * @param dev
8987  *   Pointer to Ethernet device.
8988  * @param handle
8989  *   Pointer to mlx5_flow_handle.
8990  */
8991 static void
8992 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
8993                                struct mlx5_flow_handle *handle)
8994 {
8995         if (!handle->rix_fate)
8996                 return;
8997         switch (handle->fate_action) {
8998         case MLX5_FLOW_FATE_DROP:
8999                 mlx5_hrxq_drop_release(dev);
9000                 break;
9001         case MLX5_FLOW_FATE_QUEUE:
9002                 mlx5_hrxq_release(dev, handle->rix_hrxq);
9003                 break;
9004         case MLX5_FLOW_FATE_JUMP:
9005                 flow_dv_jump_tbl_resource_release(dev, handle);
9006                 break;
9007         case MLX5_FLOW_FATE_PORT_ID:
9008                 flow_dv_port_id_action_resource_release(dev, handle);
9009                 break;
9010         case MLX5_FLOW_FATE_DEFAULT_MISS:
9011                 flow_dv_default_miss_resource_release(dev);
9012                 break;
9013         default:
9014                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
9015                 break;
9016         }
9017         handle->rix_fate = 0;
9018 }
9019
9020 /**
9021  * Remove the flow from the NIC but keeps it in memory.
9022  * Lock free, (mutex should be acquired by caller).
9023  *
9024  * @param[in] dev
9025  *   Pointer to Ethernet device.
9026  * @param[in, out] flow
9027  *   Pointer to flow structure.
9028  */
9029 static void
9030 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
9031 {
9032         struct mlx5_flow_handle *dh;
9033         uint32_t handle_idx;
9034         struct mlx5_priv *priv = dev->data->dev_private;
9035
9036         if (!flow)
9037                 return;
9038         handle_idx = flow->dev_handles;
9039         while (handle_idx) {
9040                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9041                                     handle_idx);
9042                 if (!dh)
9043                         return;
9044                 if (dh->ib_flow) {
9045                         claim_zero(mlx5_glue->dv_destroy_flow(dh->ib_flow));
9046                         dh->ib_flow = NULL;
9047                 }
9048                 if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
9049                     dh->fate_action == MLX5_FLOW_FATE_QUEUE ||
9050                     dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
9051                         flow_dv_fate_resource_release(dev, dh);
9052                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
9053                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
9054                 handle_idx = dh->next.next;
9055         }
9056 }
9057
9058 /**
9059  * Remove the flow from the NIC and the memory.
9060  * Lock free, (mutex should be acquired by caller).
9061  *
9062  * @param[in] dev
9063  *   Pointer to the Ethernet device structure.
9064  * @param[in, out] flow
9065  *   Pointer to flow structure.
9066  */
9067 static void
9068 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
9069 {
9070         struct mlx5_flow_handle *dev_handle;
9071         struct mlx5_priv *priv = dev->data->dev_private;
9072
9073         if (!flow)
9074                 return;
9075         __flow_dv_remove(dev, flow);
9076         if (flow->counter) {
9077                 flow_dv_counter_release(dev, flow->counter);
9078                 flow->counter = 0;
9079         }
9080         if (flow->meter) {
9081                 struct mlx5_flow_meter *fm;
9082
9083                 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
9084                                     flow->meter);
9085                 if (fm)
9086                         mlx5_flow_meter_detach(fm);
9087                 flow->meter = 0;
9088         }
9089         while (flow->dev_handles) {
9090                 uint32_t tmp_idx = flow->dev_handles;
9091
9092                 dev_handle = mlx5_ipool_get(priv->sh->ipool
9093                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
9094                 if (!dev_handle)
9095                         return;
9096                 flow->dev_handles = dev_handle->next.next;
9097                 if (dev_handle->dvh.matcher)
9098                         flow_dv_matcher_release(dev, dev_handle);
9099                 if (dev_handle->dvh.rix_encap_decap)
9100                         flow_dv_encap_decap_resource_release(dev, dev_handle);
9101                 if (dev_handle->dvh.modify_hdr)
9102                         flow_dv_modify_hdr_resource_release(dev_handle);
9103                 if (dev_handle->dvh.rix_push_vlan)
9104                         flow_dv_push_vlan_action_resource_release(dev,
9105                                                                   dev_handle);
9106                 if (dev_handle->dvh.rix_tag)
9107                         flow_dv_tag_release(dev,
9108                                             dev_handle->dvh.rix_tag);
9109                 flow_dv_fate_resource_release(dev, dev_handle);
9110                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9111                            tmp_idx);
9112         }
9113 }
9114
9115 /**
9116  * Query a dv flow  rule for its statistics via devx.
9117  *
9118  * @param[in] dev
9119  *   Pointer to Ethernet device.
9120  * @param[in] flow
9121  *   Pointer to the sub flow.
9122  * @param[out] data
9123  *   data retrieved by the query.
9124  * @param[out] error
9125  *   Perform verbose error reporting if not NULL.
9126  *
9127  * @return
9128  *   0 on success, a negative errno value otherwise and rte_errno is set.
9129  */
9130 static int
9131 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
9132                     void *data, struct rte_flow_error *error)
9133 {
9134         struct mlx5_priv *priv = dev->data->dev_private;
9135         struct rte_flow_query_count *qc = data;
9136
9137         if (!priv->config.devx)
9138                 return rte_flow_error_set(error, ENOTSUP,
9139                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9140                                           NULL,
9141                                           "counters are not supported");
9142         if (flow->counter) {
9143                 uint64_t pkts, bytes;
9144                 struct mlx5_flow_counter *cnt;
9145
9146                 cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
9147                                                  NULL);
9148                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
9149                                                &bytes);
9150
9151                 if (err)
9152                         return rte_flow_error_set(error, -err,
9153                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9154                                         NULL, "cannot read counters");
9155                 qc->hits_set = 1;
9156                 qc->bytes_set = 1;
9157                 qc->hits = pkts - cnt->hits;
9158                 qc->bytes = bytes - cnt->bytes;
9159                 if (qc->reset) {
9160                         cnt->hits = pkts;
9161                         cnt->bytes = bytes;
9162                 }
9163                 return 0;
9164         }
9165         return rte_flow_error_set(error, EINVAL,
9166                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9167                                   NULL,
9168                                   "counters are not available");
9169 }
9170
9171 /**
9172  * Query a flow.
9173  *
9174  * @see rte_flow_query()
9175  * @see rte_flow_ops
9176  */
9177 static int
9178 flow_dv_query(struct rte_eth_dev *dev,
9179               struct rte_flow *flow __rte_unused,
9180               const struct rte_flow_action *actions __rte_unused,
9181               void *data __rte_unused,
9182               struct rte_flow_error *error __rte_unused)
9183 {
9184         int ret = -EINVAL;
9185
9186         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
9187                 switch (actions->type) {
9188                 case RTE_FLOW_ACTION_TYPE_VOID:
9189                         break;
9190                 case RTE_FLOW_ACTION_TYPE_COUNT:
9191                         ret = flow_dv_query_count(dev, flow, data, error);
9192                         break;
9193                 default:
9194                         return rte_flow_error_set(error, ENOTSUP,
9195                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9196                                                   actions,
9197                                                   "action not supported");
9198                 }
9199         }
9200         return ret;
9201 }
9202
9203 /**
9204  * Destroy the meter table set.
9205  * Lock free, (mutex should be acquired by caller).
9206  *
9207  * @param[in] dev
9208  *   Pointer to Ethernet device.
9209  * @param[in] tbl
9210  *   Pointer to the meter table set.
9211  *
9212  * @return
9213  *   Always 0.
9214  */
9215 static int
9216 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
9217                         struct mlx5_meter_domains_infos *tbl)
9218 {
9219         struct mlx5_priv *priv = dev->data->dev_private;
9220         struct mlx5_meter_domains_infos *mtd =
9221                                 (struct mlx5_meter_domains_infos *)tbl;
9222
9223         if (!mtd || !priv->config.dv_flow_en)
9224                 return 0;
9225         if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
9226                 claim_zero(mlx5_glue->dv_destroy_flow
9227                           (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
9228         if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
9229                 claim_zero(mlx5_glue->dv_destroy_flow
9230                           (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
9231         if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
9232                 claim_zero(mlx5_glue->dv_destroy_flow
9233                           (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
9234         if (mtd->egress.color_matcher)
9235                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9236                           (mtd->egress.color_matcher));
9237         if (mtd->egress.any_matcher)
9238                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9239                           (mtd->egress.any_matcher));
9240         if (mtd->egress.tbl)
9241                 flow_dv_tbl_resource_release(dev, mtd->egress.tbl);
9242         if (mtd->egress.sfx_tbl)
9243                 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl);
9244         if (mtd->ingress.color_matcher)
9245                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9246                           (mtd->ingress.color_matcher));
9247         if (mtd->ingress.any_matcher)
9248                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9249                           (mtd->ingress.any_matcher));
9250         if (mtd->ingress.tbl)
9251                 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl);
9252         if (mtd->ingress.sfx_tbl)
9253                 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl);
9254         if (mtd->transfer.color_matcher)
9255                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9256                           (mtd->transfer.color_matcher));
9257         if (mtd->transfer.any_matcher)
9258                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9259                           (mtd->transfer.any_matcher));
9260         if (mtd->transfer.tbl)
9261                 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl);
9262         if (mtd->transfer.sfx_tbl)
9263                 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl);
9264         if (mtd->drop_actn)
9265                 claim_zero(mlx5_glue->destroy_flow_action(mtd->drop_actn));
9266         rte_free(mtd);
9267         return 0;
9268 }
9269
9270 /* Number of meter flow actions, count and jump or count and drop. */
9271 #define METER_ACTIONS 2
9272
9273 /**
9274  * Create specify domain meter table and suffix table.
9275  *
9276  * @param[in] dev
9277  *   Pointer to Ethernet device.
9278  * @param[in,out] mtb
9279  *   Pointer to DV meter table set.
9280  * @param[in] egress
9281  *   Table attribute.
9282  * @param[in] transfer
9283  *   Table attribute.
9284  * @param[in] color_reg_c_idx
9285  *   Reg C index for color match.
9286  *
9287  * @return
9288  *   0 on success, -1 otherwise and rte_errno is set.
9289  */
9290 static int
9291 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
9292                            struct mlx5_meter_domains_infos *mtb,
9293                            uint8_t egress, uint8_t transfer,
9294                            uint32_t color_reg_c_idx)
9295 {
9296         struct mlx5_priv *priv = dev->data->dev_private;
9297         struct mlx5_dev_ctx_shared *sh = priv->sh;
9298         struct mlx5_flow_dv_match_params mask = {
9299                 .size = sizeof(mask.buf),
9300         };
9301         struct mlx5_flow_dv_match_params value = {
9302                 .size = sizeof(value.buf),
9303         };
9304         struct mlx5dv_flow_matcher_attr dv_attr = {
9305                 .type = IBV_FLOW_ATTR_NORMAL,
9306                 .priority = 0,
9307                 .match_criteria_enable = 0,
9308                 .match_mask = (void *)&mask,
9309         };
9310         void *actions[METER_ACTIONS];
9311         struct mlx5_meter_domain_info *dtb;
9312         struct rte_flow_error error;
9313         int i = 0;
9314
9315         if (transfer)
9316                 dtb = &mtb->transfer;
9317         else if (egress)
9318                 dtb = &mtb->egress;
9319         else
9320                 dtb = &mtb->ingress;
9321         /* Create the meter table with METER level. */
9322         dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
9323                                             egress, transfer, &error);
9324         if (!dtb->tbl) {
9325                 DRV_LOG(ERR, "Failed to create meter policer table.");
9326                 return -1;
9327         }
9328         /* Create the meter suffix table with SUFFIX level. */
9329         dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
9330                                             MLX5_FLOW_TABLE_LEVEL_SUFFIX,
9331                                             egress, transfer, &error);
9332         if (!dtb->sfx_tbl) {
9333                 DRV_LOG(ERR, "Failed to create meter suffix table.");
9334                 return -1;
9335         }
9336         /* Create matchers, Any and Color. */
9337         dv_attr.priority = 3;
9338         dv_attr.match_criteria_enable = 0;
9339         dtb->any_matcher = mlx5_glue->dv_create_flow_matcher(sh->ctx,
9340                                                              &dv_attr,
9341                                                              dtb->tbl->obj);
9342         if (!dtb->any_matcher) {
9343                 DRV_LOG(ERR, "Failed to create meter"
9344                              " policer default matcher.");
9345                 goto error_exit;
9346         }
9347         dv_attr.priority = 0;
9348         dv_attr.match_criteria_enable =
9349                                 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
9350         flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
9351                                rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
9352         dtb->color_matcher = mlx5_glue->dv_create_flow_matcher(sh->ctx,
9353                                                                &dv_attr,
9354                                                                dtb->tbl->obj);
9355         if (!dtb->color_matcher) {
9356                 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
9357                 goto error_exit;
9358         }
9359         if (mtb->count_actns[RTE_MTR_DROPPED])
9360                 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
9361         actions[i++] = mtb->drop_actn;
9362         /* Default rule: lowest priority, match any, actions: drop. */
9363         dtb->policer_rules[RTE_MTR_DROPPED] =
9364                         mlx5_glue->dv_create_flow(dtb->any_matcher,
9365                                                  (void *)&value, i, actions);
9366         if (!dtb->policer_rules[RTE_MTR_DROPPED]) {
9367                 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
9368                 goto error_exit;
9369         }
9370         return 0;
9371 error_exit:
9372         return -1;
9373 }
9374
9375 /**
9376  * Create the needed meter and suffix tables.
9377  * Lock free, (mutex should be acquired by caller).
9378  *
9379  * @param[in] dev
9380  *   Pointer to Ethernet device.
9381  * @param[in] fm
9382  *   Pointer to the flow meter.
9383  *
9384  * @return
9385  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
9386  */
9387 static struct mlx5_meter_domains_infos *
9388 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
9389                        const struct mlx5_flow_meter *fm)
9390 {
9391         struct mlx5_priv *priv = dev->data->dev_private;
9392         struct mlx5_meter_domains_infos *mtb;
9393         int ret;
9394         int i;
9395
9396         if (!priv->mtr_en) {
9397                 rte_errno = ENOTSUP;
9398                 return NULL;
9399         }
9400         mtb = rte_calloc(__func__, 1, sizeof(*mtb), 0);
9401         if (!mtb) {
9402                 DRV_LOG(ERR, "Failed to allocate memory for meter.");
9403                 return NULL;
9404         }
9405         /* Create meter count actions */
9406         for (i = 0; i <= RTE_MTR_DROPPED; i++) {
9407                 struct mlx5_flow_counter *cnt;
9408                 if (!fm->policer_stats.cnt[i])
9409                         continue;
9410                 cnt = flow_dv_counter_get_by_idx(dev,
9411                       fm->policer_stats.cnt[i], NULL);
9412                 mtb->count_actns[i] = cnt->action;
9413         }
9414         /* Create drop action. */
9415         mtb->drop_actn = mlx5_glue->dr_create_flow_action_drop();
9416         if (!mtb->drop_actn) {
9417                 DRV_LOG(ERR, "Failed to create drop action.");
9418                 goto error_exit;
9419         }
9420         /* Egress meter table. */
9421         ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
9422         if (ret) {
9423                 DRV_LOG(ERR, "Failed to prepare egress meter table.");
9424                 goto error_exit;
9425         }
9426         /* Ingress meter table. */
9427         ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
9428         if (ret) {
9429                 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
9430                 goto error_exit;
9431         }
9432         /* FDB meter table. */
9433         if (priv->config.dv_esw_en) {
9434                 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
9435                                                  priv->mtr_color_reg);
9436                 if (ret) {
9437                         DRV_LOG(ERR, "Failed to prepare fdb meter table.");
9438                         goto error_exit;
9439                 }
9440         }
9441         return mtb;
9442 error_exit:
9443         flow_dv_destroy_mtr_tbl(dev, mtb);
9444         return NULL;
9445 }
9446
9447 /**
9448  * Destroy domain policer rule.
9449  *
9450  * @param[in] dt
9451  *   Pointer to domain table.
9452  */
9453 static void
9454 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
9455 {
9456         int i;
9457
9458         for (i = 0; i < RTE_MTR_DROPPED; i++) {
9459                 if (dt->policer_rules[i]) {
9460                         claim_zero(mlx5_glue->dv_destroy_flow
9461                                   (dt->policer_rules[i]));
9462                         dt->policer_rules[i] = NULL;
9463                 }
9464         }
9465         if (dt->jump_actn) {
9466                 claim_zero(mlx5_glue->destroy_flow_action(dt->jump_actn));
9467                 dt->jump_actn = NULL;
9468         }
9469 }
9470
9471 /**
9472  * Destroy policer rules.
9473  *
9474  * @param[in] dev
9475  *   Pointer to Ethernet device.
9476  * @param[in] fm
9477  *   Pointer to flow meter structure.
9478  * @param[in] attr
9479  *   Pointer to flow attributes.
9480  *
9481  * @return
9482  *   Always 0.
9483  */
9484 static int
9485 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
9486                               const struct mlx5_flow_meter *fm,
9487                               const struct rte_flow_attr *attr)
9488 {
9489         struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
9490
9491         if (!mtb)
9492                 return 0;
9493         if (attr->egress)
9494                 flow_dv_destroy_domain_policer_rule(&mtb->egress);
9495         if (attr->ingress)
9496                 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
9497         if (attr->transfer)
9498                 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
9499         return 0;
9500 }
9501
9502 /**
9503  * Create specify domain meter policer rule.
9504  *
9505  * @param[in] fm
9506  *   Pointer to flow meter structure.
9507  * @param[in] mtb
9508  *   Pointer to DV meter table set.
9509  * @param[in] mtr_reg_c
9510  *   Color match REG_C.
9511  *
9512  * @return
9513  *   0 on success, -1 otherwise.
9514  */
9515 static int
9516 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
9517                                     struct mlx5_meter_domain_info *dtb,
9518                                     uint8_t mtr_reg_c)
9519 {
9520         struct mlx5_flow_dv_match_params matcher = {
9521                 .size = sizeof(matcher.buf),
9522         };
9523         struct mlx5_flow_dv_match_params value = {
9524                 .size = sizeof(value.buf),
9525         };
9526         struct mlx5_meter_domains_infos *mtb = fm->mfts;
9527         void *actions[METER_ACTIONS];
9528         int i;
9529
9530         /* Create jump action. */
9531         if (!dtb->jump_actn)
9532                 dtb->jump_actn =
9533                         mlx5_glue->dr_create_flow_action_dest_flow_tbl
9534                                                         (dtb->sfx_tbl->obj);
9535         if (!dtb->jump_actn) {
9536                 DRV_LOG(ERR, "Failed to create policer jump action.");
9537                 goto error;
9538         }
9539         for (i = 0; i < RTE_MTR_DROPPED; i++) {
9540                 int j = 0;
9541
9542                 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
9543                                        rte_col_2_mlx5_col(i), UINT8_MAX);
9544                 if (mtb->count_actns[i])
9545                         actions[j++] = mtb->count_actns[i];
9546                 if (fm->action[i] == MTR_POLICER_ACTION_DROP)
9547                         actions[j++] = mtb->drop_actn;
9548                 else
9549                         actions[j++] = dtb->jump_actn;
9550                 dtb->policer_rules[i] =
9551                         mlx5_glue->dv_create_flow(dtb->color_matcher,
9552                                                  (void *)&value,
9553                                                   j, actions);
9554                 if (!dtb->policer_rules[i]) {
9555                         DRV_LOG(ERR, "Failed to create policer rule.");
9556                         goto error;
9557                 }
9558         }
9559         return 0;
9560 error:
9561         rte_errno = errno;
9562         return -1;
9563 }
9564
9565 /**
9566  * Create policer rules.
9567  *
9568  * @param[in] dev
9569  *   Pointer to Ethernet device.
9570  * @param[in] fm
9571  *   Pointer to flow meter structure.
9572  * @param[in] attr
9573  *   Pointer to flow attributes.
9574  *
9575  * @return
9576  *   0 on success, -1 otherwise.
9577  */
9578 static int
9579 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
9580                              struct mlx5_flow_meter *fm,
9581                              const struct rte_flow_attr *attr)
9582 {
9583         struct mlx5_priv *priv = dev->data->dev_private;
9584         struct mlx5_meter_domains_infos *mtb = fm->mfts;
9585         int ret;
9586
9587         if (attr->egress) {
9588                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
9589                                                 priv->mtr_color_reg);
9590                 if (ret) {
9591                         DRV_LOG(ERR, "Failed to create egress policer.");
9592                         goto error;
9593                 }
9594         }
9595         if (attr->ingress) {
9596                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
9597                                                 priv->mtr_color_reg);
9598                 if (ret) {
9599                         DRV_LOG(ERR, "Failed to create ingress policer.");
9600                         goto error;
9601                 }
9602         }
9603         if (attr->transfer) {
9604                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
9605                                                 priv->mtr_color_reg);
9606                 if (ret) {
9607                         DRV_LOG(ERR, "Failed to create transfer policer.");
9608                         goto error;
9609                 }
9610         }
9611         return 0;
9612 error:
9613         flow_dv_destroy_policer_rules(dev, fm, attr);
9614         return -1;
9615 }
9616
9617 /**
9618  * Query a devx counter.
9619  *
9620  * @param[in] dev
9621  *   Pointer to the Ethernet device structure.
9622  * @param[in] cnt
9623  *   Index to the flow counter.
9624  * @param[in] clear
9625  *   Set to clear the counter statistics.
9626  * @param[out] pkts
9627  *   The statistics value of packets.
9628  * @param[out] bytes
9629  *   The statistics value of bytes.
9630  *
9631  * @return
9632  *   0 on success, otherwise return -1.
9633  */
9634 static int
9635 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
9636                       uint64_t *pkts, uint64_t *bytes)
9637 {
9638         struct mlx5_priv *priv = dev->data->dev_private;
9639         struct mlx5_flow_counter *cnt;
9640         uint64_t inn_pkts, inn_bytes;
9641         int ret;
9642
9643         if (!priv->config.devx)
9644                 return -1;
9645
9646         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
9647         if (ret)
9648                 return -1;
9649         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
9650         *pkts = inn_pkts - cnt->hits;
9651         *bytes = inn_bytes - cnt->bytes;
9652         if (clear) {
9653                 cnt->hits = inn_pkts;
9654                 cnt->bytes = inn_bytes;
9655         }
9656         return 0;
9657 }
9658
9659 /**
9660  * Get aged-out flows.
9661  *
9662  * @param[in] dev
9663  *   Pointer to the Ethernet device structure.
9664  * @param[in] context
9665  *   The address of an array of pointers to the aged-out flows contexts.
9666  * @param[in] nb_contexts
9667  *   The length of context array pointers.
9668  * @param[out] error
9669  *   Perform verbose error reporting if not NULL. Initialized in case of
9670  *   error only.
9671  *
9672  * @return
9673  *   how many contexts get in success, otherwise negative errno value.
9674  *   if nb_contexts is 0, return the amount of all aged contexts.
9675  *   if nb_contexts is not 0 , return the amount of aged flows reported
9676  *   in the context array.
9677  * @note: only stub for now
9678  */
9679 static int
9680 flow_get_aged_flows(struct rte_eth_dev *dev,
9681                     void **context,
9682                     uint32_t nb_contexts,
9683                     struct rte_flow_error *error)
9684 {
9685         struct mlx5_priv *priv = dev->data->dev_private;
9686         struct mlx5_age_info *age_info;
9687         struct mlx5_age_param *age_param;
9688         struct mlx5_flow_counter *counter;
9689         int nb_flows = 0;
9690
9691         if (nb_contexts && !context)
9692                 return rte_flow_error_set(error, EINVAL,
9693                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9694                                           NULL,
9695                                           "Should assign at least one flow or"
9696                                           " context to get if nb_contexts != 0");
9697         age_info = GET_PORT_AGE_INFO(priv);
9698         rte_spinlock_lock(&age_info->aged_sl);
9699         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
9700                 nb_flows++;
9701                 if (nb_contexts) {
9702                         age_param = MLX5_CNT_TO_AGE(counter);
9703                         context[nb_flows - 1] = age_param->context;
9704                         if (!(--nb_contexts))
9705                                 break;
9706                 }
9707         }
9708         rte_spinlock_unlock(&age_info->aged_sl);
9709         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
9710         return nb_flows;
9711 }
9712
9713 /*
9714  * Mutex-protected thunk to lock-free  __flow_dv_translate().
9715  */
9716 static int
9717 flow_dv_translate(struct rte_eth_dev *dev,
9718                   struct mlx5_flow *dev_flow,
9719                   const struct rte_flow_attr *attr,
9720                   const struct rte_flow_item items[],
9721                   const struct rte_flow_action actions[],
9722                   struct rte_flow_error *error)
9723 {
9724         int ret;
9725
9726         flow_dv_shared_lock(dev);
9727         ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
9728         flow_dv_shared_unlock(dev);
9729         return ret;
9730 }
9731
9732 /*
9733  * Mutex-protected thunk to lock-free  __flow_dv_apply().
9734  */
9735 static int
9736 flow_dv_apply(struct rte_eth_dev *dev,
9737               struct rte_flow *flow,
9738               struct rte_flow_error *error)
9739 {
9740         int ret;
9741
9742         flow_dv_shared_lock(dev);
9743         ret = __flow_dv_apply(dev, flow, error);
9744         flow_dv_shared_unlock(dev);
9745         return ret;
9746 }
9747
9748 /*
9749  * Mutex-protected thunk to lock-free __flow_dv_remove().
9750  */
9751 static void
9752 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
9753 {
9754         flow_dv_shared_lock(dev);
9755         __flow_dv_remove(dev, flow);
9756         flow_dv_shared_unlock(dev);
9757 }
9758
9759 /*
9760  * Mutex-protected thunk to lock-free __flow_dv_destroy().
9761  */
9762 static void
9763 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
9764 {
9765         flow_dv_shared_lock(dev);
9766         __flow_dv_destroy(dev, flow);
9767         flow_dv_shared_unlock(dev);
9768 }
9769
9770 /*
9771  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
9772  */
9773 static uint32_t
9774 flow_dv_counter_allocate(struct rte_eth_dev *dev)
9775 {
9776         uint32_t cnt;
9777
9778         flow_dv_shared_lock(dev);
9779         cnt = flow_dv_counter_alloc(dev, 0, 0, 1, 0);
9780         flow_dv_shared_unlock(dev);
9781         return cnt;
9782 }
9783
9784 /*
9785  * Mutex-protected thunk to lock-free flow_dv_counter_release().
9786  */
9787 static void
9788 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
9789 {
9790         flow_dv_shared_lock(dev);
9791         flow_dv_counter_release(dev, cnt);
9792         flow_dv_shared_unlock(dev);
9793 }
9794
9795 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
9796         .validate = flow_dv_validate,
9797         .prepare = flow_dv_prepare,
9798         .translate = flow_dv_translate,
9799         .apply = flow_dv_apply,
9800         .remove = flow_dv_remove,
9801         .destroy = flow_dv_destroy,
9802         .query = flow_dv_query,
9803         .create_mtr_tbls = flow_dv_create_mtr_tbl,
9804         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
9805         .create_policer_rules = flow_dv_create_policer_rules,
9806         .destroy_policer_rules = flow_dv_destroy_policer_rules,
9807         .counter_alloc = flow_dv_counter_allocate,
9808         .counter_free = flow_dv_counter_free,
9809         .counter_query = flow_dv_counter_query,
9810         .get_aged_flows = flow_get_aged_flows,
9811 };
9812
9813 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */