drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_cycles.h>
  28 #include <rte_ip.h>
  29 #include <rte_gre.h>
  30 #include <rte_vxlan.h>
  31 #include <rte_gtp.h>
  32
  33 #include <mlx5_devx_cmds.h>
  34 #include <mlx5_prm.h>
  35 #include <mlx5_malloc.h>
  36
  37 #include "mlx5_defs.h"
  38 #include "mlx5.h"
  39 #include "mlx5_common_os.h"
  40 #include "mlx5_flow.h"
  41 #include "mlx5_flow_os.h"
  42 #include "mlx5_rxtx.h"
  43
  44 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  45
  46 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  47 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  48 #endif
  49
  50 #ifndef HAVE_MLX5DV_DR_ESWITCH
  51 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  52 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  53 #endif
  54 #endif
  55
  56 #ifndef HAVE_MLX5DV_DR
  57 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  58 #endif
  59
  60 /* VLAN header definitions */
  61 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  62 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  63 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  64 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  65 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  66
  67 union flow_dv_attr {
  68         struct {
  69                 uint32_t valid:1;
  70                 uint32_t ipv4:1;
  71                 uint32_t ipv6:1;
  72                 uint32_t tcp:1;
  73                 uint32_t udp:1;
  74                 uint32_t reserved:27;
  75         };
  76         uint32_t attr;
  77 };
  78
  79 static int
  80 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
  81                              struct mlx5_flow_tbl_resource *tbl);
  82
  83 static int
  84 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev);
  85
  86 /**
  87  * Initialize flow attributes structure according to flow items' types.
  88  *
  89  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  90  * mode. For tunnel mode, the items to be modified are the outermost ones.
  91  *
  92  * @param[in] item
  93  *   Pointer to item specification.
  94  * @param[out] attr
  95  *   Pointer to flow attributes structure.
  96  * @param[in] dev_flow
  97  *   Pointer to the sub flow.
  98  * @param[in] tunnel_decap
  99  *   Whether action is after tunnel decapsulation.
 100  */
 101 static void
 102 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 103                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 104 {
 105         uint64_t layers = dev_flow->handle->layers;
 106
 107         /*
 108          * If layers is already initialized, it means this dev_flow is the
 109          * suffix flow, the layers flags is set by the prefix flow. Need to
 110          * use the layer flags from prefix flow as the suffix flow may not
 111          * have the user defined items as the flow is split.
 112          */
 113         if (layers) {
 114                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 115                         attr->ipv4 = 1;
 116                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 117                         attr->ipv6 = 1;
 118                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 119                         attr->tcp = 1;
 120                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 121                         attr->udp = 1;
 122                 attr->valid = 1;
 123                 return;
 124         }
 125         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 126                 uint8_t next_protocol = 0xff;
 127                 switch (item->type) {
 128                 case RTE_FLOW_ITEM_TYPE_GRE:
 129                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 130                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 131                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 132                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 133                 case RTE_FLOW_ITEM_TYPE_MPLS:
 134                         if (tunnel_decap)
 135                                 attr->attr = 0;
 136                         break;
 137                 case RTE_FLOW_ITEM_TYPE_IPV4:
 138                         if (!attr->ipv6)
 139                                 attr->ipv4 = 1;
 140                         if (item->mask != NULL &&
 141                             ((const struct rte_flow_item_ipv4 *)
 142                             item->mask)->hdr.next_proto_id)
 143                                 next_protocol =
 144                                     ((const struct rte_flow_item_ipv4 *)
 145                                       (item->spec))->hdr.next_proto_id &
 146                                     ((const struct rte_flow_item_ipv4 *)
 147                                       (item->mask))->hdr.next_proto_id;
 148                         if ((next_protocol == IPPROTO_IPIP ||
 149                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 150                                 attr->attr = 0;
 151                         break;
 152                 case RTE_FLOW_ITEM_TYPE_IPV6:
 153                         if (!attr->ipv4)
 154                                 attr->ipv6 = 1;
 155                         if (item->mask != NULL &&
 156                             ((const struct rte_flow_item_ipv6 *)
 157                             item->mask)->hdr.proto)
 158                                 next_protocol =
 159                                     ((const struct rte_flow_item_ipv6 *)
 160                                       (item->spec))->hdr.proto &
 161                                     ((const struct rte_flow_item_ipv6 *)
 162                                       (item->mask))->hdr.proto;
 163                         if ((next_protocol == IPPROTO_IPIP ||
 164                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 165                                 attr->attr = 0;
 166                         break;
 167                 case RTE_FLOW_ITEM_TYPE_UDP:
 168                         if (!attr->tcp)
 169                                 attr->udp = 1;
 170                         break;
 171                 case RTE_FLOW_ITEM_TYPE_TCP:
 172                         if (!attr->udp)
 173                                 attr->tcp = 1;
 174                         break;
 175                 default:
 176                         break;
 177                 }
 178         }
 179         attr->valid = 1;
 180 }
 181
 182 /**
 183  * Convert rte_mtr_color to mlx5 color.
 184  *
 185  * @param[in] rcol
 186  *   rte_mtr_color.
 187  *
 188  * @return
 189  *   mlx5 color.
 190  */
 191 static int
 192 rte_col_2_mlx5_col(enum rte_color rcol)
 193 {
 194         switch (rcol) {
 195         case RTE_COLOR_GREEN:
 196                 return MLX5_FLOW_COLOR_GREEN;
 197         case RTE_COLOR_YELLOW:
 198                 return MLX5_FLOW_COLOR_YELLOW;
 199         case RTE_COLOR_RED:
 200                 return MLX5_FLOW_COLOR_RED;
 201         default:
 202                 break;
 203         }
 204         return MLX5_FLOW_COLOR_UNDEFINED;
 205 }
 206
 207 struct field_modify_info {
 208         uint32_t size; /* Size of field in protocol header, in bytes. */
 209         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 210         enum mlx5_modification_field id;
 211 };
 212
 213 struct field_modify_info modify_eth[] = {
 214         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 215         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 216         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 217         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 218         {0, 0, 0},
 219 };
 220
 221 struct field_modify_info modify_vlan_out_first_vid[] = {
 222         /* Size in bits !!! */
 223         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 224         {0, 0, 0},
 225 };
 226
 227 struct field_modify_info modify_ipv4[] = {
 228         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 229         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 230         {4, 12, MLX5_MODI_OUT_SIPV4},
 231         {4, 16, MLX5_MODI_OUT_DIPV4},
 232         {0, 0, 0},
 233 };
 234
 235 struct field_modify_info modify_ipv6[] = {
 236         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 237         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 238         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 239         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 240         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 241         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 242         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 243         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 244         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 245         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 246         {0, 0, 0},
 247 };
 248
 249 struct field_modify_info modify_udp[] = {
 250         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 251         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 252         {0, 0, 0},
 253 };
 254
 255 struct field_modify_info modify_tcp[] = {
 256         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 257         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 258         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 259         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 260         {0, 0, 0},
 261 };
 262
 263 static void
 264 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 265                           uint8_t next_protocol, uint64_t *item_flags,
 266                           int *tunnel)
 267 {
 268         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 269                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 270         if (next_protocol == IPPROTO_IPIP) {
 271                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 272                 *tunnel = 1;
 273         }
 274         if (next_protocol == IPPROTO_IPV6) {
 275                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 276                 *tunnel = 1;
 277         }
 278 }
 279
 280 /**
 281  * Acquire the synchronizing object to protect multithreaded access
 282  * to shared dv context. Lock occurs only if context is actually
 283  * shared, i.e. we have multiport IB device and representors are
 284  * created.
 285  *
 286  * @param[in] dev
 287  *   Pointer to the rte_eth_dev structure.
 288  */
 289 static void
 290 flow_dv_shared_lock(struct rte_eth_dev *dev)
 291 {
 292         struct mlx5_priv *priv = dev->data->dev_private;
 293         struct mlx5_dev_ctx_shared *sh = priv->sh;
 294
 295         if (sh->dv_refcnt > 1) {
 296                 int ret;
 297
 298                 ret = pthread_mutex_lock(&sh->dv_mutex);
 299                 MLX5_ASSERT(!ret);
 300                 (void)ret;
 301         }
 302 }
 303
 304 static void
 305 flow_dv_shared_unlock(struct rte_eth_dev *dev)
 306 {
 307         struct mlx5_priv *priv = dev->data->dev_private;
 308         struct mlx5_dev_ctx_shared *sh = priv->sh;
 309
 310         if (sh->dv_refcnt > 1) {
 311                 int ret;
 312
 313                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 314                 MLX5_ASSERT(!ret);
 315                 (void)ret;
 316         }
 317 }
 318
 319 /* Update VLAN's VID/PCP based on input rte_flow_action.
 320  *
 321  * @param[in] action
 322  *   Pointer to struct rte_flow_action.
 323  * @param[out] vlan
 324  *   Pointer to struct rte_vlan_hdr.
 325  */
 326 static void
 327 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 328                          struct rte_vlan_hdr *vlan)
 329 {
 330         uint16_t vlan_tci;
 331         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 332                 vlan_tci =
 333                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 334                                                action->conf)->vlan_pcp;
 335                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 336                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 337                 vlan->vlan_tci |= vlan_tci;
 338         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 339                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 340                 vlan->vlan_tci |= rte_be_to_cpu_16
 341                     (((const struct rte_flow_action_of_set_vlan_vid *)
 342                                              action->conf)->vlan_vid);
 343         }
 344 }
 345
 346 /**
 347  * Fetch 1, 2, 3 or 4 byte field from the byte array
 348  * and return as unsigned integer in host-endian format.
 349  *
 350  * @param[in] data
 351  *   Pointer to data array.
 352  * @param[in] size
 353  *   Size of field to extract.
 354  *
 355  * @return
 356  *   converted field in host endian format.
 357  */
 358 static inline uint32_t
 359 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 360 {
 361         uint32_t ret;
 362
 363         switch (size) {
 364         case 1:
 365                 ret = *data;
 366                 break;
 367         case 2:
 368                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 369                 break;
 370         case 3:
 371                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 372                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 373                 break;
 374         case 4:
 375                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 376                 break;
 377         default:
 378                 MLX5_ASSERT(false);
 379                 ret = 0;
 380                 break;
 381         }
 382         return ret;
 383 }
 384
 385 /**
 386  * Convert modify-header action to DV specification.
 387  *
 388  * Data length of each action is determined by provided field description
 389  * and the item mask. Data bit offset and width of each action is determined
 390  * by provided item mask.
 391  *
 392  * @param[in] item
 393  *   Pointer to item specification.
 394  * @param[in] field
 395  *   Pointer to field modification information.
 396  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 397  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 398  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 399  * @param[in] dcopy
 400  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 401  *   Negative offset value sets the same offset as source offset.
 402  *   size field is ignored, value is taken from source field.
 403  * @param[in,out] resource
 404  *   Pointer to the modify-header resource.
 405  * @param[in] type
 406  *   Type of modification.
 407  * @param[out] error
 408  *   Pointer to the error structure.
 409  *
 410  * @return
 411  *   0 on success, a negative errno value otherwise and rte_errno is set.
 412  */
 413 static int
 414 flow_dv_convert_modify_action(struct rte_flow_item *item,
 415                               struct field_modify_info *field,
 416                               struct field_modify_info *dcopy,
 417                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 418                               uint32_t type, struct rte_flow_error *error)
 419 {
 420         uint32_t i = resource->actions_num;
 421         struct mlx5_modification_cmd *actions = resource->actions;
 422
 423         /*
 424          * The item and mask are provided in big-endian format.
 425          * The fields should be presented as in big-endian format either.
 426          * Mask must be always present, it defines the actual field width.
 427          */
 428         MLX5_ASSERT(item->mask);
 429         MLX5_ASSERT(field->size);
 430         do {
 431                 unsigned int size_b;
 432                 unsigned int off_b;
 433                 uint32_t mask;
 434                 uint32_t data;
 435
 436                 if (i >= MLX5_MAX_MODIFY_NUM)
 437                         return rte_flow_error_set(error, EINVAL,
 438                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 439                                  "too many items to modify");
 440                 /* Fetch variable byte size mask from the array. */
 441                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 442                                            field->offset, field->size);
 443                 if (!mask) {
 444                         ++field;
 445                         continue;
 446                 }
 447                 /* Deduce actual data width in bits from mask value. */
 448                 off_b = rte_bsf32(mask);
 449                 size_b = sizeof(uint32_t) * CHAR_BIT -
 450                          off_b - __builtin_clz(mask);
 451                 MLX5_ASSERT(size_b);
 452                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 453                 actions[i] = (struct mlx5_modification_cmd) {
 454                         .action_type = type,
 455                         .field = field->id,
 456                         .offset = off_b,
 457                         .length = size_b,
 458                 };
 459                 /* Convert entire record to expected big-endian format. */
 460                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 461                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 462                         MLX5_ASSERT(dcopy);
 463                         actions[i].dst_field = dcopy->id;
 464                         actions[i].dst_offset =
 465                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 466                         /* Convert entire record to big-endian format. */
 467                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 468                 } else {
 469                         MLX5_ASSERT(item->spec);
 470                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 471                                                    field->offset, field->size);
 472                         /* Shift out the trailing masked bits from data. */
 473                         data = (data & mask) >> off_b;
 474                         actions[i].data1 = rte_cpu_to_be_32(data);
 475                 }
 476                 ++i;
 477                 ++field;
 478         } while (field->size);
 479         if (resource->actions_num == i)
 480                 return rte_flow_error_set(error, EINVAL,
 481                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 482                                           "invalid modification flow item");
 483         resource->actions_num = i;
 484         return 0;
 485 }
 486
 487 /**
 488  * Convert modify-header set IPv4 address action to DV specification.
 489  *
 490  * @param[in,out] resource
 491  *   Pointer to the modify-header resource.
 492  * @param[in] action
 493  *   Pointer to action specification.
 494  * @param[out] error
 495  *   Pointer to the error structure.
 496  *
 497  * @return
 498  *   0 on success, a negative errno value otherwise and rte_errno is set.
 499  */
 500 static int
 501 flow_dv_convert_action_modify_ipv4
 502                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 503                          const struct rte_flow_action *action,
 504                          struct rte_flow_error *error)
 505 {
 506         const struct rte_flow_action_set_ipv4 *conf =
 507                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 508         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 509         struct rte_flow_item_ipv4 ipv4;
 510         struct rte_flow_item_ipv4 ipv4_mask;
 511
 512         memset(&ipv4, 0, sizeof(ipv4));
 513         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 514         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 515                 ipv4.hdr.src_addr = conf->ipv4_addr;
 516                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 517         } else {
 518                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 519                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 520         }
 521         item.spec = &ipv4;
 522         item.mask = &ipv4_mask;
 523         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 524                                              MLX5_MODIFICATION_TYPE_SET, error);
 525 }
 526
 527 /**
 528  * Convert modify-header set IPv6 address action to DV specification.
 529  *
 530  * @param[in,out] resource
 531  *   Pointer to the modify-header resource.
 532  * @param[in] action
 533  *   Pointer to action specification.
 534  * @param[out] error
 535  *   Pointer to the error structure.
 536  *
 537  * @return
 538  *   0 on success, a negative errno value otherwise and rte_errno is set.
 539  */
 540 static int
 541 flow_dv_convert_action_modify_ipv6
 542                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 543                          const struct rte_flow_action *action,
 544                          struct rte_flow_error *error)
 545 {
 546         const struct rte_flow_action_set_ipv6 *conf =
 547                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 548         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 549         struct rte_flow_item_ipv6 ipv6;
 550         struct rte_flow_item_ipv6 ipv6_mask;
 551
 552         memset(&ipv6, 0, sizeof(ipv6));
 553         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 554         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 555                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 556                        sizeof(ipv6.hdr.src_addr));
 557                 memcpy(&ipv6_mask.hdr.src_addr,
 558                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 559                        sizeof(ipv6.hdr.src_addr));
 560         } else {
 561                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 562                        sizeof(ipv6.hdr.dst_addr));
 563                 memcpy(&ipv6_mask.hdr.dst_addr,
 564                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 565                        sizeof(ipv6.hdr.dst_addr));
 566         }
 567         item.spec = &ipv6;
 568         item.mask = &ipv6_mask;
 569         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 570                                              MLX5_MODIFICATION_TYPE_SET, error);
 571 }
 572
 573 /**
 574  * Convert modify-header set MAC address action to DV specification.
 575  *
 576  * @param[in,out] resource
 577  *   Pointer to the modify-header resource.
 578  * @param[in] action
 579  *   Pointer to action specification.
 580  * @param[out] error
 581  *   Pointer to the error structure.
 582  *
 583  * @return
 584  *   0 on success, a negative errno value otherwise and rte_errno is set.
 585  */
 586 static int
 587 flow_dv_convert_action_modify_mac
 588                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 589                          const struct rte_flow_action *action,
 590                          struct rte_flow_error *error)
 591 {
 592         const struct rte_flow_action_set_mac *conf =
 593                 (const struct rte_flow_action_set_mac *)(action->conf);
 594         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 595         struct rte_flow_item_eth eth;
 596         struct rte_flow_item_eth eth_mask;
 597
 598         memset(&eth, 0, sizeof(eth));
 599         memset(&eth_mask, 0, sizeof(eth_mask));
 600         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 601                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 602                        sizeof(eth.src.addr_bytes));
 603                 memcpy(&eth_mask.src.addr_bytes,
 604                        &rte_flow_item_eth_mask.src.addr_bytes,
 605                        sizeof(eth_mask.src.addr_bytes));
 606         } else {
 607                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 608                        sizeof(eth.dst.addr_bytes));
 609                 memcpy(&eth_mask.dst.addr_bytes,
 610                        &rte_flow_item_eth_mask.dst.addr_bytes,
 611                        sizeof(eth_mask.dst.addr_bytes));
 612         }
 613         item.spec = &eth;
 614         item.mask = &eth_mask;
 615         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 616                                              MLX5_MODIFICATION_TYPE_SET, error);
 617 }
 618
 619 /**
 620  * Convert modify-header set VLAN VID action to DV specification.
 621  *
 622  * @param[in,out] resource
 623  *   Pointer to the modify-header resource.
 624  * @param[in] action
 625  *   Pointer to action specification.
 626  * @param[out] error
 627  *   Pointer to the error structure.
 628  *
 629  * @return
 630  *   0 on success, a negative errno value otherwise and rte_errno is set.
 631  */
 632 static int
 633 flow_dv_convert_action_modify_vlan_vid
 634                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 635                          const struct rte_flow_action *action,
 636                          struct rte_flow_error *error)
 637 {
 638         const struct rte_flow_action_of_set_vlan_vid *conf =
 639                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 640         int i = resource->actions_num;
 641         struct mlx5_modification_cmd *actions = resource->actions;
 642         struct field_modify_info *field = modify_vlan_out_first_vid;
 643
 644         if (i >= MLX5_MAX_MODIFY_NUM)
 645                 return rte_flow_error_set(error, EINVAL,
 646                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 647                          "too many items to modify");
 648         actions[i] = (struct mlx5_modification_cmd) {
 649                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 650                 .field = field->id,
 651                 .length = field->size,
 652                 .offset = field->offset,
 653         };
 654         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 655         actions[i].data1 = conf->vlan_vid;
 656         actions[i].data1 = actions[i].data1 << 16;
 657         resource->actions_num = ++i;
 658         return 0;
 659 }
 660
 661 /**
 662  * Convert modify-header set TP action to DV specification.
 663  *
 664  * @param[in,out] resource
 665  *   Pointer to the modify-header resource.
 666  * @param[in] action
 667  *   Pointer to action specification.
 668  * @param[in] items
 669  *   Pointer to rte_flow_item objects list.
 670  * @param[in] attr
 671  *   Pointer to flow attributes structure.
 672  * @param[in] dev_flow
 673  *   Pointer to the sub flow.
 674  * @param[in] tunnel_decap
 675  *   Whether action is after tunnel decapsulation.
 676  * @param[out] error
 677  *   Pointer to the error structure.
 678  *
 679  * @return
 680  *   0 on success, a negative errno value otherwise and rte_errno is set.
 681  */
 682 static int
 683 flow_dv_convert_action_modify_tp
 684                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 685                          const struct rte_flow_action *action,
 686                          const struct rte_flow_item *items,
 687                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 688                          bool tunnel_decap, struct rte_flow_error *error)
 689 {
 690         const struct rte_flow_action_set_tp *conf =
 691                 (const struct rte_flow_action_set_tp *)(action->conf);
 692         struct rte_flow_item item;
 693         struct rte_flow_item_udp udp;
 694         struct rte_flow_item_udp udp_mask;
 695         struct rte_flow_item_tcp tcp;
 696         struct rte_flow_item_tcp tcp_mask;
 697         struct field_modify_info *field;
 698
 699         if (!attr->valid)
 700                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 701         if (attr->udp) {
 702                 memset(&udp, 0, sizeof(udp));
 703                 memset(&udp_mask, 0, sizeof(udp_mask));
 704                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 705                         udp.hdr.src_port = conf->port;
 706                         udp_mask.hdr.src_port =
 707                                         rte_flow_item_udp_mask.hdr.src_port;
 708                 } else {
 709                         udp.hdr.dst_port = conf->port;
 710                         udp_mask.hdr.dst_port =
 711                                         rte_flow_item_udp_mask.hdr.dst_port;
 712                 }
 713                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 714                 item.spec = &udp;
 715                 item.mask = &udp_mask;
 716                 field = modify_udp;
 717         } else {
 718                 MLX5_ASSERT(attr->tcp);
 719                 memset(&tcp, 0, sizeof(tcp));
 720                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 721                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 722                         tcp.hdr.src_port = conf->port;
 723                         tcp_mask.hdr.src_port =
 724                                         rte_flow_item_tcp_mask.hdr.src_port;
 725                 } else {
 726                         tcp.hdr.dst_port = conf->port;
 727                         tcp_mask.hdr.dst_port =
 728                                         rte_flow_item_tcp_mask.hdr.dst_port;
 729                 }
 730                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 731                 item.spec = &tcp;
 732                 item.mask = &tcp_mask;
 733                 field = modify_tcp;
 734         }
 735         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 736                                              MLX5_MODIFICATION_TYPE_SET, error);
 737 }
 738
 739 /**
 740  * Convert modify-header set TTL action to DV specification.
 741  *
 742  * @param[in,out] resource
 743  *   Pointer to the modify-header resource.
 744  * @param[in] action
 745  *   Pointer to action specification.
 746  * @param[in] items
 747  *   Pointer to rte_flow_item objects list.
 748  * @param[in] attr
 749  *   Pointer to flow attributes structure.
 750  * @param[in] dev_flow
 751  *   Pointer to the sub flow.
 752  * @param[in] tunnel_decap
 753  *   Whether action is after tunnel decapsulation.
 754  * @param[out] error
 755  *   Pointer to the error structure.
 756  *
 757  * @return
 758  *   0 on success, a negative errno value otherwise and rte_errno is set.
 759  */
 760 static int
 761 flow_dv_convert_action_modify_ttl
 762                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 763                          const struct rte_flow_action *action,
 764                          const struct rte_flow_item *items,
 765                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 766                          bool tunnel_decap, struct rte_flow_error *error)
 767 {
 768         const struct rte_flow_action_set_ttl *conf =
 769                 (const struct rte_flow_action_set_ttl *)(action->conf);
 770         struct rte_flow_item item;
 771         struct rte_flow_item_ipv4 ipv4;
 772         struct rte_flow_item_ipv4 ipv4_mask;
 773         struct rte_flow_item_ipv6 ipv6;
 774         struct rte_flow_item_ipv6 ipv6_mask;
 775         struct field_modify_info *field;
 776
 777         if (!attr->valid)
 778                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 779         if (attr->ipv4) {
 780                 memset(&ipv4, 0, sizeof(ipv4));
 781                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 782                 ipv4.hdr.time_to_live = conf->ttl_value;
 783                 ipv4_mask.hdr.time_to_live = 0xFF;
 784                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 785                 item.spec = &ipv4;
 786                 item.mask = &ipv4_mask;
 787                 field = modify_ipv4;
 788         } else {
 789                 MLX5_ASSERT(attr->ipv6);
 790                 memset(&ipv6, 0, sizeof(ipv6));
 791                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 792                 ipv6.hdr.hop_limits = conf->ttl_value;
 793                 ipv6_mask.hdr.hop_limits = 0xFF;
 794                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 795                 item.spec = &ipv6;
 796                 item.mask = &ipv6_mask;
 797                 field = modify_ipv6;
 798         }
 799         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 800                                              MLX5_MODIFICATION_TYPE_SET, error);
 801 }
 802
 803 /**
 804  * Convert modify-header decrement TTL action to DV specification.
 805  *
 806  * @param[in,out] resource
 807  *   Pointer to the modify-header resource.
 808  * @param[in] action
 809  *   Pointer to action specification.
 810  * @param[in] items
 811  *   Pointer to rte_flow_item objects list.
 812  * @param[in] attr
 813  *   Pointer to flow attributes structure.
 814  * @param[in] dev_flow
 815  *   Pointer to the sub flow.
 816  * @param[in] tunnel_decap
 817  *   Whether action is after tunnel decapsulation.
 818  * @param[out] error
 819  *   Pointer to the error structure.
 820  *
 821  * @return
 822  *   0 on success, a negative errno value otherwise and rte_errno is set.
 823  */
 824 static int
 825 flow_dv_convert_action_modify_dec_ttl
 826                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 827                          const struct rte_flow_item *items,
 828                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 829                          bool tunnel_decap, struct rte_flow_error *error)
 830 {
 831         struct rte_flow_item item;
 832         struct rte_flow_item_ipv4 ipv4;
 833         struct rte_flow_item_ipv4 ipv4_mask;
 834         struct rte_flow_item_ipv6 ipv6;
 835         struct rte_flow_item_ipv6 ipv6_mask;
 836         struct field_modify_info *field;
 837
 838         if (!attr->valid)
 839                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 840         if (attr->ipv4) {
 841                 memset(&ipv4, 0, sizeof(ipv4));
 842                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 843                 ipv4.hdr.time_to_live = 0xFF;
 844                 ipv4_mask.hdr.time_to_live = 0xFF;
 845                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 846                 item.spec = &ipv4;
 847                 item.mask = &ipv4_mask;
 848                 field = modify_ipv4;
 849         } else {
 850                 MLX5_ASSERT(attr->ipv6);
 851                 memset(&ipv6, 0, sizeof(ipv6));
 852                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 853                 ipv6.hdr.hop_limits = 0xFF;
 854                 ipv6_mask.hdr.hop_limits = 0xFF;
 855                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 856                 item.spec = &ipv6;
 857                 item.mask = &ipv6_mask;
 858                 field = modify_ipv6;
 859         }
 860         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 861                                              MLX5_MODIFICATION_TYPE_ADD, error);
 862 }
 863
 864 /**
 865  * Convert modify-header increment/decrement TCP Sequence number
 866  * to DV specification.
 867  *
 868  * @param[in,out] resource
 869  *   Pointer to the modify-header resource.
 870  * @param[in] action
 871  *   Pointer to action specification.
 872  * @param[out] error
 873  *   Pointer to the error structure.
 874  *
 875  * @return
 876  *   0 on success, a negative errno value otherwise and rte_errno is set.
 877  */
 878 static int
 879 flow_dv_convert_action_modify_tcp_seq
 880                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 881                          const struct rte_flow_action *action,
 882                          struct rte_flow_error *error)
 883 {
 884         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 885         uint64_t value = rte_be_to_cpu_32(*conf);
 886         struct rte_flow_item item;
 887         struct rte_flow_item_tcp tcp;
 888         struct rte_flow_item_tcp tcp_mask;
 889
 890         memset(&tcp, 0, sizeof(tcp));
 891         memset(&tcp_mask, 0, sizeof(tcp_mask));
 892         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 893                 /*
 894                  * The HW has no decrement operation, only increment operation.
 895                  * To simulate decrement X from Y using increment operation
 896                  * we need to add UINT32_MAX X times to Y.
 897                  * Each adding of UINT32_MAX decrements Y by 1.
 898                  */
 899                 value *= UINT32_MAX;
 900         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 901         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 902         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 903         item.spec = &tcp;
 904         item.mask = &tcp_mask;
 905         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 906                                              MLX5_MODIFICATION_TYPE_ADD, error);
 907 }
 908
 909 /**
 910  * Convert modify-header increment/decrement TCP Acknowledgment number
 911  * to DV specification.
 912  *
 913  * @param[in,out] resource
 914  *   Pointer to the modify-header resource.
 915  * @param[in] action
 916  *   Pointer to action specification.
 917  * @param[out] error
 918  *   Pointer to the error structure.
 919  *
 920  * @return
 921  *   0 on success, a negative errno value otherwise and rte_errno is set.
 922  */
 923 static int
 924 flow_dv_convert_action_modify_tcp_ack
 925                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 926                          const struct rte_flow_action *action,
 927                          struct rte_flow_error *error)
 928 {
 929         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 930         uint64_t value = rte_be_to_cpu_32(*conf);
 931         struct rte_flow_item item;
 932         struct rte_flow_item_tcp tcp;
 933         struct rte_flow_item_tcp tcp_mask;
 934
 935         memset(&tcp, 0, sizeof(tcp));
 936         memset(&tcp_mask, 0, sizeof(tcp_mask));
 937         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 938                 /*
 939                  * The HW has no decrement operation, only increment operation.
 940                  * To simulate decrement X from Y using increment operation
 941                  * we need to add UINT32_MAX X times to Y.
 942                  * Each adding of UINT32_MAX decrements Y by 1.
 943                  */
 944                 value *= UINT32_MAX;
 945         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 946         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 947         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 948         item.spec = &tcp;
 949         item.mask = &tcp_mask;
 950         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 951                                              MLX5_MODIFICATION_TYPE_ADD, error);
 952 }
 953
 954 static enum mlx5_modification_field reg_to_field[] = {
 955         [REG_NONE] = MLX5_MODI_OUT_NONE,
 956         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 957         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 958         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 959         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 960         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 961         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 962         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 963         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 964         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 965         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 966 };
 967
 968 /**
 969  * Convert register set to DV specification.
 970  *
 971  * @param[in,out] resource
 972  *   Pointer to the modify-header resource.
 973  * @param[in] action
 974  *   Pointer to action specification.
 975  * @param[out] error
 976  *   Pointer to the error structure.
 977  *
 978  * @return
 979  *   0 on success, a negative errno value otherwise and rte_errno is set.
 980  */
 981 static int
 982 flow_dv_convert_action_set_reg
 983                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 984                          const struct rte_flow_action *action,
 985                          struct rte_flow_error *error)
 986 {
 987         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 988         struct mlx5_modification_cmd *actions = resource->actions;
 989         uint32_t i = resource->actions_num;
 990
 991         if (i >= MLX5_MAX_MODIFY_NUM)
 992                 return rte_flow_error_set(error, EINVAL,
 993                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 994                                           "too many items to modify");
 995         MLX5_ASSERT(conf->id != REG_NONE);
 996         MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field));
 997         actions[i] = (struct mlx5_modification_cmd) {
 998                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 999                 .field = reg_to_field[conf->id],
1000         };
1001         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1002         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1003         ++i;
1004         resource->actions_num = i;
1005         return 0;
1006 }
1007
1008 /**
1009  * Convert SET_TAG action to DV specification.
1010  *
1011  * @param[in] dev
1012  *   Pointer to the rte_eth_dev structure.
1013  * @param[in,out] resource
1014  *   Pointer to the modify-header resource.
1015  * @param[in] conf
1016  *   Pointer to action specification.
1017  * @param[out] error
1018  *   Pointer to the error structure.
1019  *
1020  * @return
1021  *   0 on success, a negative errno value otherwise and rte_errno is set.
1022  */
1023 static int
1024 flow_dv_convert_action_set_tag
1025                         (struct rte_eth_dev *dev,
1026                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1027                          const struct rte_flow_action_set_tag *conf,
1028                          struct rte_flow_error *error)
1029 {
1030         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1031         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1032         struct rte_flow_item item = {
1033                 .spec = &data,
1034                 .mask = &mask,
1035         };
1036         struct field_modify_info reg_c_x[] = {
1037                 [1] = {0, 0, 0},
1038         };
1039         enum mlx5_modification_field reg_type;
1040         int ret;
1041
1042         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1043         if (ret < 0)
1044                 return ret;
1045         MLX5_ASSERT(ret != REG_NONE);
1046         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1047         reg_type = reg_to_field[ret];
1048         MLX5_ASSERT(reg_type > 0);
1049         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1050         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1051                                              MLX5_MODIFICATION_TYPE_SET, error);
1052 }
1053
1054 /**
1055  * Convert internal COPY_REG action to DV specification.
1056  *
1057  * @param[in] dev
1058  *   Pointer to the rte_eth_dev structure.
1059  * @param[in,out] res
1060  *   Pointer to the modify-header resource.
1061  * @param[in] action
1062  *   Pointer to action specification.
1063  * @param[out] error
1064  *   Pointer to the error structure.
1065  *
1066  * @return
1067  *   0 on success, a negative errno value otherwise and rte_errno is set.
1068  */
1069 static int
1070 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1071                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1072                                  const struct rte_flow_action *action,
1073                                  struct rte_flow_error *error)
1074 {
1075         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1076         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1077         struct rte_flow_item item = {
1078                 .spec = NULL,
1079                 .mask = &mask,
1080         };
1081         struct field_modify_info reg_src[] = {
1082                 {4, 0, reg_to_field[conf->src]},
1083                 {0, 0, 0},
1084         };
1085         struct field_modify_info reg_dst = {
1086                 .offset = 0,
1087                 .id = reg_to_field[conf->dst],
1088         };
1089         /* Adjust reg_c[0] usage according to reported mask. */
1090         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1091                 struct mlx5_priv *priv = dev->data->dev_private;
1092                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1093
1094                 MLX5_ASSERT(reg_c0);
1095                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1096                 if (conf->dst == REG_C_0) {
1097                         /* Copy to reg_c[0], within mask only. */
1098                         reg_dst.offset = rte_bsf32(reg_c0);
1099                         /*
1100                          * Mask is ignoring the enianness, because
1101                          * there is no conversion in datapath.
1102                          */
1103 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1104                         /* Copy from destination lower bits to reg_c[0]. */
1105                         mask = reg_c0 >> reg_dst.offset;
1106 #else
1107                         /* Copy from destination upper bits to reg_c[0]. */
1108                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1109                                           rte_fls_u32(reg_c0));
1110 #endif
1111                 } else {
1112                         mask = rte_cpu_to_be_32(reg_c0);
1113 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1114                         /* Copy from reg_c[0] to destination lower bits. */
1115                         reg_dst.offset = 0;
1116 #else
1117                         /* Copy from reg_c[0] to destination upper bits. */
1118                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1119                                          (rte_fls_u32(reg_c0) -
1120                                           rte_bsf32(reg_c0));
1121 #endif
1122                 }
1123         }
1124         return flow_dv_convert_modify_action(&item,
1125                                              reg_src, &reg_dst, res,
1126                                              MLX5_MODIFICATION_TYPE_COPY,
1127                                              error);
1128 }
1129
1130 /**
1131  * Convert MARK action to DV specification. This routine is used
1132  * in extensive metadata only and requires metadata register to be
1133  * handled. In legacy mode hardware tag resource is engaged.
1134  *
1135  * @param[in] dev
1136  *   Pointer to the rte_eth_dev structure.
1137  * @param[in] conf
1138  *   Pointer to MARK action specification.
1139  * @param[in,out] resource
1140  *   Pointer to the modify-header resource.
1141  * @param[out] error
1142  *   Pointer to the error structure.
1143  *
1144  * @return
1145  *   0 on success, a negative errno value otherwise and rte_errno is set.
1146  */
1147 static int
1148 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1149                             const struct rte_flow_action_mark *conf,
1150                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1151                             struct rte_flow_error *error)
1152 {
1153         struct mlx5_priv *priv = dev->data->dev_private;
1154         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1155                                            priv->sh->dv_mark_mask);
1156         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1157         struct rte_flow_item item = {
1158                 .spec = &data,
1159                 .mask = &mask,
1160         };
1161         struct field_modify_info reg_c_x[] = {
1162                 {4, 0, 0}, /* dynamic instead of MLX5_MODI_META_REG_C_1. */
1163                 {0, 0, 0},
1164         };
1165         int reg;
1166
1167         if (!mask)
1168                 return rte_flow_error_set(error, EINVAL,
1169                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1170                                           NULL, "zero mark action mask");
1171         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1172         if (reg < 0)
1173                 return reg;
1174         MLX5_ASSERT(reg > 0);
1175         if (reg == REG_C_0) {
1176                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1177                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1178
1179                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1180                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1181                 mask = rte_cpu_to_be_32(mask << shl_c0);
1182         }
1183         reg_c_x[0].id = reg_to_field[reg];
1184         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1185                                              MLX5_MODIFICATION_TYPE_SET, error);
1186 }
1187
1188 /**
1189  * Get metadata register index for specified steering domain.
1190  *
1191  * @param[in] dev
1192  *   Pointer to the rte_eth_dev structure.
1193  * @param[in] attr
1194  *   Attributes of flow to determine steering domain.
1195  * @param[out] error
1196  *   Pointer to the error structure.
1197  *
1198  * @return
1199  *   positive index on success, a negative errno value otherwise
1200  *   and rte_errno is set.
1201  */
1202 static enum modify_reg
1203 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1204                          const struct rte_flow_attr *attr,
1205                          struct rte_flow_error *error)
1206 {
1207         int reg =
1208                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1209                                           MLX5_METADATA_FDB :
1210                                             attr->egress ?
1211                                             MLX5_METADATA_TX :
1212                                             MLX5_METADATA_RX, 0, error);
1213         if (reg < 0)
1214                 return rte_flow_error_set(error,
1215                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1216                                           NULL, "unavailable "
1217                                           "metadata register");
1218         return reg;
1219 }
1220
1221 /**
1222  * Convert SET_META action to DV specification.
1223  *
1224  * @param[in] dev
1225  *   Pointer to the rte_eth_dev structure.
1226  * @param[in,out] resource
1227  *   Pointer to the modify-header resource.
1228  * @param[in] attr
1229  *   Attributes of flow that includes this item.
1230  * @param[in] conf
1231  *   Pointer to action specification.
1232  * @param[out] error
1233  *   Pointer to the error structure.
1234  *
1235  * @return
1236  *   0 on success, a negative errno value otherwise and rte_errno is set.
1237  */
1238 static int
1239 flow_dv_convert_action_set_meta
1240                         (struct rte_eth_dev *dev,
1241                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1242                          const struct rte_flow_attr *attr,
1243                          const struct rte_flow_action_set_meta *conf,
1244                          struct rte_flow_error *error)
1245 {
1246         uint32_t data = conf->data;
1247         uint32_t mask = conf->mask;
1248         struct rte_flow_item item = {
1249                 .spec = &data,
1250                 .mask = &mask,
1251         };
1252         struct field_modify_info reg_c_x[] = {
1253                 [1] = {0, 0, 0},
1254         };
1255         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1256
1257         if (reg < 0)
1258                 return reg;
1259         /*
1260          * In datapath code there is no endianness
1261          * coversions for perfromance reasons, all
1262          * pattern conversions are done in rte_flow.
1263          */
1264         if (reg == REG_C_0) {
1265                 struct mlx5_priv *priv = dev->data->dev_private;
1266                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1267                 uint32_t shl_c0;
1268
1269                 MLX5_ASSERT(msk_c0);
1270 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1271                 shl_c0 = rte_bsf32(msk_c0);
1272 #else
1273                 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1274 #endif
1275                 mask <<= shl_c0;
1276                 data <<= shl_c0;
1277                 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1278         }
1279         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1280         /* The routine expects parameters in memory as big-endian ones. */
1281         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1282                                              MLX5_MODIFICATION_TYPE_SET, error);
1283 }
1284
1285 /**
1286  * Convert modify-header set IPv4 DSCP action to DV specification.
1287  *
1288  * @param[in,out] resource
1289  *   Pointer to the modify-header resource.
1290  * @param[in] action
1291  *   Pointer to action specification.
1292  * @param[out] error
1293  *   Pointer to the error structure.
1294  *
1295  * @return
1296  *   0 on success, a negative errno value otherwise and rte_errno is set.
1297  */
1298 static int
1299 flow_dv_convert_action_modify_ipv4_dscp
1300                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1301                          const struct rte_flow_action *action,
1302                          struct rte_flow_error *error)
1303 {
1304         const struct rte_flow_action_set_dscp *conf =
1305                 (const struct rte_flow_action_set_dscp *)(action->conf);
1306         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1307         struct rte_flow_item_ipv4 ipv4;
1308         struct rte_flow_item_ipv4 ipv4_mask;
1309
1310         memset(&ipv4, 0, sizeof(ipv4));
1311         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1312         ipv4.hdr.type_of_service = conf->dscp;
1313         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1314         item.spec = &ipv4;
1315         item.mask = &ipv4_mask;
1316         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1317                                              MLX5_MODIFICATION_TYPE_SET, error);
1318 }
1319
1320 /**
1321  * Convert modify-header set IPv6 DSCP action to DV specification.
1322  *
1323  * @param[in,out] resource
1324  *   Pointer to the modify-header resource.
1325  * @param[in] action
1326  *   Pointer to action specification.
1327  * @param[out] error
1328  *   Pointer to the error structure.
1329  *
1330  * @return
1331  *   0 on success, a negative errno value otherwise and rte_errno is set.
1332  */
1333 static int
1334 flow_dv_convert_action_modify_ipv6_dscp
1335                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1336                          const struct rte_flow_action *action,
1337                          struct rte_flow_error *error)
1338 {
1339         const struct rte_flow_action_set_dscp *conf =
1340                 (const struct rte_flow_action_set_dscp *)(action->conf);
1341         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1342         struct rte_flow_item_ipv6 ipv6;
1343         struct rte_flow_item_ipv6 ipv6_mask;
1344
1345         memset(&ipv6, 0, sizeof(ipv6));
1346         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1347         /*
1348          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1349          * rdma-core only accept the DSCP bits byte aligned start from
1350          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1351          * bits in IPv6 case as rdma-core requires byte aligned value.
1352          */
1353         ipv6.hdr.vtc_flow = conf->dscp;
1354         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1355         item.spec = &ipv6;
1356         item.mask = &ipv6_mask;
1357         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1358                                              MLX5_MODIFICATION_TYPE_SET, error);
1359 }
1360
1361 /**
1362  * Validate MARK item.
1363  *
1364  * @param[in] dev
1365  *   Pointer to the rte_eth_dev structure.
1366  * @param[in] item
1367  *   Item specification.
1368  * @param[in] attr
1369  *   Attributes of flow that includes this item.
1370  * @param[out] error
1371  *   Pointer to error structure.
1372  *
1373  * @return
1374  *   0 on success, a negative errno value otherwise and rte_errno is set.
1375  */
1376 static int
1377 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1378                            const struct rte_flow_item *item,
1379                            const struct rte_flow_attr *attr __rte_unused,
1380                            struct rte_flow_error *error)
1381 {
1382         struct mlx5_priv *priv = dev->data->dev_private;
1383         struct mlx5_dev_config *config = &priv->config;
1384         const struct rte_flow_item_mark *spec = item->spec;
1385         const struct rte_flow_item_mark *mask = item->mask;
1386         const struct rte_flow_item_mark nic_mask = {
1387                 .id = priv->sh->dv_mark_mask,
1388         };
1389         int ret;
1390
1391         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1392                 return rte_flow_error_set(error, ENOTSUP,
1393                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1394                                           "extended metadata feature"
1395                                           " isn't enabled");
1396         if (!mlx5_flow_ext_mreg_supported(dev))
1397                 return rte_flow_error_set(error, ENOTSUP,
1398                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1399                                           "extended metadata register"
1400                                           " isn't supported");
1401         if (!nic_mask.id)
1402                 return rte_flow_error_set(error, ENOTSUP,
1403                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1404                                           "extended metadata register"
1405                                           " isn't available");
1406         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1407         if (ret < 0)
1408                 return ret;
1409         if (!spec)
1410                 return rte_flow_error_set(error, EINVAL,
1411                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1412                                           item->spec,
1413                                           "data cannot be empty");
1414         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1415                 return rte_flow_error_set(error, EINVAL,
1416                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1417                                           &spec->id,
1418                                           "mark id exceeds the limit");
1419         if (!mask)
1420                 mask = &nic_mask;
1421         if (!mask->id)
1422                 return rte_flow_error_set(error, EINVAL,
1423                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1424                                         "mask cannot be zero");
1425
1426         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1427                                         (const uint8_t *)&nic_mask,
1428                                         sizeof(struct rte_flow_item_mark),
1429                                         error);
1430         if (ret < 0)
1431                 return ret;
1432         return 0;
1433 }
1434
1435 /**
1436  * Validate META item.
1437  *
1438  * @param[in] dev
1439  *   Pointer to the rte_eth_dev structure.
1440  * @param[in] item
1441  *   Item specification.
1442  * @param[in] attr
1443  *   Attributes of flow that includes this item.
1444  * @param[out] error
1445  *   Pointer to error structure.
1446  *
1447  * @return
1448  *   0 on success, a negative errno value otherwise and rte_errno is set.
1449  */
1450 static int
1451 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1452                            const struct rte_flow_item *item,
1453                            const struct rte_flow_attr *attr,
1454                            struct rte_flow_error *error)
1455 {
1456         struct mlx5_priv *priv = dev->data->dev_private;
1457         struct mlx5_dev_config *config = &priv->config;
1458         const struct rte_flow_item_meta *spec = item->spec;
1459         const struct rte_flow_item_meta *mask = item->mask;
1460         struct rte_flow_item_meta nic_mask = {
1461                 .data = UINT32_MAX
1462         };
1463         int reg;
1464         int ret;
1465
1466         if (!spec)
1467                 return rte_flow_error_set(error, EINVAL,
1468                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1469                                           item->spec,
1470                                           "data cannot be empty");
1471         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1472                 if (!mlx5_flow_ext_mreg_supported(dev))
1473                         return rte_flow_error_set(error, ENOTSUP,
1474                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1475                                           "extended metadata register"
1476                                           " isn't supported");
1477                 reg = flow_dv_get_metadata_reg(dev, attr, error);
1478                 if (reg < 0)
1479                         return reg;
1480                 if (reg == REG_B)
1481                         return rte_flow_error_set(error, ENOTSUP,
1482                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1483                                           "match on reg_b "
1484                                           "isn't supported");
1485                 if (reg != REG_A)
1486                         nic_mask.data = priv->sh->dv_meta_mask;
1487         } else if (attr->transfer) {
1488                 return rte_flow_error_set(error, ENOTSUP,
1489                                         RTE_FLOW_ERROR_TYPE_ITEM, item,
1490                                         "extended metadata feature "
1491                                         "should be enabled when "
1492                                         "meta item is requested "
1493                                         "with e-switch mode ");
1494         }
1495         if (!mask)
1496                 mask = &rte_flow_item_meta_mask;
1497         if (!mask->data)
1498                 return rte_flow_error_set(error, EINVAL,
1499                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1500                                         "mask cannot be zero");
1501
1502         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1503                                         (const uint8_t *)&nic_mask,
1504                                         sizeof(struct rte_flow_item_meta),
1505                                         error);
1506         return ret;
1507 }
1508
1509 /**
1510  * Validate TAG item.
1511  *
1512  * @param[in] dev
1513  *   Pointer to the rte_eth_dev structure.
1514  * @param[in] item
1515  *   Item specification.
1516  * @param[in] attr
1517  *   Attributes of flow that includes this item.
1518  * @param[out] error
1519  *   Pointer to error structure.
1520  *
1521  * @return
1522  *   0 on success, a negative errno value otherwise and rte_errno is set.
1523  */
1524 static int
1525 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1526                           const struct rte_flow_item *item,
1527                           const struct rte_flow_attr *attr __rte_unused,
1528                           struct rte_flow_error *error)
1529 {
1530         const struct rte_flow_item_tag *spec = item->spec;
1531         const struct rte_flow_item_tag *mask = item->mask;
1532         const struct rte_flow_item_tag nic_mask = {
1533                 .data = RTE_BE32(UINT32_MAX),
1534                 .index = 0xff,
1535         };
1536         int ret;
1537
1538         if (!mlx5_flow_ext_mreg_supported(dev))
1539                 return rte_flow_error_set(error, ENOTSUP,
1540                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1541                                           "extensive metadata register"
1542                                           " isn't supported");
1543         if (!spec)
1544                 return rte_flow_error_set(error, EINVAL,
1545                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1546                                           item->spec,
1547                                           "data cannot be empty");
1548         if (!mask)
1549                 mask = &rte_flow_item_tag_mask;
1550         if (!mask->data)
1551                 return rte_flow_error_set(error, EINVAL,
1552                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1553                                         "mask cannot be zero");
1554
1555         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1556                                         (const uint8_t *)&nic_mask,
1557                                         sizeof(struct rte_flow_item_tag),
1558                                         error);
1559         if (ret < 0)
1560                 return ret;
1561         if (mask->index != 0xff)
1562                 return rte_flow_error_set(error, EINVAL,
1563                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1564                                           "partial mask for tag index"
1565                                           " is not supported");
1566         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1567         if (ret < 0)
1568                 return ret;
1569         MLX5_ASSERT(ret != REG_NONE);
1570         return 0;
1571 }
1572
1573 /**
1574  * Validate vport item.
1575  *
1576  * @param[in] dev
1577  *   Pointer to the rte_eth_dev structure.
1578  * @param[in] item
1579  *   Item specification.
1580  * @param[in] attr
1581  *   Attributes of flow that includes this item.
1582  * @param[in] item_flags
1583  *   Bit-fields that holds the items detected until now.
1584  * @param[out] error
1585  *   Pointer to error structure.
1586  *
1587  * @return
1588  *   0 on success, a negative errno value otherwise and rte_errno is set.
1589  */
1590 static int
1591 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1592                               const struct rte_flow_item *item,
1593                               const struct rte_flow_attr *attr,
1594                               uint64_t item_flags,
1595                               struct rte_flow_error *error)
1596 {
1597         const struct rte_flow_item_port_id *spec = item->spec;
1598         const struct rte_flow_item_port_id *mask = item->mask;
1599         const struct rte_flow_item_port_id switch_mask = {
1600                         .id = 0xffffffff,
1601         };
1602         struct mlx5_priv *esw_priv;
1603         struct mlx5_priv *dev_priv;
1604         int ret;
1605
1606         if (!attr->transfer)
1607                 return rte_flow_error_set(error, EINVAL,
1608                                           RTE_FLOW_ERROR_TYPE_ITEM,
1609                                           NULL,
1610                                           "match on port id is valid only"
1611                                           " when transfer flag is enabled");
1612         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1613                 return rte_flow_error_set(error, ENOTSUP,
1614                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1615                                           "multiple source ports are not"
1616                                           " supported");
1617         if (!mask)
1618                 mask = &switch_mask;
1619         if (mask->id != 0xffffffff)
1620                 return rte_flow_error_set(error, ENOTSUP,
1621                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1622                                            mask,
1623                                            "no support for partial mask on"
1624                                            " \"id\" field");
1625         ret = mlx5_flow_item_acceptable
1626                                 (item, (const uint8_t *)mask,
1627                                  (const uint8_t *)&rte_flow_item_port_id_mask,
1628                                  sizeof(struct rte_flow_item_port_id),
1629                                  error);
1630         if (ret)
1631                 return ret;
1632         if (!spec)
1633                 return 0;
1634         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1635         if (!esw_priv)
1636                 return rte_flow_error_set(error, rte_errno,
1637                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1638                                           "failed to obtain E-Switch info for"
1639                                           " port");
1640         dev_priv = mlx5_dev_to_eswitch_info(dev);
1641         if (!dev_priv)
1642                 return rte_flow_error_set(error, rte_errno,
1643                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1644                                           NULL,
1645                                           "failed to obtain E-Switch info");
1646         if (esw_priv->domain_id != dev_priv->domain_id)
1647                 return rte_flow_error_set(error, EINVAL,
1648                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1649                                           "cannot match on a port from a"
1650                                           " different E-Switch");
1651         return 0;
1652 }
1653
1654 /**
1655  * Validate VLAN item.
1656  *
1657  * @param[in] item
1658  *   Item specification.
1659  * @param[in] item_flags
1660  *   Bit-fields that holds the items detected until now.
1661  * @param[in] dev
1662  *   Ethernet device flow is being created on.
1663  * @param[out] error
1664  *   Pointer to error structure.
1665  *
1666  * @return
1667  *   0 on success, a negative errno value otherwise and rte_errno is set.
1668  */
1669 static int
1670 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
1671                            uint64_t item_flags,
1672                            struct rte_eth_dev *dev,
1673                            struct rte_flow_error *error)
1674 {
1675         const struct rte_flow_item_vlan *mask = item->mask;
1676         const struct rte_flow_item_vlan nic_mask = {
1677                 .tci = RTE_BE16(UINT16_MAX),
1678                 .inner_type = RTE_BE16(UINT16_MAX),
1679         };
1680         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1681         int ret;
1682         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1683                                         MLX5_FLOW_LAYER_INNER_L4) :
1684                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1685                                         MLX5_FLOW_LAYER_OUTER_L4);
1686         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1687                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1688
1689         if (item_flags & vlanm)
1690                 return rte_flow_error_set(error, EINVAL,
1691                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1692                                           "multiple VLAN layers not supported");
1693         else if ((item_flags & l34m) != 0)
1694                 return rte_flow_error_set(error, EINVAL,
1695                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1696                                           "VLAN cannot follow L3/L4 layer");
1697         if (!mask)
1698                 mask = &rte_flow_item_vlan_mask;
1699         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1700                                         (const uint8_t *)&nic_mask,
1701                                         sizeof(struct rte_flow_item_vlan),
1702                                         error);
1703         if (ret)
1704                 return ret;
1705         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1706                 struct mlx5_priv *priv = dev->data->dev_private;
1707
1708                 if (priv->vmwa_context) {
1709                         /*
1710                          * Non-NULL context means we have a virtual machine
1711                          * and SR-IOV enabled, we have to create VLAN interface
1712                          * to make hypervisor to setup E-Switch vport
1713                          * context correctly. We avoid creating the multiple
1714                          * VLAN interfaces, so we cannot support VLAN tag mask.
1715                          */
1716                         return rte_flow_error_set(error, EINVAL,
1717                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1718                                                   item,
1719                                                   "VLAN tag mask is not"
1720                                                   " supported in virtual"
1721                                                   " environment");
1722                 }
1723         }
1724         return 0;
1725 }
1726
1727 /*
1728  * GTP flags are contained in 1 byte of the format:
1729  * -------------------------------------------
1730  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
1731  * |-----------------------------------------|
1732  * | value | Version | PT | Res | E | S | PN |
1733  * -------------------------------------------
1734  *
1735  * Matching is supported only for GTP flags E, S, PN.
1736  */
1737 #define MLX5_GTP_FLAGS_MASK     0x07
1738
1739 /**
1740  * Validate GTP item.
1741  *
1742  * @param[in] dev
1743  *   Pointer to the rte_eth_dev structure.
1744  * @param[in] item
1745  *   Item specification.
1746  * @param[in] item_flags
1747  *   Bit-fields that holds the items detected until now.
1748  * @param[out] error
1749  *   Pointer to error structure.
1750  *
1751  * @return
1752  *   0 on success, a negative errno value otherwise and rte_errno is set.
1753  */
1754 static int
1755 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
1756                           const struct rte_flow_item *item,
1757                           uint64_t item_flags,
1758                           struct rte_flow_error *error)
1759 {
1760         struct mlx5_priv *priv = dev->data->dev_private;
1761         const struct rte_flow_item_gtp *spec = item->spec;
1762         const struct rte_flow_item_gtp *mask = item->mask;
1763         const struct rte_flow_item_gtp nic_mask = {
1764                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
1765                 .msg_type = 0xff,
1766                 .teid = RTE_BE32(0xffffffff),
1767         };
1768
1769         if (!priv->config.hca_attr.tunnel_stateless_gtp)
1770                 return rte_flow_error_set(error, ENOTSUP,
1771                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1772                                           "GTP support is not enabled");
1773         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1774                 return rte_flow_error_set(error, ENOTSUP,
1775                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1776                                           "multiple tunnel layers not"
1777                                           " supported");
1778         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1779                 return rte_flow_error_set(error, EINVAL,
1780                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1781                                           "no outer UDP layer found");
1782         if (!mask)
1783                 mask = &rte_flow_item_gtp_mask;
1784         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
1785                 return rte_flow_error_set(error, ENOTSUP,
1786                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1787                                           "Match is supported for GTP"
1788                                           " flags only");
1789         return mlx5_flow_item_acceptable
1790                 (item, (const uint8_t *)mask,
1791                  (const uint8_t *)&nic_mask,
1792                  sizeof(struct rte_flow_item_gtp),
1793                  error);
1794 }
1795
1796 /**
1797  * Validate the pop VLAN action.
1798  *
1799  * @param[in] dev
1800  *   Pointer to the rte_eth_dev structure.
1801  * @param[in] action_flags
1802  *   Holds the actions detected until now.
1803  * @param[in] action
1804  *   Pointer to the pop vlan action.
1805  * @param[in] item_flags
1806  *   The items found in this flow rule.
1807  * @param[in] attr
1808  *   Pointer to flow attributes.
1809  * @param[out] error
1810  *   Pointer to error structure.
1811  *
1812  * @return
1813  *   0 on success, a negative errno value otherwise and rte_errno is set.
1814  */
1815 static int
1816 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
1817                                  uint64_t action_flags,
1818                                  const struct rte_flow_action *action,
1819                                  uint64_t item_flags,
1820                                  const struct rte_flow_attr *attr,
1821                                  struct rte_flow_error *error)
1822 {
1823         const struct mlx5_priv *priv = dev->data->dev_private;
1824
1825         (void)action;
1826         (void)attr;
1827         if (!priv->sh->pop_vlan_action)
1828                 return rte_flow_error_set(error, ENOTSUP,
1829                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1830                                           NULL,
1831                                           "pop vlan action is not supported");
1832         if (attr->egress)
1833                 return rte_flow_error_set(error, ENOTSUP,
1834                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1835                                           NULL,
1836                                           "pop vlan action not supported for "
1837                                           "egress");
1838         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
1839                 return rte_flow_error_set(error, ENOTSUP,
1840                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1841                                           "no support for multiple VLAN "
1842                                           "actions");
1843         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
1844                 return rte_flow_error_set(error, ENOTSUP,
1845                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1846                                           NULL,
1847                                           "cannot pop vlan without a "
1848                                           "match on (outer) vlan in the flow");
1849         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1850                 return rte_flow_error_set(error, EINVAL,
1851                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1852                                           "wrong action order, port_id should "
1853                                           "be after pop VLAN action");
1854         if (!attr->transfer && priv->representor)
1855                 return rte_flow_error_set(error, ENOTSUP,
1856                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1857                                           "pop vlan action for VF representor "
1858                                           "not supported on NIC table");
1859         return 0;
1860 }
1861
1862 /**
1863  * Get VLAN default info from vlan match info.
1864  *
1865  * @param[in] items
1866  *   the list of item specifications.
1867  * @param[out] vlan
1868  *   pointer VLAN info to fill to.
1869  *
1870  * @return
1871  *   0 on success, a negative errno value otherwise and rte_errno is set.
1872  */
1873 static void
1874 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
1875                                   struct rte_vlan_hdr *vlan)
1876 {
1877         const struct rte_flow_item_vlan nic_mask = {
1878                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
1879                                 MLX5DV_FLOW_VLAN_VID_MASK),
1880                 .inner_type = RTE_BE16(0xffff),
1881         };
1882
1883         if (items == NULL)
1884                 return;
1885         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1886                 int type = items->type;
1887
1888                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
1889                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
1890                         break;
1891         }
1892         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
1893                 const struct rte_flow_item_vlan *vlan_m = items->mask;
1894                 const struct rte_flow_item_vlan *vlan_v = items->spec;
1895
1896                 /* If VLAN item in pattern doesn't contain data, return here. */
1897                 if (!vlan_v)
1898                         return;
1899                 if (!vlan_m)
1900                         vlan_m = &nic_mask;
1901                 /* Only full match values are accepted */
1902                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
1903                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
1904                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
1905                         vlan->vlan_tci |=
1906                                 rte_be_to_cpu_16(vlan_v->tci &
1907                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
1908                 }
1909                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
1910                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
1911                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
1912                         vlan->vlan_tci |=
1913                                 rte_be_to_cpu_16(vlan_v->tci &
1914                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
1915                 }
1916                 if (vlan_m->inner_type == nic_mask.inner_type)
1917                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
1918                                                            vlan_m->inner_type);
1919         }
1920 }
1921
1922 /**
1923  * Validate the push VLAN action.
1924  *
1925  * @param[in] dev
1926  *   Pointer to the rte_eth_dev structure.
1927  * @param[in] action_flags
1928  *   Holds the actions detected until now.
1929  * @param[in] item_flags
1930  *   The items found in this flow rule.
1931  * @param[in] action
1932  *   Pointer to the action structure.
1933  * @param[in] attr
1934  *   Pointer to flow attributes
1935  * @param[out] error
1936  *   Pointer to error structure.
1937  *
1938  * @return
1939  *   0 on success, a negative errno value otherwise and rte_errno is set.
1940  */
1941 static int
1942 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
1943                                   uint64_t action_flags,
1944                                   const struct rte_flow_item_vlan *vlan_m,
1945                                   const struct rte_flow_action *action,
1946                                   const struct rte_flow_attr *attr,
1947                                   struct rte_flow_error *error)
1948 {
1949         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
1950         const struct mlx5_priv *priv = dev->data->dev_private;
1951
1952         if (!attr->transfer && attr->ingress)
1953                 return rte_flow_error_set(error, ENOTSUP,
1954                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1955                                           NULL,
1956                                           "push VLAN action not supported for "
1957                                           "ingress");
1958         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
1959             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
1960                 return rte_flow_error_set(error, EINVAL,
1961                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1962                                           "invalid vlan ethertype");
1963         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
1964                 return rte_flow_error_set(error, ENOTSUP,
1965                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1966                                           "no support for multiple VLAN "
1967                                           "actions");
1968         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1969                 return rte_flow_error_set(error, EINVAL,
1970                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1971                                           "wrong action order, port_id should "
1972                                           "be after push VLAN");
1973         if (!attr->transfer && priv->representor)
1974                 return rte_flow_error_set(error, ENOTSUP,
1975                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1976                                           "push vlan action for VF representor "
1977                                           "not supported on NIC table");
1978         if (vlan_m &&
1979             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
1980             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
1981                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
1982             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
1983             !(mlx5_flow_find_action
1984                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
1985                 return rte_flow_error_set(error, EINVAL,
1986                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1987                                           "not full match mask on VLAN PCP and "
1988                                           "there is no of_set_vlan_pcp action, "
1989                                           "push VLAN action cannot figure out "
1990                                           "PCP value");
1991         if (vlan_m &&
1992             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
1993             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
1994                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
1995             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
1996             !(mlx5_flow_find_action
1997                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
1998                 return rte_flow_error_set(error, EINVAL,
1999                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2000                                           "not full match mask on VLAN VID and "
2001                                           "there is no of_set_vlan_vid action, "
2002                                           "push VLAN action cannot figure out "
2003                                           "VID value");
2004         (void)attr;
2005         return 0;
2006 }
2007
2008 /**
2009  * Validate the set VLAN PCP.
2010  *
2011  * @param[in] action_flags
2012  *   Holds the actions detected until now.
2013  * @param[in] actions
2014  *   Pointer to the list of actions remaining in the flow rule.
2015  * @param[out] error
2016  *   Pointer to error structure.
2017  *
2018  * @return
2019  *   0 on success, a negative errno value otherwise and rte_errno is set.
2020  */
2021 static int
2022 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2023                                      const struct rte_flow_action actions[],
2024                                      struct rte_flow_error *error)
2025 {
2026         const struct rte_flow_action *action = actions;
2027         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2028
2029         if (conf->vlan_pcp > 7)
2030                 return rte_flow_error_set(error, EINVAL,
2031                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2032                                           "VLAN PCP value is too big");
2033         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2034                 return rte_flow_error_set(error, ENOTSUP,
2035                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2036                                           "set VLAN PCP action must follow "
2037                                           "the push VLAN action");
2038         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2039                 return rte_flow_error_set(error, ENOTSUP,
2040                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2041                                           "Multiple VLAN PCP modification are "
2042                                           "not supported");
2043         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2044                 return rte_flow_error_set(error, EINVAL,
2045                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2046                                           "wrong action order, port_id should "
2047                                           "be after set VLAN PCP");
2048         return 0;
2049 }
2050
2051 /**
2052  * Validate the set VLAN VID.
2053  *
2054  * @param[in] item_flags
2055  *   Holds the items detected in this rule.
2056  * @param[in] action_flags
2057  *   Holds the actions detected until now.
2058  * @param[in] actions
2059  *   Pointer to the list of actions remaining in the flow rule.
2060  * @param[out] error
2061  *   Pointer to error structure.
2062  *
2063  * @return
2064  *   0 on success, a negative errno value otherwise and rte_errno is set.
2065  */
2066 static int
2067 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2068                                      uint64_t action_flags,
2069                                      const struct rte_flow_action actions[],
2070                                      struct rte_flow_error *error)
2071 {
2072         const struct rte_flow_action *action = actions;
2073         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2074
2075         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2076                 return rte_flow_error_set(error, EINVAL,
2077                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2078                                           "VLAN VID value is too big");
2079         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2080             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2081                 return rte_flow_error_set(error, ENOTSUP,
2082                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2083                                           "set VLAN VID action must follow push"
2084                                           " VLAN action or match on VLAN item");
2085         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2086                 return rte_flow_error_set(error, ENOTSUP,
2087                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2088                                           "Multiple VLAN VID modifications are "
2089                                           "not supported");
2090         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2091                 return rte_flow_error_set(error, EINVAL,
2092                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2093                                           "wrong action order, port_id should "
2094                                           "be after set VLAN VID");
2095         return 0;
2096 }
2097
2098 /*
2099  * Validate the FLAG action.
2100  *
2101  * @param[in] dev
2102  *   Pointer to the rte_eth_dev structure.
2103  * @param[in] action_flags
2104  *   Holds the actions detected until now.
2105  * @param[in] attr
2106  *   Pointer to flow attributes
2107  * @param[out] error
2108  *   Pointer to error structure.
2109  *
2110  * @return
2111  *   0 on success, a negative errno value otherwise and rte_errno is set.
2112  */
2113 static int
2114 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
2115                              uint64_t action_flags,
2116                              const struct rte_flow_attr *attr,
2117                              struct rte_flow_error *error)
2118 {
2119         struct mlx5_priv *priv = dev->data->dev_private;
2120         struct mlx5_dev_config *config = &priv->config;
2121         int ret;
2122
2123         /* Fall back if no extended metadata register support. */
2124         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2125                 return mlx5_flow_validate_action_flag(action_flags, attr,
2126                                                       error);
2127         /* Extensive metadata mode requires registers. */
2128         if (!mlx5_flow_ext_mreg_supported(dev))
2129                 return rte_flow_error_set(error, ENOTSUP,
2130                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2131                                           "no metadata registers "
2132                                           "to support flag action");
2133         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
2134                 return rte_flow_error_set(error, ENOTSUP,
2135                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2136                                           "extended metadata register"
2137                                           " isn't available");
2138         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2139         if (ret < 0)
2140                 return ret;
2141         MLX5_ASSERT(ret > 0);
2142         if (action_flags & MLX5_FLOW_ACTION_MARK)
2143                 return rte_flow_error_set(error, EINVAL,
2144                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2145                                           "can't mark and flag in same flow");
2146         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2147                 return rte_flow_error_set(error, EINVAL,
2148                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2149                                           "can't have 2 flag"
2150                                           " actions in same flow");
2151         return 0;
2152 }
2153
2154 /**
2155  * Validate MARK action.
2156  *
2157  * @param[in] dev
2158  *   Pointer to the rte_eth_dev structure.
2159  * @param[in] action
2160  *   Pointer to action.
2161  * @param[in] action_flags
2162  *   Holds the actions detected until now.
2163  * @param[in] attr
2164  *   Pointer to flow attributes
2165  * @param[out] error
2166  *   Pointer to error structure.
2167  *
2168  * @return
2169  *   0 on success, a negative errno value otherwise and rte_errno is set.
2170  */
2171 static int
2172 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
2173                              const struct rte_flow_action *action,
2174                              uint64_t action_flags,
2175                              const struct rte_flow_attr *attr,
2176                              struct rte_flow_error *error)
2177 {
2178         struct mlx5_priv *priv = dev->data->dev_private;
2179         struct mlx5_dev_config *config = &priv->config;
2180         const struct rte_flow_action_mark *mark = action->conf;
2181         int ret;
2182
2183         /* Fall back if no extended metadata register support. */
2184         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2185                 return mlx5_flow_validate_action_mark(action, action_flags,
2186                                                       attr, error);
2187         /* Extensive metadata mode requires registers. */
2188         if (!mlx5_flow_ext_mreg_supported(dev))
2189                 return rte_flow_error_set(error, ENOTSUP,
2190                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2191                                           "no metadata registers "
2192                                           "to support mark action");
2193         if (!priv->sh->dv_mark_mask)
2194                 return rte_flow_error_set(error, ENOTSUP,
2195                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2196                                           "extended metadata register"
2197                                           " isn't available");
2198         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2199         if (ret < 0)
2200                 return ret;
2201         MLX5_ASSERT(ret > 0);
2202         if (!mark)
2203                 return rte_flow_error_set(error, EINVAL,
2204                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2205                                           "configuration cannot be null");
2206         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
2207                 return rte_flow_error_set(error, EINVAL,
2208                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2209                                           &mark->id,
2210                                           "mark id exceeds the limit");
2211         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2212                 return rte_flow_error_set(error, EINVAL,
2213                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2214                                           "can't flag and mark in same flow");
2215         if (action_flags & MLX5_FLOW_ACTION_MARK)
2216                 return rte_flow_error_set(error, EINVAL,
2217                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2218                                           "can't have 2 mark actions in same"
2219                                           " flow");
2220         return 0;
2221 }
2222
2223 /**
2224  * Validate SET_META action.
2225  *
2226  * @param[in] dev
2227  *   Pointer to the rte_eth_dev structure.
2228  * @param[in] action
2229  *   Pointer to the action structure.
2230  * @param[in] action_flags
2231  *   Holds the actions detected until now.
2232  * @param[in] attr
2233  *   Pointer to flow attributes
2234  * @param[out] error
2235  *   Pointer to error structure.
2236  *
2237  * @return
2238  *   0 on success, a negative errno value otherwise and rte_errno is set.
2239  */
2240 static int
2241 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
2242                                  const struct rte_flow_action *action,
2243                                  uint64_t action_flags __rte_unused,
2244                                  const struct rte_flow_attr *attr,
2245                                  struct rte_flow_error *error)
2246 {
2247         const struct rte_flow_action_set_meta *conf;
2248         uint32_t nic_mask = UINT32_MAX;
2249         int reg;
2250
2251         if (!mlx5_flow_ext_mreg_supported(dev))
2252                 return rte_flow_error_set(error, ENOTSUP,
2253                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2254                                           "extended metadata register"
2255                                           " isn't supported");
2256         reg = flow_dv_get_metadata_reg(dev, attr, error);
2257         if (reg < 0)
2258                 return reg;
2259         if (reg != REG_A && reg != REG_B) {
2260                 struct mlx5_priv *priv = dev->data->dev_private;
2261
2262                 nic_mask = priv->sh->dv_meta_mask;
2263         }
2264         if (!(action->conf))
2265                 return rte_flow_error_set(error, EINVAL,
2266                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2267                                           "configuration cannot be null");
2268         conf = (const struct rte_flow_action_set_meta *)action->conf;
2269         if (!conf->mask)
2270                 return rte_flow_error_set(error, EINVAL,
2271                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2272                                           "zero mask doesn't have any effect");
2273         if (conf->mask & ~nic_mask)
2274                 return rte_flow_error_set(error, EINVAL,
2275                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2276                                           "meta data must be within reg C0");
2277         return 0;
2278 }
2279
2280 /**
2281  * Validate SET_TAG action.
2282  *
2283  * @param[in] dev
2284  *   Pointer to the rte_eth_dev structure.
2285  * @param[in] action
2286  *   Pointer to the action structure.
2287  * @param[in] action_flags
2288  *   Holds the actions detected until now.
2289  * @param[in] attr
2290  *   Pointer to flow attributes
2291  * @param[out] error
2292  *   Pointer to error structure.
2293  *
2294  * @return
2295  *   0 on success, a negative errno value otherwise and rte_errno is set.
2296  */
2297 static int
2298 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
2299                                 const struct rte_flow_action *action,
2300                                 uint64_t action_flags,
2301                                 const struct rte_flow_attr *attr,
2302                                 struct rte_flow_error *error)
2303 {
2304         const struct rte_flow_action_set_tag *conf;
2305         const uint64_t terminal_action_flags =
2306                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
2307                 MLX5_FLOW_ACTION_RSS;
2308         int ret;
2309
2310         if (!mlx5_flow_ext_mreg_supported(dev))
2311                 return rte_flow_error_set(error, ENOTSUP,
2312                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2313                                           "extensive metadata register"
2314                                           " isn't supported");
2315         if (!(action->conf))
2316                 return rte_flow_error_set(error, EINVAL,
2317                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2318                                           "configuration cannot be null");
2319         conf = (const struct rte_flow_action_set_tag *)action->conf;
2320         if (!conf->mask)
2321                 return rte_flow_error_set(error, EINVAL,
2322                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2323                                           "zero mask doesn't have any effect");
2324         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
2325         if (ret < 0)
2326                 return ret;
2327         if (!attr->transfer && attr->ingress &&
2328             (action_flags & terminal_action_flags))
2329                 return rte_flow_error_set(error, EINVAL,
2330                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2331                                           "set_tag has no effect"
2332                                           " with terminal actions");
2333         return 0;
2334 }
2335
2336 /**
2337  * Validate count action.
2338  *
2339  * @param[in] dev
2340  *   Pointer to rte_eth_dev structure.
2341  * @param[out] error
2342  *   Pointer to error structure.
2343  *
2344  * @return
2345  *   0 on success, a negative errno value otherwise and rte_errno is set.
2346  */
2347 static int
2348 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2349                               struct rte_flow_error *error)
2350 {
2351         struct mlx5_priv *priv = dev->data->dev_private;
2352
2353         if (!priv->config.devx)
2354                 goto notsup_err;
2355 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2356         return 0;
2357 #endif
2358 notsup_err:
2359         return rte_flow_error_set
2360                       (error, ENOTSUP,
2361                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2362                        NULL,
2363                        "count action not supported");
2364 }
2365
2366 /**
2367  * Validate the L2 encap action.
2368  *
2369  * @param[in] dev
2370  *   Pointer to the rte_eth_dev structure.
2371  * @param[in] action_flags
2372  *   Holds the actions detected until now.
2373  * @param[in] action
2374  *   Pointer to the action structure.
2375  * @param[in] attr
2376  *   Pointer to flow attributes.
2377  * @param[out] error
2378  *   Pointer to error structure.
2379  *
2380  * @return
2381  *   0 on success, a negative errno value otherwise and rte_errno is set.
2382  */
2383 static int
2384 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
2385                                  uint64_t action_flags,
2386                                  const struct rte_flow_action *action,
2387                                  const struct rte_flow_attr *attr,
2388                                  struct rte_flow_error *error)
2389 {
2390         const struct mlx5_priv *priv = dev->data->dev_private;
2391
2392         if (!(action->conf))
2393                 return rte_flow_error_set(error, EINVAL,
2394                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2395                                           "configuration cannot be null");
2396         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
2397                 return rte_flow_error_set(error, EINVAL,
2398                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2399                                           "can only have a single encap action "
2400                                           "in a flow");
2401         if (!attr->transfer && priv->representor)
2402                 return rte_flow_error_set(error, ENOTSUP,
2403                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2404                                           "encap action for VF representor "
2405                                           "not supported on NIC table");
2406         return 0;
2407 }
2408
2409 /**
2410  * Validate a decap action.
2411  *
2412  * @param[in] dev
2413  *   Pointer to the rte_eth_dev structure.
2414  * @param[in] action_flags
2415  *   Holds the actions detected until now.
2416  * @param[in] attr
2417  *   Pointer to flow attributes
2418  * @param[out] error
2419  *   Pointer to error structure.
2420  *
2421  * @return
2422  *   0 on success, a negative errno value otherwise and rte_errno is set.
2423  */
2424 static int
2425 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
2426                               uint64_t action_flags,
2427                               const struct rte_flow_attr *attr,
2428                               struct rte_flow_error *error)
2429 {
2430         const struct mlx5_priv *priv = dev->data->dev_private;
2431
2432         if (priv->config.hca_attr.scatter_fcs_w_decap_disable &&
2433             !priv->config.decap_en)
2434                 return rte_flow_error_set(error, ENOTSUP,
2435                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2436                                           "decap is not enabled");
2437         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
2438                 return rte_flow_error_set(error, ENOTSUP,
2439                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2440                                           action_flags &
2441                                           MLX5_FLOW_ACTION_DECAP ? "can only "
2442                                           "have a single decap action" : "decap "
2443                                           "after encap is not supported");
2444         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2445                 return rte_flow_error_set(error, EINVAL,
2446                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2447                                           "can't have decap action after"
2448                                           " modify action");
2449         if (attr->egress)
2450                 return rte_flow_error_set(error, ENOTSUP,
2451                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2452                                           NULL,
2453                                           "decap action not supported for "
2454                                           "egress");
2455         if (!attr->transfer && priv->representor)
2456                 return rte_flow_error_set(error, ENOTSUP,
2457                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2458                                           "decap action for VF representor "
2459                                           "not supported on NIC table");
2460         return 0;
2461 }
2462
2463 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
2464
2465 /**
2466  * Validate the raw encap and decap actions.
2467  *
2468  * @param[in] dev
2469  *   Pointer to the rte_eth_dev structure.
2470  * @param[in] decap
2471  *   Pointer to the decap action.
2472  * @param[in] encap
2473  *   Pointer to the encap action.
2474  * @param[in] attr
2475  *   Pointer to flow attributes
2476  * @param[in/out] action_flags
2477  *   Holds the actions detected until now.
2478  * @param[out] actions_n
2479  *   pointer to the number of actions counter.
2480  * @param[out] error
2481  *   Pointer to error structure.
2482  *
2483  * @return
2484  *   0 on success, a negative errno value otherwise and rte_errno is set.
2485  */
2486 static int
2487 flow_dv_validate_action_raw_encap_decap
2488         (struct rte_eth_dev *dev,
2489          const struct rte_flow_action_raw_decap *decap,
2490          const struct rte_flow_action_raw_encap *encap,
2491          const struct rte_flow_attr *attr, uint64_t *action_flags,
2492          int *actions_n, struct rte_flow_error *error)
2493 {
2494         const struct mlx5_priv *priv = dev->data->dev_private;
2495         int ret;
2496
2497         if (encap && (!encap->size || !encap->data))
2498                 return rte_flow_error_set(error, EINVAL,
2499                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2500                                           "raw encap data cannot be empty");
2501         if (decap && encap) {
2502                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
2503                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2504                         /* L3 encap. */
2505                         decap = NULL;
2506                 else if (encap->size <=
2507                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2508                            decap->size >
2509                            MLX5_ENCAPSULATION_DECISION_SIZE)
2510                         /* L3 decap. */
2511                         encap = NULL;
2512                 else if (encap->size >
2513                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2514                            decap->size >
2515                            MLX5_ENCAPSULATION_DECISION_SIZE)
2516                         /* 2 L2 actions: encap and decap. */
2517                         ;
2518                 else
2519                         return rte_flow_error_set(error,
2520                                 ENOTSUP,
2521                                 RTE_FLOW_ERROR_TYPE_ACTION,
2522                                 NULL, "unsupported too small "
2523                                 "raw decap and too small raw "
2524                                 "encap combination");
2525         }
2526         if (decap) {
2527                 ret = flow_dv_validate_action_decap(dev, *action_flags, attr,
2528                                                     error);
2529                 if (ret < 0)
2530                         return ret;
2531                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
2532                 ++(*actions_n);
2533         }
2534         if (encap) {
2535                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
2536                         return rte_flow_error_set(error, ENOTSUP,
2537                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2538                                                   NULL,
2539                                                   "small raw encap size");
2540                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
2541                         return rte_flow_error_set(error, EINVAL,
2542                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2543                                                   NULL,
2544                                                   "more than one encap action");
2545                 if (!attr->transfer && priv->representor)
2546                         return rte_flow_error_set
2547                                         (error, ENOTSUP,
2548                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2549                                          "encap action for VF representor "
2550                                          "not supported on NIC table");
2551                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
2552                 ++(*actions_n);
2553         }
2554         return 0;
2555 }
2556
2557 /**
2558  * Find existing encap/decap resource or create and register a new one.
2559  *
2560  * @param[in, out] dev
2561  *   Pointer to rte_eth_dev structure.
2562  * @param[in, out] resource
2563  *   Pointer to encap/decap resource.
2564  * @parm[in, out] dev_flow
2565  *   Pointer to the dev_flow.
2566  * @param[out] error
2567  *   pointer to error structure.
2568  *
2569  * @return
2570  *   0 on success otherwise -errno and errno is set.
2571  */
2572 static int
2573 flow_dv_encap_decap_resource_register
2574                         (struct rte_eth_dev *dev,
2575                          struct mlx5_flow_dv_encap_decap_resource *resource,
2576                          struct mlx5_flow *dev_flow,
2577                          struct rte_flow_error *error)
2578 {
2579         struct mlx5_priv *priv = dev->data->dev_private;
2580         struct mlx5_dev_ctx_shared *sh = priv->sh;
2581         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2582         struct mlx5dv_dr_domain *domain;
2583         uint32_t idx = 0;
2584         int ret;
2585
2586         resource->flags = dev_flow->dv.group ? 0 : 1;
2587         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2588                 domain = sh->fdb_domain;
2589         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2590                 domain = sh->rx_domain;
2591         else
2592                 domain = sh->tx_domain;
2593         /* Lookup a matching resource from cache. */
2594         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], sh->encaps_decaps, idx,
2595                       cache_resource, next) {
2596                 if (resource->reformat_type == cache_resource->reformat_type &&
2597                     resource->ft_type == cache_resource->ft_type &&
2598                     resource->flags == cache_resource->flags &&
2599                     resource->size == cache_resource->size &&
2600                     !memcmp((const void *)resource->buf,
2601                             (const void *)cache_resource->buf,
2602                             resource->size)) {
2603                         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2604                                 (void *)cache_resource,
2605                                 rte_atomic32_read(&cache_resource->refcnt));
2606                         rte_atomic32_inc(&cache_resource->refcnt);
2607                         dev_flow->handle->dvh.rix_encap_decap = idx;
2608                         dev_flow->dv.encap_decap = cache_resource;
2609                         return 0;
2610                 }
2611         }
2612         /* Register new encap/decap resource. */
2613         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2614                                        &dev_flow->handle->dvh.rix_encap_decap);
2615         if (!cache_resource)
2616                 return rte_flow_error_set(error, ENOMEM,
2617                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2618                                           "cannot allocate resource memory");
2619         *cache_resource = *resource;
2620         ret = mlx5_flow_os_create_flow_action_packet_reformat
2621                                         (sh->ctx, domain, cache_resource,
2622                                          &cache_resource->action);
2623         if (ret) {
2624                 mlx5_free(cache_resource);
2625                 return rte_flow_error_set(error, ENOMEM,
2626                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2627                                           NULL, "cannot create action");
2628         }
2629         rte_atomic32_init(&cache_resource->refcnt);
2630         rte_atomic32_inc(&cache_resource->refcnt);
2631         ILIST_INSERT(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &sh->encaps_decaps,
2632                      dev_flow->handle->dvh.rix_encap_decap, cache_resource,
2633                      next);
2634         dev_flow->dv.encap_decap = cache_resource;
2635         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2636                 (void *)cache_resource,
2637                 rte_atomic32_read(&cache_resource->refcnt));
2638         return 0;
2639 }
2640
2641 /**
2642  * Find existing table jump resource or create and register a new one.
2643  *
2644  * @param[in, out] dev
2645  *   Pointer to rte_eth_dev structure.
2646  * @param[in, out] tbl
2647  *   Pointer to flow table resource.
2648  * @parm[in, out] dev_flow
2649  *   Pointer to the dev_flow.
2650  * @param[out] error
2651  *   pointer to error structure.
2652  *
2653  * @return
2654  *   0 on success otherwise -errno and errno is set.
2655  */
2656 static int
2657 flow_dv_jump_tbl_resource_register
2658                         (struct rte_eth_dev *dev __rte_unused,
2659                          struct mlx5_flow_tbl_resource *tbl,
2660                          struct mlx5_flow *dev_flow,
2661                          struct rte_flow_error *error)
2662 {
2663         struct mlx5_flow_tbl_data_entry *tbl_data =
2664                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2665         int cnt, ret;
2666
2667         MLX5_ASSERT(tbl);
2668         cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
2669         if (!cnt) {
2670                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
2671                                 (tbl->obj, &tbl_data->jump.action);
2672                 if (ret)
2673                         return rte_flow_error_set(error, ENOMEM,
2674                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2675                                         NULL, "cannot create jump action");
2676                 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2677                         (void *)&tbl_data->jump, cnt);
2678         } else {
2679                 /* old jump should not make the table ref++. */
2680                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
2681                 MLX5_ASSERT(tbl_data->jump.action);
2682                 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2683                         (void *)&tbl_data->jump, cnt);
2684         }
2685         rte_atomic32_inc(&tbl_data->jump.refcnt);
2686         dev_flow->handle->rix_jump = tbl_data->idx;
2687         dev_flow->dv.jump = &tbl_data->jump;
2688         return 0;
2689 }
2690
2691 /**
2692  * Find existing default miss resource or create and register a new one.
2693  *
2694  * @param[in, out] dev
2695  *   Pointer to rte_eth_dev structure.
2696  * @param[out] error
2697  *   pointer to error structure.
2698  *
2699  * @return
2700  *   0 on success otherwise -errno and errno is set.
2701  */
2702 static int
2703 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
2704                 struct rte_flow_error *error)
2705 {
2706         struct mlx5_priv *priv = dev->data->dev_private;
2707         struct mlx5_dev_ctx_shared *sh = priv->sh;
2708         struct mlx5_flow_default_miss_resource *cache_resource =
2709                         &sh->default_miss;
2710         int cnt = rte_atomic32_read(&cache_resource->refcnt);
2711
2712         if (!cnt) {
2713                 MLX5_ASSERT(cache_resource->action);
2714                 cache_resource->action =
2715                 mlx5_glue->dr_create_flow_action_default_miss();
2716                 if (!cache_resource->action)
2717                         return rte_flow_error_set(error, ENOMEM,
2718                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2719                                         "cannot create default miss action");
2720                 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++",
2721                                 (void *)cache_resource->action, cnt);
2722         }
2723         rte_atomic32_inc(&cache_resource->refcnt);
2724         return 0;
2725 }
2726
2727 /**
2728  * Find existing table port ID resource or create and register a new one.
2729  *
2730  * @param[in, out] dev
2731  *   Pointer to rte_eth_dev structure.
2732  * @param[in, out] resource
2733  *   Pointer to port ID action resource.
2734  * @parm[in, out] dev_flow
2735  *   Pointer to the dev_flow.
2736  * @param[out] error
2737  *   pointer to error structure.
2738  *
2739  * @return
2740  *   0 on success otherwise -errno and errno is set.
2741  */
2742 static int
2743 flow_dv_port_id_action_resource_register
2744                         (struct rte_eth_dev *dev,
2745                          struct mlx5_flow_dv_port_id_action_resource *resource,
2746                          struct mlx5_flow *dev_flow,
2747                          struct rte_flow_error *error)
2748 {
2749         struct mlx5_priv *priv = dev->data->dev_private;
2750         struct mlx5_dev_ctx_shared *sh = priv->sh;
2751         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
2752         uint32_t idx = 0;
2753         int ret;
2754
2755         /* Lookup a matching resource from cache. */
2756         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list,
2757                       idx, cache_resource, next) {
2758                 if (resource->port_id == cache_resource->port_id) {
2759                         DRV_LOG(DEBUG, "port id action resource resource %p: "
2760                                 "refcnt %d++",
2761                                 (void *)cache_resource,
2762                                 rte_atomic32_read(&cache_resource->refcnt));
2763                         rte_atomic32_inc(&cache_resource->refcnt);
2764                         dev_flow->handle->rix_port_id_action = idx;
2765                         dev_flow->dv.port_id_action = cache_resource;
2766                         return 0;
2767                 }
2768         }
2769         /* Register new port id action resource. */
2770         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID],
2771                                        &dev_flow->handle->rix_port_id_action);
2772         if (!cache_resource)
2773                 return rte_flow_error_set(error, ENOMEM,
2774                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2775                                           "cannot allocate resource memory");
2776         *cache_resource = *resource;
2777         ret = mlx5_flow_os_create_flow_action_dest_port
2778                                 (priv->sh->fdb_domain, resource->port_id,
2779                                  &cache_resource->action);
2780         if (ret) {
2781                 mlx5_free(cache_resource);
2782                 return rte_flow_error_set(error, ENOMEM,
2783                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2784                                           NULL, "cannot create action");
2785         }
2786         rte_atomic32_init(&cache_resource->refcnt);
2787         rte_atomic32_inc(&cache_resource->refcnt);
2788         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
2789                      dev_flow->handle->rix_port_id_action, cache_resource,
2790                      next);
2791         dev_flow->dv.port_id_action = cache_resource;
2792         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
2793                 (void *)cache_resource,
2794                 rte_atomic32_read(&cache_resource->refcnt));
2795         return 0;
2796 }
2797
2798 /**
2799  * Find existing push vlan resource or create and register a new one.
2800  *
2801  * @param [in, out] dev
2802  *   Pointer to rte_eth_dev structure.
2803  * @param[in, out] resource
2804  *   Pointer to port ID action resource.
2805  * @parm[in, out] dev_flow
2806  *   Pointer to the dev_flow.
2807  * @param[out] error
2808  *   pointer to error structure.
2809  *
2810  * @return
2811  *   0 on success otherwise -errno and errno is set.
2812  */
2813 static int
2814 flow_dv_push_vlan_action_resource_register
2815                        (struct rte_eth_dev *dev,
2816                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
2817                         struct mlx5_flow *dev_flow,
2818                         struct rte_flow_error *error)
2819 {
2820         struct mlx5_priv *priv = dev->data->dev_private;
2821         struct mlx5_dev_ctx_shared *sh = priv->sh;
2822         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
2823         struct mlx5dv_dr_domain *domain;
2824         uint32_t idx = 0;
2825         int ret;
2826
2827         /* Lookup a matching resource from cache. */
2828         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2829                       sh->push_vlan_action_list, idx, cache_resource, next) {
2830                 if (resource->vlan_tag == cache_resource->vlan_tag &&
2831                     resource->ft_type == cache_resource->ft_type) {
2832                         DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
2833                                 "refcnt %d++",
2834                                 (void *)cache_resource,
2835                                 rte_atomic32_read(&cache_resource->refcnt));
2836                         rte_atomic32_inc(&cache_resource->refcnt);
2837                         dev_flow->handle->dvh.rix_push_vlan = idx;
2838                         dev_flow->dv.push_vlan_res = cache_resource;
2839                         return 0;
2840                 }
2841         }
2842         /* Register new push_vlan action resource. */
2843         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2844                                        &dev_flow->handle->dvh.rix_push_vlan);
2845         if (!cache_resource)
2846                 return rte_flow_error_set(error, ENOMEM,
2847                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2848                                           "cannot allocate resource memory");
2849         *cache_resource = *resource;
2850         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2851                 domain = sh->fdb_domain;
2852         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2853                 domain = sh->rx_domain;
2854         else
2855                 domain = sh->tx_domain;
2856         ret = mlx5_flow_os_create_flow_action_push_vlan
2857                                         (domain, resource->vlan_tag,
2858                                          &cache_resource->action);
2859         if (ret) {
2860                 mlx5_free(cache_resource);
2861                 return rte_flow_error_set(error, ENOMEM,
2862                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2863                                           NULL, "cannot create action");
2864         }
2865         rte_atomic32_init(&cache_resource->refcnt);
2866         rte_atomic32_inc(&cache_resource->refcnt);
2867         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2868                      &sh->push_vlan_action_list,
2869                      dev_flow->handle->dvh.rix_push_vlan,
2870                      cache_resource, next);
2871         dev_flow->dv.push_vlan_res = cache_resource;
2872         DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
2873                 (void *)cache_resource,
2874                 rte_atomic32_read(&cache_resource->refcnt));
2875         return 0;
2876 }
2877 /**
2878  * Get the size of specific rte_flow_item_type
2879  *
2880  * @param[in] item_type
2881  *   Tested rte_flow_item_type.
2882  *
2883  * @return
2884  *   sizeof struct item_type, 0 if void or irrelevant.
2885  */
2886 static size_t
2887 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
2888 {
2889         size_t retval;
2890
2891         switch (item_type) {
2892         case RTE_FLOW_ITEM_TYPE_ETH:
2893                 retval = sizeof(struct rte_flow_item_eth);
2894                 break;
2895         case RTE_FLOW_ITEM_TYPE_VLAN:
2896                 retval = sizeof(struct rte_flow_item_vlan);
2897                 break;
2898         case RTE_FLOW_ITEM_TYPE_IPV4:
2899                 retval = sizeof(struct rte_flow_item_ipv4);
2900                 break;
2901         case RTE_FLOW_ITEM_TYPE_IPV6:
2902                 retval = sizeof(struct rte_flow_item_ipv6);
2903                 break;
2904         case RTE_FLOW_ITEM_TYPE_UDP:
2905                 retval = sizeof(struct rte_flow_item_udp);
2906                 break;
2907         case RTE_FLOW_ITEM_TYPE_TCP:
2908                 retval = sizeof(struct rte_flow_item_tcp);
2909                 break;
2910         case RTE_FLOW_ITEM_TYPE_VXLAN:
2911                 retval = sizeof(struct rte_flow_item_vxlan);
2912                 break;
2913         case RTE_FLOW_ITEM_TYPE_GRE:
2914                 retval = sizeof(struct rte_flow_item_gre);
2915                 break;
2916         case RTE_FLOW_ITEM_TYPE_NVGRE:
2917                 retval = sizeof(struct rte_flow_item_nvgre);
2918                 break;
2919         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
2920                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
2921                 break;
2922         case RTE_FLOW_ITEM_TYPE_MPLS:
2923                 retval = sizeof(struct rte_flow_item_mpls);
2924                 break;
2925         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
2926         default:
2927                 retval = 0;
2928                 break;
2929         }
2930         return retval;
2931 }
2932
2933 #define MLX5_ENCAP_IPV4_VERSION         0x40
2934 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
2935 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
2936 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
2937 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
2938 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
2939 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
2940
2941 /**
2942  * Convert the encap action data from list of rte_flow_item to raw buffer
2943  *
2944  * @param[in] items
2945  *   Pointer to rte_flow_item objects list.
2946  * @param[out] buf
2947  *   Pointer to the output buffer.
2948  * @param[out] size
2949  *   Pointer to the output buffer size.
2950  * @param[out] error
2951  *   Pointer to the error structure.
2952  *
2953  * @return
2954  *   0 on success, a negative errno value otherwise and rte_errno is set.
2955  */
2956 static int
2957 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
2958                            size_t *size, struct rte_flow_error *error)
2959 {
2960         struct rte_ether_hdr *eth = NULL;
2961         struct rte_vlan_hdr *vlan = NULL;
2962         struct rte_ipv4_hdr *ipv4 = NULL;
2963         struct rte_ipv6_hdr *ipv6 = NULL;
2964         struct rte_udp_hdr *udp = NULL;
2965         struct rte_vxlan_hdr *vxlan = NULL;
2966         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
2967         struct rte_gre_hdr *gre = NULL;
2968         size_t len;
2969         size_t temp_size = 0;
2970
2971         if (!items)
2972                 return rte_flow_error_set(error, EINVAL,
2973                                           RTE_FLOW_ERROR_TYPE_ACTION,
2974                                           NULL, "invalid empty data");
2975         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2976                 len = flow_dv_get_item_len(items->type);
2977                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
2978                         return rte_flow_error_set(error, EINVAL,
2979                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2980                                                   (void *)items->type,
2981                                                   "items total size is too big"
2982                                                   " for encap action");
2983                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
2984                 switch (items->type) {
2985                 case RTE_FLOW_ITEM_TYPE_ETH:
2986                         eth = (struct rte_ether_hdr *)&buf[temp_size];
2987                         break;
2988                 case RTE_FLOW_ITEM_TYPE_VLAN:
2989                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
2990                         if (!eth)
2991                                 return rte_flow_error_set(error, EINVAL,
2992                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2993                                                 (void *)items->type,
2994                                                 "eth header not found");
2995                         if (!eth->ether_type)
2996                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
2997                         break;
2998                 case RTE_FLOW_ITEM_TYPE_IPV4:
2999                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
3000                         if (!vlan && !eth)
3001                                 return rte_flow_error_set(error, EINVAL,
3002                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3003                                                 (void *)items->type,
3004                                                 "neither eth nor vlan"
3005                                                 " header found");
3006                         if (vlan && !vlan->eth_proto)
3007                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3008                         else if (eth && !eth->ether_type)
3009                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3010                         if (!ipv4->version_ihl)
3011                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
3012                                                     MLX5_ENCAP_IPV4_IHL_MIN;
3013                         if (!ipv4->time_to_live)
3014                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
3015                         break;
3016                 case RTE_FLOW_ITEM_TYPE_IPV6:
3017                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
3018                         if (!vlan && !eth)
3019                                 return rte_flow_error_set(error, EINVAL,
3020                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3021                                                 (void *)items->type,
3022                                                 "neither eth nor vlan"
3023                                                 " header found");
3024                         if (vlan && !vlan->eth_proto)
3025                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3026                         else if (eth && !eth->ether_type)
3027                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3028                         if (!ipv6->vtc_flow)
3029                                 ipv6->vtc_flow =
3030                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
3031                         if (!ipv6->hop_limits)
3032                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
3033                         break;
3034                 case RTE_FLOW_ITEM_TYPE_UDP:
3035                         udp = (struct rte_udp_hdr *)&buf[temp_size];
3036                         if (!ipv4 && !ipv6)
3037                                 return rte_flow_error_set(error, EINVAL,
3038                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3039                                                 (void *)items->type,
3040                                                 "ip header not found");
3041                         if (ipv4 && !ipv4->next_proto_id)
3042                                 ipv4->next_proto_id = IPPROTO_UDP;
3043                         else if (ipv6 && !ipv6->proto)
3044                                 ipv6->proto = IPPROTO_UDP;
3045                         break;
3046                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3047                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
3048                         if (!udp)
3049                                 return rte_flow_error_set(error, EINVAL,
3050                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3051                                                 (void *)items->type,
3052                                                 "udp header not found");
3053                         if (!udp->dst_port)
3054                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
3055                         if (!vxlan->vx_flags)
3056                                 vxlan->vx_flags =
3057                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
3058                         break;
3059                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3060                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
3061                         if (!udp)
3062                                 return rte_flow_error_set(error, EINVAL,
3063                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3064                                                 (void *)items->type,
3065                                                 "udp header not found");
3066                         if (!vxlan_gpe->proto)
3067                                 return rte_flow_error_set(error, EINVAL,
3068                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3069                                                 (void *)items->type,
3070                                                 "next protocol not found");
3071                         if (!udp->dst_port)
3072                                 udp->dst_port =
3073                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
3074                         if (!vxlan_gpe->vx_flags)
3075                                 vxlan_gpe->vx_flags =
3076                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
3077                         break;
3078                 case RTE_FLOW_ITEM_TYPE_GRE:
3079                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3080                         gre = (struct rte_gre_hdr *)&buf[temp_size];
3081                         if (!gre->proto)
3082                                 return rte_flow_error_set(error, EINVAL,
3083                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3084                                                 (void *)items->type,
3085                                                 "next protocol not found");
3086                         if (!ipv4 && !ipv6)
3087                                 return rte_flow_error_set(error, EINVAL,
3088                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3089                                                 (void *)items->type,
3090                                                 "ip header not found");
3091                         if (ipv4 && !ipv4->next_proto_id)
3092                                 ipv4->next_proto_id = IPPROTO_GRE;
3093                         else if (ipv6 && !ipv6->proto)
3094                                 ipv6->proto = IPPROTO_GRE;
3095                         break;
3096                 case RTE_FLOW_ITEM_TYPE_VOID:
3097                         break;
3098                 default:
3099                         return rte_flow_error_set(error, EINVAL,
3100                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3101                                                   (void *)items->type,
3102                                                   "unsupported item type");
3103                         break;
3104                 }
3105                 temp_size += len;
3106         }
3107         *size = temp_size;
3108         return 0;
3109 }
3110
3111 static int
3112 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
3113 {
3114         struct rte_ether_hdr *eth = NULL;
3115         struct rte_vlan_hdr *vlan = NULL;
3116         struct rte_ipv6_hdr *ipv6 = NULL;
3117         struct rte_udp_hdr *udp = NULL;
3118         char *next_hdr;
3119         uint16_t proto;
3120
3121         eth = (struct rte_ether_hdr *)data;
3122         next_hdr = (char *)(eth + 1);
3123         proto = RTE_BE16(eth->ether_type);
3124
3125         /* VLAN skipping */
3126         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
3127                 vlan = (struct rte_vlan_hdr *)next_hdr;
3128                 proto = RTE_BE16(vlan->eth_proto);
3129                 next_hdr += sizeof(struct rte_vlan_hdr);
3130         }
3131
3132         /* HW calculates IPv4 csum. no need to proceed */
3133         if (proto == RTE_ETHER_TYPE_IPV4)
3134                 return 0;
3135
3136         /* non IPv4/IPv6 header. not supported */
3137         if (proto != RTE_ETHER_TYPE_IPV6) {
3138                 return rte_flow_error_set(error, ENOTSUP,
3139                                           RTE_FLOW_ERROR_TYPE_ACTION,
3140                                           NULL, "Cannot offload non IPv4/IPv6");
3141         }
3142
3143         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
3144
3145         /* ignore non UDP */
3146         if (ipv6->proto != IPPROTO_UDP)
3147                 return 0;
3148
3149         udp = (struct rte_udp_hdr *)(ipv6 + 1);
3150         udp->dgram_cksum = 0;
3151
3152         return 0;
3153 }
3154
3155 /**
3156  * Convert L2 encap action to DV specification.
3157  *
3158  * @param[in] dev
3159  *   Pointer to rte_eth_dev structure.
3160  * @param[in] action
3161  *   Pointer to action structure.
3162  * @param[in, out] dev_flow
3163  *   Pointer to the mlx5_flow.
3164  * @param[in] transfer
3165  *   Mark if the flow is E-Switch flow.
3166  * @param[out] error
3167  *   Pointer to the error structure.
3168  *
3169  * @return
3170  *   0 on success, a negative errno value otherwise and rte_errno is set.
3171  */
3172 static int
3173 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
3174                                const struct rte_flow_action *action,
3175                                struct mlx5_flow *dev_flow,
3176                                uint8_t transfer,
3177                                struct rte_flow_error *error)
3178 {
3179         const struct rte_flow_item *encap_data;
3180         const struct rte_flow_action_raw_encap *raw_encap_data;
3181         struct mlx5_flow_dv_encap_decap_resource res = {
3182                 .reformat_type =
3183                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
3184                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3185                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
3186         };
3187
3188         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3189                 raw_encap_data =
3190                         (const struct rte_flow_action_raw_encap *)action->conf;
3191                 res.size = raw_encap_data->size;
3192                 memcpy(res.buf, raw_encap_data->data, res.size);
3193         } else {
3194                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
3195                         encap_data =
3196                                 ((const struct rte_flow_action_vxlan_encap *)
3197                                                 action->conf)->definition;
3198                 else
3199                         encap_data =
3200                                 ((const struct rte_flow_action_nvgre_encap *)
3201                                                 action->conf)->definition;
3202                 if (flow_dv_convert_encap_data(encap_data, res.buf,
3203                                                &res.size, error))
3204                         return -rte_errno;
3205         }
3206         if (flow_dv_zero_encap_udp_csum(res.buf, error))
3207                 return -rte_errno;
3208         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3209                 return rte_flow_error_set(error, EINVAL,
3210                                           RTE_FLOW_ERROR_TYPE_ACTION,
3211                                           NULL, "can't create L2 encap action");
3212         return 0;
3213 }
3214
3215 /**
3216  * Convert L2 decap action to DV specification.
3217  *
3218  * @param[in] dev
3219  *   Pointer to rte_eth_dev structure.
3220  * @param[in, out] dev_flow
3221  *   Pointer to the mlx5_flow.
3222  * @param[in] transfer
3223  *   Mark if the flow is E-Switch flow.
3224  * @param[out] error
3225  *   Pointer to the error structure.
3226  *
3227  * @return
3228  *   0 on success, a negative errno value otherwise and rte_errno is set.
3229  */
3230 static int
3231 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
3232                                struct mlx5_flow *dev_flow,
3233                                uint8_t transfer,
3234                                struct rte_flow_error *error)
3235 {
3236         struct mlx5_flow_dv_encap_decap_resource res = {
3237                 .size = 0,
3238                 .reformat_type =
3239                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
3240                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3241                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
3242         };
3243
3244         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3245                 return rte_flow_error_set(error, EINVAL,
3246                                           RTE_FLOW_ERROR_TYPE_ACTION,
3247                                           NULL, "can't create L2 decap action");
3248         return 0;
3249 }
3250
3251 /**
3252  * Convert raw decap/encap (L3 tunnel) action to DV specification.
3253  *
3254  * @param[in] dev
3255  *   Pointer to rte_eth_dev structure.
3256  * @param[in] action
3257  *   Pointer to action structure.
3258  * @param[in, out] dev_flow
3259  *   Pointer to the mlx5_flow.
3260  * @param[in] attr
3261  *   Pointer to the flow attributes.
3262  * @param[out] error
3263  *   Pointer to the error structure.
3264  *
3265  * @return
3266  *   0 on success, a negative errno value otherwise and rte_errno is set.
3267  */
3268 static int
3269 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
3270                                 const struct rte_flow_action *action,
3271                                 struct mlx5_flow *dev_flow,
3272                                 const struct rte_flow_attr *attr,
3273                                 struct rte_flow_error *error)
3274 {
3275         const struct rte_flow_action_raw_encap *encap_data;
3276         struct mlx5_flow_dv_encap_decap_resource res;
3277
3278         memset(&res, 0, sizeof(res));
3279         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
3280         res.size = encap_data->size;
3281         memcpy(res.buf, encap_data->data, res.size);
3282         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
3283                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
3284                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3285         if (attr->transfer)
3286                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3287         else
3288                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3289                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3290         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3291                 return rte_flow_error_set(error, EINVAL,
3292                                           RTE_FLOW_ERROR_TYPE_ACTION,
3293                                           NULL, "can't create encap action");
3294         return 0;
3295 }
3296
3297 /**
3298  * Create action push VLAN.
3299  *
3300  * @param[in] dev
3301  *   Pointer to rte_eth_dev structure.
3302  * @param[in] attr
3303  *   Pointer to the flow attributes.
3304  * @param[in] vlan
3305  *   Pointer to the vlan to push to the Ethernet header.
3306  * @param[in, out] dev_flow
3307  *   Pointer to the mlx5_flow.
3308  * @param[out] error
3309  *   Pointer to the error structure.
3310  *
3311  * @return
3312  *   0 on success, a negative errno value otherwise and rte_errno is set.
3313  */
3314 static int
3315 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
3316                                 const struct rte_flow_attr *attr,
3317                                 const struct rte_vlan_hdr *vlan,
3318                                 struct mlx5_flow *dev_flow,
3319                                 struct rte_flow_error *error)
3320 {
3321         struct mlx5_flow_dv_push_vlan_action_resource res;
3322
3323         memset(&res, 0, sizeof(res));
3324         res.vlan_tag =
3325                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
3326                                  vlan->vlan_tci);
3327         if (attr->transfer)
3328                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3329         else
3330                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3331                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3332         return flow_dv_push_vlan_action_resource_register
3333                                             (dev, &res, dev_flow, error);
3334 }
3335
3336 /**
3337  * Validate the modify-header actions.
3338  *
3339  * @param[in] action_flags
3340  *   Holds the actions detected until now.
3341  * @param[in] action
3342  *   Pointer to the modify action.
3343  * @param[out] error
3344  *   Pointer to error structure.
3345  *
3346  * @return
3347  *   0 on success, a negative errno value otherwise and rte_errno is set.
3348  */
3349 static int
3350 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
3351                                    const struct rte_flow_action *action,
3352                                    struct rte_flow_error *error)
3353 {
3354         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
3355                 return rte_flow_error_set(error, EINVAL,
3356                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3357                                           NULL, "action configuration not set");
3358         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3359                 return rte_flow_error_set(error, EINVAL,
3360                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3361                                           "can't have encap action before"
3362                                           " modify action");
3363         return 0;
3364 }
3365
3366 /**
3367  * Validate the modify-header MAC address actions.
3368  *
3369  * @param[in] action_flags
3370  *   Holds the actions detected until now.
3371  * @param[in] action
3372  *   Pointer to the modify action.
3373  * @param[in] item_flags
3374  *   Holds the items detected.
3375  * @param[out] error
3376  *   Pointer to error structure.
3377  *
3378  * @return
3379  *   0 on success, a negative errno value otherwise and rte_errno is set.
3380  */
3381 static int
3382 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
3383                                    const struct rte_flow_action *action,
3384                                    const uint64_t item_flags,
3385                                    struct rte_flow_error *error)
3386 {
3387         int ret = 0;
3388
3389         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3390         if (!ret) {
3391                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
3392                         return rte_flow_error_set(error, EINVAL,
3393                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3394                                                   NULL,
3395                                                   "no L2 item in pattern");
3396         }
3397         return ret;
3398 }
3399
3400 /**
3401  * Validate the modify-header IPv4 address actions.
3402  *
3403  * @param[in] action_flags
3404  *   Holds the actions detected until now.
3405  * @param[in] action
3406  *   Pointer to the modify action.
3407  * @param[in] item_flags
3408  *   Holds the items detected.
3409  * @param[out] error
3410  *   Pointer to error structure.
3411  *
3412  * @return
3413  *   0 on success, a negative errno value otherwise and rte_errno is set.
3414  */
3415 static int
3416 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3417                                     const struct rte_flow_action *action,
3418                                     const uint64_t item_flags,
3419                                     struct rte_flow_error *error)
3420 {
3421         int ret = 0;
3422         uint64_t layer;
3423
3424         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3425         if (!ret) {
3426                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3427                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3428                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3429                 if (!(item_flags & layer))
3430                         return rte_flow_error_set(error, EINVAL,
3431                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3432                                                   NULL,
3433                                                   "no ipv4 item in pattern");
3434         }
3435         return ret;
3436 }
3437
3438 /**
3439  * Validate the modify-header IPv6 address actions.
3440  *
3441  * @param[in] action_flags
3442  *   Holds the actions detected until now.
3443  * @param[in] action
3444  *   Pointer to the modify action.
3445  * @param[in] item_flags
3446  *   Holds the items detected.
3447  * @param[out] error
3448  *   Pointer to error structure.
3449  *
3450  * @return
3451  *   0 on success, a negative errno value otherwise and rte_errno is set.
3452  */
3453 static int
3454 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3455                                     const struct rte_flow_action *action,
3456                                     const uint64_t item_flags,
3457                                     struct rte_flow_error *error)
3458 {
3459         int ret = 0;
3460         uint64_t layer;
3461
3462         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3463         if (!ret) {
3464                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3465                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3466                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3467                 if (!(item_flags & layer))
3468                         return rte_flow_error_set(error, EINVAL,
3469                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3470                                                   NULL,
3471                                                   "no ipv6 item in pattern");
3472         }
3473         return ret;
3474 }
3475
3476 /**
3477  * Validate the modify-header TP actions.
3478  *
3479  * @param[in] action_flags
3480  *   Holds the actions detected until now.
3481  * @param[in] action
3482  *   Pointer to the modify action.
3483  * @param[in] item_flags
3484  *   Holds the items detected.
3485  * @param[out] error
3486  *   Pointer to error structure.
3487  *
3488  * @return
3489  *   0 on success, a negative errno value otherwise and rte_errno is set.
3490  */
3491 static int
3492 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3493                                   const struct rte_flow_action *action,
3494                                   const uint64_t item_flags,
3495                                   struct rte_flow_error *error)
3496 {
3497         int ret = 0;
3498         uint64_t layer;
3499
3500         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3501         if (!ret) {
3502                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3503                                  MLX5_FLOW_LAYER_INNER_L4 :
3504                                  MLX5_FLOW_LAYER_OUTER_L4;
3505                 if (!(item_flags & layer))
3506                         return rte_flow_error_set(error, EINVAL,
3507                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3508                                                   NULL, "no transport layer "
3509                                                   "in pattern");
3510         }
3511         return ret;
3512 }
3513
3514 /**
3515  * Validate the modify-header actions of increment/decrement
3516  * TCP Sequence-number.
3517  *
3518  * @param[in] action_flags
3519  *   Holds the actions detected until now.
3520  * @param[in] action
3521  *   Pointer to the modify action.
3522  * @param[in] item_flags
3523  *   Holds the items detected.
3524  * @param[out] error
3525  *   Pointer to error structure.
3526  *
3527  * @return
3528  *   0 on success, a negative errno value otherwise and rte_errno is set.
3529  */
3530 static int
3531 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3532                                        const struct rte_flow_action *action,
3533                                        const uint64_t item_flags,
3534                                        struct rte_flow_error *error)
3535 {
3536         int ret = 0;
3537         uint64_t layer;
3538
3539         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3540         if (!ret) {
3541                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3542                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3543                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3544                 if (!(item_flags & layer))
3545                         return rte_flow_error_set(error, EINVAL,
3546                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3547                                                   NULL, "no TCP item in"
3548                                                   " pattern");
3549                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3550                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3551                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3552                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3553                         return rte_flow_error_set(error, EINVAL,
3554                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3555                                                   NULL,
3556                                                   "cannot decrease and increase"
3557                                                   " TCP sequence number"
3558                                                   " at the same time");
3559         }
3560         return ret;
3561 }
3562
3563 /**
3564  * Validate the modify-header actions of increment/decrement
3565  * TCP Acknowledgment number.
3566  *
3567  * @param[in] action_flags
3568  *   Holds the actions detected until now.
3569  * @param[in] action
3570  *   Pointer to the modify action.
3571  * @param[in] item_flags
3572  *   Holds the items detected.
3573  * @param[out] error
3574  *   Pointer to error structure.
3575  *
3576  * @return
3577  *   0 on success, a negative errno value otherwise and rte_errno is set.
3578  */
3579 static int
3580 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3581                                        const struct rte_flow_action *action,
3582                                        const uint64_t item_flags,
3583                                        struct rte_flow_error *error)
3584 {
3585         int ret = 0;
3586         uint64_t layer;
3587
3588         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3589         if (!ret) {
3590                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3591                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3592                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3593                 if (!(item_flags & layer))
3594                         return rte_flow_error_set(error, EINVAL,
3595                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3596                                                   NULL, "no TCP item in"
3597                                                   " pattern");
3598                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3599                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3600                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3601                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3602                         return rte_flow_error_set(error, EINVAL,
3603                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3604                                                   NULL,
3605                                                   "cannot decrease and increase"
3606                                                   " TCP acknowledgment number"
3607                                                   " at the same time");
3608         }
3609         return ret;
3610 }
3611
3612 /**
3613  * Validate the modify-header TTL actions.
3614  *
3615  * @param[in] action_flags
3616  *   Holds the actions detected until now.
3617  * @param[in] action
3618  *   Pointer to the modify action.
3619  * @param[in] item_flags
3620  *   Holds the items detected.
3621  * @param[out] error
3622  *   Pointer to error structure.
3623  *
3624  * @return
3625  *   0 on success, a negative errno value otherwise and rte_errno is set.
3626  */
3627 static int
3628 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3629                                    const struct rte_flow_action *action,
3630                                    const uint64_t item_flags,
3631                                    struct rte_flow_error *error)
3632 {
3633         int ret = 0;
3634         uint64_t layer;
3635
3636         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3637         if (!ret) {
3638                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3639                                  MLX5_FLOW_LAYER_INNER_L3 :
3640                                  MLX5_FLOW_LAYER_OUTER_L3;
3641                 if (!(item_flags & layer))
3642                         return rte_flow_error_set(error, EINVAL,
3643                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3644                                                   NULL,
3645                                                   "no IP protocol in pattern");
3646         }
3647         return ret;
3648 }
3649
3650 /**
3651  * Validate jump action.
3652  *
3653  * @param[in] action
3654  *   Pointer to the jump action.
3655  * @param[in] action_flags
3656  *   Holds the actions detected until now.
3657  * @param[in] attributes
3658  *   Pointer to flow attributes
3659  * @param[in] external
3660  *   Action belongs to flow rule created by request external to PMD.
3661  * @param[out] error
3662  *   Pointer to error structure.
3663  *
3664  * @return
3665  *   0 on success, a negative errno value otherwise and rte_errno is set.
3666  */
3667 static int
3668 flow_dv_validate_action_jump(const struct rte_flow_action *action,
3669                              uint64_t action_flags,
3670                              const struct rte_flow_attr *attributes,
3671                              bool external, struct rte_flow_error *error)
3672 {
3673         uint32_t target_group, table;
3674         int ret = 0;
3675
3676         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3677                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3678                 return rte_flow_error_set(error, EINVAL,
3679                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3680                                           "can't have 2 fate actions in"
3681                                           " same flow");
3682         if (action_flags & MLX5_FLOW_ACTION_METER)
3683                 return rte_flow_error_set(error, ENOTSUP,
3684                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3685                                           "jump with meter not support");
3686         if (!action->conf)
3687                 return rte_flow_error_set(error, EINVAL,
3688                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3689                                           NULL, "action configuration not set");
3690         target_group =
3691                 ((const struct rte_flow_action_jump *)action->conf)->group;
3692         ret = mlx5_flow_group_to_table(attributes, external, target_group,
3693                                        true, &table, error);
3694         if (ret)
3695                 return ret;
3696         if (attributes->group == target_group)
3697                 return rte_flow_error_set(error, EINVAL,
3698                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3699                                           "target group must be other than"
3700                                           " the current flow group");
3701         return 0;
3702 }
3703
3704 /*
3705  * Validate the port_id action.
3706  *
3707  * @param[in] dev
3708  *   Pointer to rte_eth_dev structure.
3709  * @param[in] action_flags
3710  *   Bit-fields that holds the actions detected until now.
3711  * @param[in] action
3712  *   Port_id RTE action structure.
3713  * @param[in] attr
3714  *   Attributes of flow that includes this action.
3715  * @param[out] error
3716  *   Pointer to error structure.
3717  *
3718  * @return
3719  *   0 on success, a negative errno value otherwise and rte_errno is set.
3720  */
3721 static int
3722 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
3723                                 uint64_t action_flags,
3724                                 const struct rte_flow_action *action,
3725                                 const struct rte_flow_attr *attr,
3726                                 struct rte_flow_error *error)
3727 {
3728         const struct rte_flow_action_port_id *port_id;
3729         struct mlx5_priv *act_priv;
3730         struct mlx5_priv *dev_priv;
3731         uint16_t port;
3732
3733         if (!attr->transfer)
3734                 return rte_flow_error_set(error, ENOTSUP,
3735                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3736                                           NULL,
3737                                           "port id action is valid in transfer"
3738                                           " mode only");
3739         if (!action || !action->conf)
3740                 return rte_flow_error_set(error, ENOTSUP,
3741                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3742                                           NULL,
3743                                           "port id action parameters must be"
3744                                           " specified");
3745         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3746                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3747                 return rte_flow_error_set(error, EINVAL,
3748                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3749                                           "can have only one fate actions in"
3750                                           " a flow");
3751         dev_priv = mlx5_dev_to_eswitch_info(dev);
3752         if (!dev_priv)
3753                 return rte_flow_error_set(error, rte_errno,
3754                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3755                                           NULL,
3756                                           "failed to obtain E-Switch info");
3757         port_id = action->conf;
3758         port = port_id->original ? dev->data->port_id : port_id->id;
3759         act_priv = mlx5_port_to_eswitch_info(port, false);
3760         if (!act_priv)
3761                 return rte_flow_error_set
3762                                 (error, rte_errno,
3763                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
3764                                  "failed to obtain E-Switch port id for port");
3765         if (act_priv->domain_id != dev_priv->domain_id)
3766                 return rte_flow_error_set
3767                                 (error, EINVAL,
3768                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3769                                  "port does not belong to"
3770                                  " E-Switch being configured");
3771         return 0;
3772 }
3773
3774 /**
3775  * Get the maximum number of modify header actions.
3776  *
3777  * @param dev
3778  *   Pointer to rte_eth_dev structure.
3779  * @param flags
3780  *   Flags bits to check if root level.
3781  *
3782  * @return
3783  *   Max number of modify header actions device can support.
3784  */
3785 static inline unsigned int
3786 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
3787                               uint64_t flags)
3788 {
3789         /*
3790          * There's no way to directly query the max capacity from FW.
3791          * The maximal value on root table should be assumed to be supported.
3792          */
3793         if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
3794                 return MLX5_MAX_MODIFY_NUM;
3795         else
3796                 return MLX5_ROOT_TBL_MODIFY_NUM;
3797 }
3798
3799 /**
3800  * Validate the meter action.
3801  *
3802  * @param[in] dev
3803  *   Pointer to rte_eth_dev structure.
3804  * @param[in] action_flags
3805  *   Bit-fields that holds the actions detected until now.
3806  * @param[in] action
3807  *   Pointer to the meter action.
3808  * @param[in] attr
3809  *   Attributes of flow that includes this action.
3810  * @param[out] error
3811  *   Pointer to error structure.
3812  *
3813  * @return
3814  *   0 on success, a negative errno value otherwise and rte_ernno is set.
3815  */
3816 static int
3817 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
3818                                 uint64_t action_flags,
3819                                 const struct rte_flow_action *action,
3820                                 const struct rte_flow_attr *attr,
3821                                 struct rte_flow_error *error)
3822 {
3823         struct mlx5_priv *priv = dev->data->dev_private;
3824         const struct rte_flow_action_meter *am = action->conf;
3825         struct mlx5_flow_meter *fm;
3826
3827         if (!am)
3828                 return rte_flow_error_set(error, EINVAL,
3829                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3830                                           "meter action conf is NULL");
3831
3832         if (action_flags & MLX5_FLOW_ACTION_METER)
3833                 return rte_flow_error_set(error, ENOTSUP,
3834                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3835                                           "meter chaining not support");
3836         if (action_flags & MLX5_FLOW_ACTION_JUMP)
3837                 return rte_flow_error_set(error, ENOTSUP,
3838                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3839                                           "meter with jump not support");
3840         if (!priv->mtr_en)
3841                 return rte_flow_error_set(error, ENOTSUP,
3842                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3843                                           NULL,
3844                                           "meter action not supported");
3845         fm = mlx5_flow_meter_find(priv, am->mtr_id);
3846         if (!fm)
3847                 return rte_flow_error_set(error, EINVAL,
3848                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3849                                           "Meter not found");
3850         if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
3851               (!fm->ingress && !attr->ingress && attr->egress) ||
3852               (!fm->egress && !attr->egress && attr->ingress))))
3853                 return rte_flow_error_set(error, EINVAL,
3854                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3855                                           "Flow attributes are either invalid "
3856                                           "or have a conflict with current "
3857                                           "meter attributes");
3858         return 0;
3859 }
3860
3861 /**
3862  * Validate the age action.
3863  *
3864  * @param[in] action_flags
3865  *   Holds the actions detected until now.
3866  * @param[in] action
3867  *   Pointer to the age action.
3868  * @param[in] dev
3869  *   Pointer to the Ethernet device structure.
3870  * @param[out] error
3871  *   Pointer to error structure.
3872  *
3873  * @return
3874  *   0 on success, a negative errno value otherwise and rte_errno is set.
3875  */
3876 static int
3877 flow_dv_validate_action_age(uint64_t action_flags,
3878                             const struct rte_flow_action *action,
3879                             struct rte_eth_dev *dev,
3880                             struct rte_flow_error *error)
3881 {
3882         struct mlx5_priv *priv = dev->data->dev_private;
3883         const struct rte_flow_action_age *age = action->conf;
3884
3885         if (!priv->config.devx || priv->counter_fallback)
3886                 return rte_flow_error_set(error, ENOTSUP,
3887                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3888                                           NULL,
3889                                           "age action not supported");
3890         if (!(action->conf))
3891                 return rte_flow_error_set(error, EINVAL,
3892                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3893                                           "configuration cannot be null");
3894         if (age->timeout >= UINT16_MAX / 2 / 10)
3895                 return rte_flow_error_set(error, ENOTSUP,
3896                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3897                                           "Max age time: 3275 seconds");
3898         if (action_flags & MLX5_FLOW_ACTION_AGE)
3899                 return rte_flow_error_set(error, EINVAL,
3900                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3901                                           "Duplicate age ctions set");
3902         return 0;
3903 }
3904
3905 /**
3906  * Validate the modify-header IPv4 DSCP actions.
3907  *
3908  * @param[in] action_flags
3909  *   Holds the actions detected until now.
3910  * @param[in] action
3911  *   Pointer to the modify action.
3912  * @param[in] item_flags
3913  *   Holds the items detected.
3914  * @param[out] error
3915  *   Pointer to error structure.
3916  *
3917  * @return
3918  *   0 on success, a negative errno value otherwise and rte_errno is set.
3919  */
3920 static int
3921 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
3922                                          const struct rte_flow_action *action,
3923                                          const uint64_t item_flags,
3924                                          struct rte_flow_error *error)
3925 {
3926         int ret = 0;
3927
3928         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3929         if (!ret) {
3930                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
3931                         return rte_flow_error_set(error, EINVAL,
3932                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3933                                                   NULL,
3934                                                   "no ipv4 item in pattern");
3935         }
3936         return ret;
3937 }
3938
3939 /**
3940  * Validate the modify-header IPv6 DSCP actions.
3941  *
3942  * @param[in] action_flags
3943  *   Holds the actions detected until now.
3944  * @param[in] action
3945  *   Pointer to the modify action.
3946  * @param[in] item_flags
3947  *   Holds the items detected.
3948  * @param[out] error
3949  *   Pointer to error structure.
3950  *
3951  * @return
3952  *   0 on success, a negative errno value otherwise and rte_errno is set.
3953  */
3954 static int
3955 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
3956                                          const struct rte_flow_action *action,
3957                                          const uint64_t item_flags,
3958                                          struct rte_flow_error *error)
3959 {
3960         int ret = 0;
3961
3962         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3963         if (!ret) {
3964                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
3965                         return rte_flow_error_set(error, EINVAL,
3966                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3967                                                   NULL,
3968                                                   "no ipv6 item in pattern");
3969         }
3970         return ret;
3971 }
3972
3973 /**
3974  * Find existing modify-header resource or create and register a new one.
3975  *
3976  * @param dev[in, out]
3977  *   Pointer to rte_eth_dev structure.
3978  * @param[in, out] resource
3979  *   Pointer to modify-header resource.
3980  * @parm[in, out] dev_flow
3981  *   Pointer to the dev_flow.
3982  * @param[out] error
3983  *   pointer to error structure.
3984  *
3985  * @return
3986  *   0 on success otherwise -errno and errno is set.
3987  */
3988 static int
3989 flow_dv_modify_hdr_resource_register
3990                         (struct rte_eth_dev *dev,
3991                          struct mlx5_flow_dv_modify_hdr_resource *resource,
3992                          struct mlx5_flow *dev_flow,
3993                          struct rte_flow_error *error)
3994 {
3995         struct mlx5_priv *priv = dev->data->dev_private;
3996         struct mlx5_dev_ctx_shared *sh = priv->sh;
3997         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
3998         struct mlx5dv_dr_domain *ns;
3999         uint32_t actions_len;
4000         int ret;
4001
4002         resource->flags = dev_flow->dv.group ? 0 :
4003                           MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4004         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
4005                                     resource->flags))
4006                 return rte_flow_error_set(error, EOVERFLOW,
4007                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4008                                           "too many modify header items");
4009         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
4010                 ns = sh->fdb_domain;
4011         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
4012                 ns = sh->tx_domain;
4013         else
4014                 ns = sh->rx_domain;
4015         /* Lookup a matching resource from cache. */
4016         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4017         LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
4018                 if (resource->ft_type == cache_resource->ft_type &&
4019                     resource->actions_num == cache_resource->actions_num &&
4020                     resource->flags == cache_resource->flags &&
4021                     !memcmp((const void *)resource->actions,
4022                             (const void *)cache_resource->actions,
4023                             actions_len)) {
4024                         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
4025                                 (void *)cache_resource,
4026                                 rte_atomic32_read(&cache_resource->refcnt));
4027                         rte_atomic32_inc(&cache_resource->refcnt);
4028                         dev_flow->handle->dvh.modify_hdr = cache_resource;
4029                         return 0;
4030                 }
4031         }
4032         /* Register new modify-header resource. */
4033         cache_resource = mlx5_malloc(MLX5_MEM_ZERO,
4034                                     sizeof(*cache_resource) + actions_len, 0,
4035                                     SOCKET_ID_ANY);
4036         if (!cache_resource)
4037                 return rte_flow_error_set(error, ENOMEM,
4038                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4039                                           "cannot allocate resource memory");
4040         *cache_resource = *resource;
4041         rte_memcpy(cache_resource->actions, resource->actions, actions_len);
4042         ret = mlx5_flow_os_create_flow_action_modify_header
4043                                         (sh->ctx, ns, cache_resource,
4044                                          actions_len, &cache_resource->action);
4045         if (ret) {
4046                 mlx5_free(cache_resource);
4047                 return rte_flow_error_set(error, ENOMEM,
4048                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4049                                           NULL, "cannot create action");
4050         }
4051         rte_atomic32_init(&cache_resource->refcnt);
4052         rte_atomic32_inc(&cache_resource->refcnt);
4053         LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
4054         dev_flow->handle->dvh.modify_hdr = cache_resource;
4055         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
4056                 (void *)cache_resource,
4057                 rte_atomic32_read(&cache_resource->refcnt));
4058         return 0;
4059 }
4060
4061 /**
4062  * Get DV flow counter by index.
4063  *
4064  * @param[in] dev
4065  *   Pointer to the Ethernet device structure.
4066  * @param[in] idx
4067  *   mlx5 flow counter index in the container.
4068  * @param[out] ppool
4069  *   mlx5 flow counter pool in the container,
4070  *
4071  * @return
4072  *   Pointer to the counter, NULL otherwise.
4073  */
4074 static struct mlx5_flow_counter *
4075 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
4076                            uint32_t idx,
4077                            struct mlx5_flow_counter_pool **ppool)
4078 {
4079         struct mlx5_priv *priv = dev->data->dev_private;
4080         struct mlx5_pools_container *cont;
4081         struct mlx5_flow_counter_pool *pool;
4082         uint32_t batch = 0, age = 0;
4083
4084         idx--;
4085         age = MLX_CNT_IS_AGE(idx);
4086         idx = age ? idx - MLX5_CNT_AGE_OFFSET : idx;
4087         if (idx >= MLX5_CNT_BATCH_OFFSET) {
4088                 idx -= MLX5_CNT_BATCH_OFFSET;
4089                 batch = 1;
4090         }
4091         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4092         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n);
4093         pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
4094         MLX5_ASSERT(pool);
4095         if (ppool)
4096                 *ppool = pool;
4097         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
4098 }
4099
4100 /**
4101  * Check the devx counter belongs to the pool.
4102  *
4103  * @param[in] pool
4104  *   Pointer to the counter pool.
4105  * @param[in] id
4106  *   The counter devx ID.
4107  *
4108  * @return
4109  *   True if counter belongs to the pool, false otherwise.
4110  */
4111 static bool
4112 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
4113 {
4114         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
4115                    MLX5_COUNTERS_PER_POOL;
4116
4117         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
4118                 return true;
4119         return false;
4120 }
4121
4122 /**
4123  * Get a pool by devx counter ID.
4124  *
4125  * @param[in] cont
4126  *   Pointer to the counter container.
4127  * @param[in] id
4128  *   The counter devx ID.
4129  *
4130  * @return
4131  *   The counter pool pointer if exists, NULL otherwise,
4132  */
4133 static struct mlx5_flow_counter_pool *
4134 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
4135 {
4136         uint32_t i;
4137
4138         /* Check last used pool. */
4139         if (cont->last_pool_idx != POOL_IDX_INVALID &&
4140             flow_dv_is_counter_in_pool(cont->pools[cont->last_pool_idx], id))
4141                 return cont->pools[cont->last_pool_idx];
4142         /* ID out of range means no suitable pool in the container. */
4143         if (id > cont->max_id || id < cont->min_id)
4144                 return NULL;
4145         /*
4146          * Find the pool from the end of the container, since mostly counter
4147          * ID is sequence increasing, and the last pool should be the needed
4148          * one.
4149          */
4150         i = rte_atomic16_read(&cont->n_valid);
4151         while (i--) {
4152                 struct mlx5_flow_counter_pool *pool = cont->pools[i];
4153
4154                 if (flow_dv_is_counter_in_pool(pool, id))
4155                         return pool;
4156         }
4157         return NULL;
4158 }
4159
4160 /**
4161  * Allocate a new memory for the counter values wrapped by all the needed
4162  * management.
4163  *
4164  * @param[in] dev
4165  *   Pointer to the Ethernet device structure.
4166  * @param[in] raws_n
4167  *   The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
4168  *
4169  * @return
4170  *   The new memory management pointer on success, otherwise NULL and rte_errno
4171  *   is set.
4172  */
4173 static struct mlx5_counter_stats_mem_mng *
4174 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
4175 {
4176         struct mlx5_priv *priv = dev->data->dev_private;
4177         struct mlx5_dev_ctx_shared *sh = priv->sh;
4178         struct mlx5_devx_mkey_attr mkey_attr;
4179         struct mlx5_counter_stats_mem_mng *mem_mng;
4180         volatile struct flow_counter_stats *raw_data;
4181         int size = (sizeof(struct flow_counter_stats) *
4182                         MLX5_COUNTERS_PER_POOL +
4183                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
4184                         sizeof(struct mlx5_counter_stats_mem_mng);
4185         uint8_t *mem = mlx5_malloc(MLX5_MEM_ZERO, size, sysconf(_SC_PAGESIZE),
4186                                   SOCKET_ID_ANY);
4187         int i;
4188
4189         if (!mem) {
4190                 rte_errno = ENOMEM;
4191                 return NULL;
4192         }
4193         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
4194         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
4195         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
4196                                                  IBV_ACCESS_LOCAL_WRITE);
4197         if (!mem_mng->umem) {
4198                 rte_errno = errno;
4199                 mlx5_free(mem);
4200                 return NULL;
4201         }
4202         mkey_attr.addr = (uintptr_t)mem;
4203         mkey_attr.size = size;
4204         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
4205         mkey_attr.pd = sh->pdn;
4206         mkey_attr.log_entity_size = 0;
4207         mkey_attr.pg_access = 0;
4208         mkey_attr.klm_array = NULL;
4209         mkey_attr.klm_num = 0;
4210         if (priv->config.hca_attr.relaxed_ordering_write &&
4211                 priv->config.hca_attr.relaxed_ordering_read  &&
4212                 !haswell_broadwell_cpu)
4213                 mkey_attr.relaxed_ordering = 1;
4214         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
4215         if (!mem_mng->dm) {
4216                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
4217                 rte_errno = errno;
4218                 mlx5_free(mem);
4219                 return NULL;
4220         }
4221         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
4222         raw_data = (volatile struct flow_counter_stats *)mem;
4223         for (i = 0; i < raws_n; ++i) {
4224                 mem_mng->raws[i].mem_mng = mem_mng;
4225                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
4226         }
4227         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
4228         return mem_mng;
4229 }
4230
4231 /**
4232  * Resize a counter container.
4233  *
4234  * @param[in] dev
4235  *   Pointer to the Ethernet device structure.
4236  * @param[in] batch
4237  *   Whether the pool is for counter that was allocated by batch command.
4238  * @param[in] age
4239  *   Whether the pool is for Aging counter.
4240  *
4241  * @return
4242  *   0 on success, otherwise negative errno value and rte_errno is set.
4243  */
4244 static int
4245 flow_dv_container_resize(struct rte_eth_dev *dev,
4246                                 uint32_t batch, uint32_t age)
4247 {
4248         struct mlx5_priv *priv = dev->data->dev_private;
4249         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4250                                                                age);
4251         struct mlx5_counter_stats_mem_mng *mem_mng = NULL;
4252         void *old_pools = cont->pools;
4253         uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
4254         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
4255         void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
4256
4257         if (!pools) {
4258                 rte_errno = ENOMEM;
4259                 return -ENOMEM;
4260         }
4261         if (old_pools)
4262                 memcpy(pools, old_pools, cont->n *
4263                                        sizeof(struct mlx5_flow_counter_pool *));
4264         /*
4265          * Fallback mode query the counter directly, no background query
4266          * resources are needed.
4267          */
4268         if (!priv->counter_fallback) {
4269                 int i;
4270
4271                 mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
4272                           MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
4273                 if (!mem_mng) {
4274                         mlx5_free(pools);
4275                         return -ENOMEM;
4276                 }
4277                 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
4278                         LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
4279                                          mem_mng->raws +
4280                                          MLX5_CNT_CONTAINER_RESIZE +
4281                                          i, next);
4282         }
4283         rte_spinlock_lock(&cont->resize_sl);
4284         cont->n = resize;
4285         cont->mem_mng = mem_mng;
4286         cont->pools = pools;
4287         rte_spinlock_unlock(&cont->resize_sl);
4288         if (old_pools)
4289                 mlx5_free(old_pools);
4290         return 0;
4291 }
4292
4293 /**
4294  * Query a devx flow counter.
4295  *
4296  * @param[in] dev
4297  *   Pointer to the Ethernet device structure.
4298  * @param[in] cnt
4299  *   Index to the flow counter.
4300  * @param[out] pkts
4301  *   The statistics value of packets.
4302  * @param[out] bytes
4303  *   The statistics value of bytes.
4304  *
4305  * @return
4306  *   0 on success, otherwise a negative errno value and rte_errno is set.
4307  */
4308 static inline int
4309 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
4310                      uint64_t *bytes)
4311 {
4312         struct mlx5_priv *priv = dev->data->dev_private;
4313         struct mlx5_flow_counter_pool *pool = NULL;
4314         struct mlx5_flow_counter *cnt;
4315         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4316         int offset;
4317
4318         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4319         MLX5_ASSERT(pool);
4320         if (counter < MLX5_CNT_BATCH_OFFSET) {
4321                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4322                 if (priv->counter_fallback)
4323                         return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
4324                                         0, pkts, bytes, 0, NULL, NULL, 0);
4325         }
4326
4327         rte_spinlock_lock(&pool->sl);
4328         /*
4329          * The single counters allocation may allocate smaller ID than the
4330          * current allocated in parallel to the host reading.
4331          * In this case the new counter values must be reported as 0.
4332          */
4333         if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) {
4334                 *pkts = 0;
4335                 *bytes = 0;
4336         } else {
4337                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
4338                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
4339                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
4340         }
4341         rte_spinlock_unlock(&pool->sl);
4342         return 0;
4343 }
4344
4345 /**
4346  * Create and initialize a new counter pool.
4347  *
4348  * @param[in] dev
4349  *   Pointer to the Ethernet device structure.
4350  * @param[out] dcs
4351  *   The devX counter handle.
4352  * @param[in] batch
4353  *   Whether the pool is for counter that was allocated by batch command.
4354  * @param[in] age
4355  *   Whether the pool is for counter that was allocated for aging.
4356  * @param[in/out] cont_cur
4357  *   Pointer to the container pointer, it will be update in pool resize.
4358  *
4359  * @return
4360  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
4361  */
4362 static struct mlx5_flow_counter_pool *
4363 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
4364                     uint32_t batch, uint32_t age)
4365 {
4366         struct mlx5_priv *priv = dev->data->dev_private;
4367         struct mlx5_flow_counter_pool *pool;
4368         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4369                                                                age);
4370         int16_t n_valid = rte_atomic16_read(&cont->n_valid);
4371         uint32_t size = sizeof(*pool);
4372
4373         if (cont->n == n_valid && flow_dv_container_resize(dev, batch, age))
4374                 return NULL;
4375         size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
4376         size += (batch ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
4377         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE);
4378         pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
4379         if (!pool) {
4380                 rte_errno = ENOMEM;
4381                 return NULL;
4382         }
4383         pool->min_dcs = dcs;
4384         if (!priv->counter_fallback)
4385                 pool->raw = cont->mem_mng->raws + n_valid %
4386                                                       MLX5_CNT_CONTAINER_RESIZE;
4387         pool->raw_hw = NULL;
4388         pool->type = 0;
4389         pool->type |= (batch ? 0 :  CNT_POOL_TYPE_EXT);
4390         pool->type |= (!age ? 0 :  CNT_POOL_TYPE_AGE);
4391         pool->query_gen = 0;
4392         rte_spinlock_init(&pool->sl);
4393         TAILQ_INIT(&pool->counters[0]);
4394         TAILQ_INIT(&pool->counters[1]);
4395         TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
4396         pool->index = n_valid;
4397         cont->pools[n_valid] = pool;
4398         if (!batch) {
4399                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
4400
4401                 if (base < cont->min_id)
4402                         cont->min_id = base;
4403                 if (base > cont->max_id)
4404                         cont->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
4405                 cont->last_pool_idx = pool->index;
4406         }
4407         /* Pool initialization must be updated before host thread access. */
4408         rte_cio_wmb();
4409         rte_atomic16_add(&cont->n_valid, 1);
4410         return pool;
4411 }
4412
4413 /**
4414  * Update the minimum dcs-id for aged or no-aged counter pool.
4415  *
4416  * @param[in] dev
4417  *   Pointer to the Ethernet device structure.
4418  * @param[in] pool
4419  *   Current counter pool.
4420  * @param[in] batch
4421  *   Whether the pool is for counter that was allocated by batch command.
4422  * @param[in] age
4423  *   Whether the counter is for aging.
4424  */
4425 static void
4426 flow_dv_counter_update_min_dcs(struct rte_eth_dev *dev,
4427                         struct mlx5_flow_counter_pool *pool,
4428                         uint32_t batch, uint32_t age)
4429 {
4430         struct mlx5_priv *priv = dev->data->dev_private;
4431         struct mlx5_flow_counter_pool *other;
4432         struct mlx5_pools_container *cont;
4433
4434         cont = MLX5_CNT_CONTAINER(priv->sh, batch, (age ^ 0x1));
4435         other = flow_dv_find_pool_by_id(cont, pool->min_dcs->id);
4436         if (!other)
4437                 return;
4438         if (pool->min_dcs->id < other->min_dcs->id) {
4439                 rte_atomic64_set(&other->a64_dcs,
4440                         rte_atomic64_read(&pool->a64_dcs));
4441         } else {
4442                 rte_atomic64_set(&pool->a64_dcs,
4443                         rte_atomic64_read(&other->a64_dcs));
4444         }
4445 }
4446 /**
4447  * Prepare a new counter and/or a new counter pool.
4448  *
4449  * @param[in] dev
4450  *   Pointer to the Ethernet device structure.
4451  * @param[out] cnt_free
4452  *   Where to put the pointer of a new counter.
4453  * @param[in] batch
4454  *   Whether the pool is for counter that was allocated by batch command.
4455  * @param[in] age
4456  *   Whether the pool is for counter that was allocated for aging.
4457  *
4458  * @return
4459  *   The counter pool pointer and @p cnt_free is set on success,
4460  *   NULL otherwise and rte_errno is set.
4461  */
4462 static struct mlx5_flow_counter_pool *
4463 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
4464                              struct mlx5_flow_counter **cnt_free,
4465                              uint32_t batch, uint32_t age)
4466 {
4467         struct mlx5_priv *priv = dev->data->dev_private;
4468         struct mlx5_pools_container *cont;
4469         struct mlx5_flow_counter_pool *pool;
4470         struct mlx5_counters tmp_tq;
4471         struct mlx5_devx_obj *dcs = NULL;
4472         struct mlx5_flow_counter *cnt;
4473         uint32_t i;
4474
4475         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4476         if (!batch) {
4477                 /* bulk_bitmap must be 0 for single counter allocation. */
4478                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
4479                 if (!dcs)
4480                         return NULL;
4481                 pool = flow_dv_find_pool_by_id(cont, dcs->id);
4482                 if (!pool) {
4483                         pool = flow_dv_pool_create(dev, dcs, batch, age);
4484                         if (!pool) {
4485                                 mlx5_devx_cmd_destroy(dcs);
4486                                 return NULL;
4487                         }
4488                 } else if (dcs->id < pool->min_dcs->id) {
4489                         rte_atomic64_set(&pool->a64_dcs,
4490                                          (int64_t)(uintptr_t)dcs);
4491                 }
4492                 flow_dv_counter_update_min_dcs(dev,
4493                                                 pool, batch, age);
4494                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
4495                 cnt = MLX5_POOL_GET_CNT(pool, i);
4496                 cnt->pool = pool;
4497                 MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs;
4498                 *cnt_free = cnt;
4499                 return pool;
4500         }
4501         /* bulk_bitmap is in 128 counters units. */
4502         if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
4503                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
4504         if (!dcs) {
4505                 rte_errno = ENODATA;
4506                 return NULL;
4507         }
4508         pool = flow_dv_pool_create(dev, dcs, batch, age);
4509         if (!pool) {
4510                 mlx5_devx_cmd_destroy(dcs);
4511                 return NULL;
4512         }
4513         TAILQ_INIT(&tmp_tq);
4514         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
4515                 cnt = MLX5_POOL_GET_CNT(pool, i);
4516                 cnt->pool = pool;
4517                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
4518         }
4519         rte_spinlock_lock(&cont->csl);
4520         TAILQ_CONCAT(&cont->counters, &tmp_tq, next);
4521         rte_spinlock_unlock(&cont->csl);
4522         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
4523         (*cnt_free)->pool = pool;
4524         return pool;
4525 }
4526
4527 /**
4528  * Search for existed shared counter.
4529  *
4530  * @param[in] dev
4531  *   Pointer to the Ethernet device structure.
4532  * @param[in] id
4533  *   The shared counter ID to search.
4534  * @param[out] ppool
4535  *   mlx5 flow counter pool in the container,
4536  *
4537  * @return
4538  *   NULL if not existed, otherwise pointer to the shared extend counter.
4539  */
4540 static struct mlx5_flow_counter_ext *
4541 flow_dv_counter_shared_search(struct rte_eth_dev *dev, uint32_t id,
4542                               struct mlx5_flow_counter_pool **ppool)
4543 {
4544         struct mlx5_priv *priv = dev->data->dev_private;
4545         union mlx5_l3t_data data;
4546         uint32_t cnt_idx;
4547
4548         if (mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) || !data.dword)
4549                 return NULL;
4550         cnt_idx = data.dword;
4551         /*
4552          * Shared counters don't have age info. The counter extend is after
4553          * the counter datat structure.
4554          */
4555         return (struct mlx5_flow_counter_ext *)
4556                ((flow_dv_counter_get_by_idx(dev, cnt_idx, ppool)) + 1);
4557 }
4558
4559 /**
4560  * Allocate a flow counter.
4561  *
4562  * @param[in] dev
4563  *   Pointer to the Ethernet device structure.
4564  * @param[in] shared
4565  *   Indicate if this counter is shared with other flows.
4566  * @param[in] id
4567  *   Counter identifier.
4568  * @param[in] group
4569  *   Counter flow group.
4570  * @param[in] age
4571  *   Whether the counter was allocated for aging.
4572  *
4573  * @return
4574  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
4575  */
4576 static uint32_t
4577 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
4578                       uint16_t group, uint32_t age)
4579 {
4580         struct mlx5_priv *priv = dev->data->dev_private;
4581         struct mlx5_flow_counter_pool *pool = NULL;
4582         struct mlx5_flow_counter *cnt_free = NULL;
4583         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4584         /*
4585          * Currently group 0 flow counter cannot be assigned to a flow if it is
4586          * not the first one in the batch counter allocation, so it is better
4587          * to allocate counters one by one for these flows in a separate
4588          * container.
4589          * A counter can be shared between different groups so need to take
4590          * shared counters from the single container.
4591          */
4592         uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0;
4593         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4594                                                                age);
4595         uint32_t cnt_idx;
4596
4597         if (!priv->config.devx) {
4598                 rte_errno = ENOTSUP;
4599                 return 0;
4600         }
4601         if (shared) {
4602                 cnt_ext = flow_dv_counter_shared_search(dev, id, &pool);
4603                 if (cnt_ext) {
4604                         if (cnt_ext->ref_cnt + 1 == 0) {
4605                                 rte_errno = E2BIG;
4606                                 return 0;
4607                         }
4608                         cnt_ext->ref_cnt++;
4609                         cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL +
4610                                   (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL)
4611                                   + 1;
4612                         return cnt_idx;
4613                 }
4614         }
4615         /* Get free counters from container. */
4616         rte_spinlock_lock(&cont->csl);
4617         cnt_free = TAILQ_FIRST(&cont->counters);
4618         if (cnt_free)
4619                 TAILQ_REMOVE(&cont->counters, cnt_free, next);
4620         rte_spinlock_unlock(&cont->csl);
4621         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free,
4622                                                        batch, age))
4623                 goto err;
4624         pool = cnt_free->pool;
4625         if (!batch)
4626                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free);
4627         /* Create a DV counter action only in the first time usage. */
4628         if (!cnt_free->action) {
4629                 uint16_t offset;
4630                 struct mlx5_devx_obj *dcs;
4631                 int ret;
4632
4633                 if (batch) {
4634                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
4635                         dcs = pool->min_dcs;
4636                 } else {
4637                         offset = 0;
4638                         dcs = cnt_ext->dcs;
4639                 }
4640                 ret = mlx5_flow_os_create_flow_action_count(dcs->obj, offset,
4641                                                             &cnt_free->action);
4642                 if (ret) {
4643                         rte_errno = errno;
4644                         goto err;
4645                 }
4646         }
4647         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
4648                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
4649         cnt_idx += batch * MLX5_CNT_BATCH_OFFSET;
4650         cnt_idx += age * MLX5_CNT_AGE_OFFSET;
4651         /* Update the counter reset values. */
4652         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
4653                                  &cnt_free->bytes))
4654                 goto err;
4655         if (cnt_ext) {
4656                 cnt_ext->shared = shared;
4657                 cnt_ext->ref_cnt = 1;
4658                 cnt_ext->id = id;
4659                 if (shared) {
4660                         union mlx5_l3t_data data;
4661
4662                         data.dword = cnt_idx;
4663                         if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
4664                                 return 0;
4665                 }
4666         }
4667         if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on)
4668                 /* Start the asynchronous batch query by the host thread. */
4669                 mlx5_set_query_alarm(priv->sh);
4670         return cnt_idx;
4671 err:
4672         if (cnt_free) {
4673                 cnt_free->pool = pool;
4674                 rte_spinlock_lock(&cont->csl);
4675                 TAILQ_INSERT_TAIL(&cont->counters, cnt_free, next);
4676                 rte_spinlock_unlock(&cont->csl);
4677         }
4678         return 0;
4679 }
4680
4681 /**
4682  * Get age param from counter index.
4683  *
4684  * @param[in] dev
4685  *   Pointer to the Ethernet device structure.
4686  * @param[in] counter
4687  *   Index to the counter handler.
4688  *
4689  * @return
4690  *   The aging parameter specified for the counter index.
4691  */
4692 static struct mlx5_age_param*
4693 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
4694                                 uint32_t counter)
4695 {
4696         struct mlx5_flow_counter *cnt;
4697         struct mlx5_flow_counter_pool *pool = NULL;
4698
4699         flow_dv_counter_get_by_idx(dev, counter, &pool);
4700         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
4701         cnt = MLX5_POOL_GET_CNT(pool, counter);
4702         return MLX5_CNT_TO_AGE(cnt);
4703 }
4704
4705 /**
4706  * Remove a flow counter from aged counter list.
4707  *
4708  * @param[in] dev
4709  *   Pointer to the Ethernet device structure.
4710  * @param[in] counter
4711  *   Index to the counter handler.
4712  * @param[in] cnt
4713  *   Pointer to the counter handler.
4714  */
4715 static void
4716 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
4717                                 uint32_t counter, struct mlx5_flow_counter *cnt)
4718 {
4719         struct mlx5_age_info *age_info;
4720         struct mlx5_age_param *age_param;
4721         struct mlx5_priv *priv = dev->data->dev_private;
4722
4723         age_info = GET_PORT_AGE_INFO(priv);
4724         age_param = flow_dv_counter_idx_get_age(dev, counter);
4725         if (rte_atomic16_cmpset((volatile uint16_t *)
4726                         &age_param->state,
4727                         AGE_CANDIDATE, AGE_FREE)
4728                         != AGE_CANDIDATE) {
4729                 /**
4730                  * We need the lock even it is age timeout,
4731                  * since counter may still in process.
4732                  */
4733                 rte_spinlock_lock(&age_info->aged_sl);
4734                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
4735                 rte_spinlock_unlock(&age_info->aged_sl);
4736         }
4737         rte_atomic16_set(&age_param->state, AGE_FREE);
4738 }
4739 /**
4740  * Release a flow counter.
4741  *
4742  * @param[in] dev
4743  *   Pointer to the Ethernet device structure.
4744  * @param[in] counter
4745  *   Index to the counter handler.
4746  */
4747 static void
4748 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
4749 {
4750         struct mlx5_priv *priv = dev->data->dev_private;
4751         struct mlx5_flow_counter_pool *pool = NULL;
4752         struct mlx5_flow_counter *cnt;
4753         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4754
4755         if (!counter)
4756                 return;
4757         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4758         MLX5_ASSERT(pool);
4759         if (counter < MLX5_CNT_BATCH_OFFSET) {
4760                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4761                 if (cnt_ext) {
4762                         if (--cnt_ext->ref_cnt)
4763                                 return;
4764                         if (cnt_ext->shared)
4765                                 mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
4766                                                      cnt_ext->id);
4767                 }
4768         }
4769         if (IS_AGE_POOL(pool))
4770                 flow_dv_counter_remove_from_age(dev, counter, cnt);
4771         cnt->pool = pool;
4772         /*
4773          * Put the counter back to list to be updated in none fallback mode.
4774          * Currently, we are using two list alternately, while one is in query,
4775          * add the freed counter to the other list based on the pool query_gen
4776          * value. After query finishes, add counter the list to the global
4777          * container counter list. The list changes while query starts. In
4778          * this case, lock will not be needed as query callback and release
4779          * function both operate with the different list.
4780          *
4781          */
4782         if (!priv->counter_fallback)
4783                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
4784         else
4785                 TAILQ_INSERT_TAIL(&((MLX5_CNT_CONTAINER
4786                                   (priv->sh, 0, 0))->counters),
4787                                   cnt, next);
4788 }
4789
4790 /**
4791  * Verify the @p attributes will be correctly understood by the NIC and store
4792  * them in the @p flow if everything is correct.
4793  *
4794  * @param[in] dev
4795  *   Pointer to dev struct.
4796  * @param[in] attributes
4797  *   Pointer to flow attributes
4798  * @param[in] external
4799  *   This flow rule is created by request external to PMD.
4800  * @param[out] error
4801  *   Pointer to error structure.
4802  *
4803  * @return
4804  *   - 0 on success and non root table.
4805  *   - 1 on success and root table.
4806  *   - a negative errno value otherwise and rte_errno is set.
4807  */
4808 static int
4809 flow_dv_validate_attributes(struct rte_eth_dev *dev,
4810                             const struct rte_flow_attr *attributes,
4811                             bool external __rte_unused,
4812                             struct rte_flow_error *error)
4813 {
4814         struct mlx5_priv *priv = dev->data->dev_private;
4815         uint32_t priority_max = priv->config.flow_prio - 1;
4816         int ret = 0;
4817
4818 #ifndef HAVE_MLX5DV_DR
4819         if (attributes->group)
4820                 return rte_flow_error_set(error, ENOTSUP,
4821                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
4822                                           NULL,
4823                                           "groups are not supported");
4824 #else
4825         uint32_t table = 0;
4826
4827         ret = mlx5_flow_group_to_table(attributes, external,
4828                                        attributes->group, !!priv->fdb_def_rule,
4829                                        &table, error);
4830         if (ret)
4831                 return ret;
4832         if (!table)
4833                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4834 #endif
4835         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
4836             attributes->priority >= priority_max)
4837                 return rte_flow_error_set(error, ENOTSUP,
4838                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
4839                                           NULL,
4840                                           "priority out of range");
4841         if (attributes->transfer) {
4842                 if (!priv->config.dv_esw_en)
4843                         return rte_flow_error_set
4844                                 (error, ENOTSUP,
4845                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4846                                  "E-Switch dr is not supported");
4847                 if (!(priv->representor || priv->master))
4848                         return rte_flow_error_set
4849                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4850                                  NULL, "E-Switch configuration can only be"
4851                                  " done by a master or a representor device");
4852                 if (attributes->egress)
4853                         return rte_flow_error_set
4854                                 (error, ENOTSUP,
4855                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
4856                                  "egress is not supported");
4857         }
4858         if (!(attributes->egress ^ attributes->ingress))
4859                 return rte_flow_error_set(error, ENOTSUP,
4860                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
4861                                           "must specify exactly one of "
4862                                           "ingress or egress");
4863         return ret;
4864 }
4865
4866 /**
4867  * Internal validation function. For validating both actions and items.
4868  *
4869  * @param[in] dev
4870  *   Pointer to the rte_eth_dev structure.
4871  * @param[in] attr
4872  *   Pointer to the flow attributes.
4873  * @param[in] items
4874  *   Pointer to the list of items.
4875  * @param[in] actions
4876  *   Pointer to the list of actions.
4877  * @param[in] external
4878  *   This flow rule is created by request external to PMD.
4879  * @param[in] hairpin
4880  *   Number of hairpin TX actions, 0 means classic flow.
4881  * @param[out] error
4882  *   Pointer to the error structure.
4883  *
4884  * @return
4885  *   0 on success, a negative errno value otherwise and rte_errno is set.
4886  */
4887 static int
4888 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
4889                  const struct rte_flow_item items[],
4890                  const struct rte_flow_action actions[],
4891                  bool external, int hairpin, struct rte_flow_error *error)
4892 {
4893         int ret;
4894         uint64_t action_flags = 0;
4895         uint64_t item_flags = 0;
4896         uint64_t last_item = 0;
4897         uint8_t next_protocol = 0xff;
4898         uint16_t ether_type = 0;
4899         int actions_n = 0;
4900         uint8_t item_ipv6_proto = 0;
4901         const struct rte_flow_item *gre_item = NULL;
4902         const struct rte_flow_action_raw_decap *decap;
4903         const struct rte_flow_action_raw_encap *encap;
4904         const struct rte_flow_action_rss *rss;
4905         const struct rte_flow_item_tcp nic_tcp_mask = {
4906                 .hdr = {
4907                         .tcp_flags = 0xFF,
4908                         .src_port = RTE_BE16(UINT16_MAX),
4909                         .dst_port = RTE_BE16(UINT16_MAX),
4910                 }
4911         };
4912         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
4913                 .hdr = {
4914                         .src_addr = RTE_BE32(0xffffffff),
4915                         .dst_addr = RTE_BE32(0xffffffff),
4916                         .type_of_service = 0xff,
4917                         .next_proto_id = 0xff,
4918                         .time_to_live = 0xff,
4919                 },
4920         };
4921         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
4922                 .hdr = {
4923                         .src_addr =
4924                         "\xff\xff\xff\xff\xff\xff\xff\xff"
4925                         "\xff\xff\xff\xff\xff\xff\xff\xff",
4926                         .dst_addr =
4927                         "\xff\xff\xff\xff\xff\xff\xff\xff"
4928                         "\xff\xff\xff\xff\xff\xff\xff\xff",
4929                         .vtc_flow = RTE_BE32(0xffffffff),
4930                         .proto = 0xff,
4931                         .hop_limits = 0xff,
4932                 },
4933         };
4934         const struct rte_flow_item_ecpri nic_ecpri_mask = {
4935                 .hdr = {
4936                         .common = {
4937                                 .u32 =
4938                                 RTE_BE32(((const struct rte_ecpri_common_hdr) {
4939                                         .type = 0xFF,
4940                                         }).u32),
4941                         },
4942                         .dummy[0] = 0xffffffff,
4943                 },
4944         };
4945         struct mlx5_priv *priv = dev->data->dev_private;
4946         struct mlx5_dev_config *dev_conf = &priv->config;
4947         uint16_t queue_index = 0xFFFF;
4948         const struct rte_flow_item_vlan *vlan_m = NULL;
4949         int16_t rw_act_num = 0;
4950         uint64_t is_root;
4951
4952         if (items == NULL)
4953                 return -1;
4954         ret = flow_dv_validate_attributes(dev, attr, external, error);
4955         if (ret < 0)
4956                 return ret;
4957         is_root = (uint64_t)ret;
4958         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4959                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
4960                 int type = items->type;
4961
4962                 if (!mlx5_flow_os_item_supported(type))
4963                         return rte_flow_error_set(error, ENOTSUP,
4964                                                   RTE_FLOW_ERROR_TYPE_ITEM,
4965                                                   NULL, "item not supported");
4966                 switch (type) {
4967                 case RTE_FLOW_ITEM_TYPE_VOID:
4968                         break;
4969                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4970                         ret = flow_dv_validate_item_port_id
4971                                         (dev, items, attr, item_flags, error);
4972                         if (ret < 0)
4973                                 return ret;
4974                         last_item = MLX5_FLOW_ITEM_PORT_ID;
4975                         break;
4976                 case RTE_FLOW_ITEM_TYPE_ETH:
4977                         ret = mlx5_flow_validate_item_eth(items, item_flags,
4978                                                           error);
4979                         if (ret < 0)
4980                                 return ret;
4981                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
4982                                              MLX5_FLOW_LAYER_OUTER_L2;
4983                         if (items->mask != NULL && items->spec != NULL) {
4984                                 ether_type =
4985                                         ((const struct rte_flow_item_eth *)
4986                                          items->spec)->type;
4987                                 ether_type &=
4988                                         ((const struct rte_flow_item_eth *)
4989                                          items->mask)->type;
4990                                 ether_type = rte_be_to_cpu_16(ether_type);
4991                         } else {
4992                                 ether_type = 0;
4993                         }
4994                         break;
4995                 case RTE_FLOW_ITEM_TYPE_VLAN:
4996                         ret = flow_dv_validate_item_vlan(items, item_flags,
4997                                                          dev, error);
4998                         if (ret < 0)
4999                                 return ret;
5000                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
5001                                              MLX5_FLOW_LAYER_OUTER_VLAN;
5002                         if (items->mask != NULL && items->spec != NULL) {
5003                                 ether_type =
5004                                         ((const struct rte_flow_item_vlan *)
5005                                          items->spec)->inner_type;
5006                                 ether_type &=
5007                                         ((const struct rte_flow_item_vlan *)
5008                                          items->mask)->inner_type;
5009                                 ether_type = rte_be_to_cpu_16(ether_type);
5010                         } else {
5011                                 ether_type = 0;
5012                         }
5013                         /* Store outer VLAN mask for of_push_vlan action. */
5014                         if (!tunnel)
5015                                 vlan_m = items->mask;
5016                         break;
5017                 case RTE_FLOW_ITEM_TYPE_IPV4:
5018                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5019                                                   &item_flags, &tunnel);
5020                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
5021                                                            last_item,
5022                                                            ether_type,
5023                                                            &nic_ipv4_mask,
5024                                                            error);
5025                         if (ret < 0)
5026                                 return ret;
5027                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5028                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5029                         if (items->mask != NULL &&
5030                             ((const struct rte_flow_item_ipv4 *)
5031                              items->mask)->hdr.next_proto_id) {
5032                                 next_protocol =
5033                                         ((const struct rte_flow_item_ipv4 *)
5034                                          (items->spec))->hdr.next_proto_id;
5035                                 next_protocol &=
5036                                         ((const struct rte_flow_item_ipv4 *)
5037                                          (items->mask))->hdr.next_proto_id;
5038                         } else {
5039                                 /* Reset for inner layer. */
5040                                 next_protocol = 0xff;
5041                         }
5042                         break;
5043                 case RTE_FLOW_ITEM_TYPE_IPV6:
5044                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5045                                                   &item_flags, &tunnel);
5046                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
5047                                                            last_item,
5048                                                            ether_type,
5049                                                            &nic_ipv6_mask,
5050                                                            error);
5051                         if (ret < 0)
5052                                 return ret;
5053                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5054                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5055                         if (items->mask != NULL &&
5056                             ((const struct rte_flow_item_ipv6 *)
5057                              items->mask)->hdr.proto) {
5058                                 item_ipv6_proto =
5059                                         ((const struct rte_flow_item_ipv6 *)
5060                                          items->spec)->hdr.proto;
5061                                 next_protocol =
5062                                         ((const struct rte_flow_item_ipv6 *)
5063                                          items->spec)->hdr.proto;
5064                                 next_protocol &=
5065                                         ((const struct rte_flow_item_ipv6 *)
5066                                          items->mask)->hdr.proto;
5067                         } else {
5068                                 /* Reset for inner layer. */
5069                                 next_protocol = 0xff;
5070                         }
5071                         break;
5072                 case RTE_FLOW_ITEM_TYPE_TCP:
5073                         ret = mlx5_flow_validate_item_tcp
5074                                                 (items, item_flags,
5075                                                  next_protocol,
5076                                                  &nic_tcp_mask,
5077                                                  error);
5078                         if (ret < 0)
5079                                 return ret;
5080                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5081                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5082                         break;
5083                 case RTE_FLOW_ITEM_TYPE_UDP:
5084                         ret = mlx5_flow_validate_item_udp(items, item_flags,
5085                                                           next_protocol,
5086                                                           error);
5087                         if (ret < 0)
5088                                 return ret;
5089                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5090                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5091                         break;
5092                 case RTE_FLOW_ITEM_TYPE_GRE:
5093                         ret = mlx5_flow_validate_item_gre(items, item_flags,
5094                                                           next_protocol, error);
5095                         if (ret < 0)
5096                                 return ret;
5097                         gre_item = items;
5098                         last_item = MLX5_FLOW_LAYER_GRE;
5099                         break;
5100                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5101                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
5102                                                             next_protocol,
5103                                                             error);
5104                         if (ret < 0)
5105                                 return ret;
5106                         last_item = MLX5_FLOW_LAYER_NVGRE;
5107                         break;
5108                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5109                         ret = mlx5_flow_validate_item_gre_key
5110                                 (items, item_flags, gre_item, error);
5111                         if (ret < 0)
5112                                 return ret;
5113                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5114                         break;
5115                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5116                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
5117                                                             error);
5118                         if (ret < 0)
5119                                 return ret;
5120                         last_item = MLX5_FLOW_LAYER_VXLAN;
5121                         break;
5122                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5123                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
5124                                                                 item_flags, dev,
5125                                                                 error);
5126                         if (ret < 0)
5127                                 return ret;
5128                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5129                         break;
5130                 case RTE_FLOW_ITEM_TYPE_GENEVE:
5131                         ret = mlx5_flow_validate_item_geneve(items,
5132                                                              item_flags, dev,
5133                                                              error);
5134                         if (ret < 0)
5135                                 return ret;
5136                         last_item = MLX5_FLOW_LAYER_GENEVE;
5137                         break;
5138                 case RTE_FLOW_ITEM_TYPE_MPLS:
5139                         ret = mlx5_flow_validate_item_mpls(dev, items,
5140                                                            item_flags,
5141                                                            last_item, error);
5142                         if (ret < 0)
5143                                 return ret;
5144                         last_item = MLX5_FLOW_LAYER_MPLS;
5145                         break;
5146
5147                 case RTE_FLOW_ITEM_TYPE_MARK:
5148                         ret = flow_dv_validate_item_mark(dev, items, attr,
5149                                                          error);
5150                         if (ret < 0)
5151                                 return ret;
5152                         last_item = MLX5_FLOW_ITEM_MARK;
5153                         break;
5154                 case RTE_FLOW_ITEM_TYPE_META:
5155                         ret = flow_dv_validate_item_meta(dev, items, attr,
5156                                                          error);
5157                         if (ret < 0)
5158                                 return ret;
5159                         last_item = MLX5_FLOW_ITEM_METADATA;
5160                         break;
5161                 case RTE_FLOW_ITEM_TYPE_ICMP:
5162                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
5163                                                            next_protocol,
5164                                                            error);
5165                         if (ret < 0)
5166                                 return ret;
5167                         last_item = MLX5_FLOW_LAYER_ICMP;
5168                         break;
5169                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5170                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
5171                                                             next_protocol,
5172                                                             error);
5173                         if (ret < 0)
5174                                 return ret;
5175                         item_ipv6_proto = IPPROTO_ICMPV6;
5176                         last_item = MLX5_FLOW_LAYER_ICMP6;
5177                         break;
5178                 case RTE_FLOW_ITEM_TYPE_TAG:
5179                         ret = flow_dv_validate_item_tag(dev, items,
5180                                                         attr, error);
5181                         if (ret < 0)
5182                                 return ret;
5183                         last_item = MLX5_FLOW_ITEM_TAG;
5184                         break;
5185                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
5186                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
5187                         break;
5188                 case RTE_FLOW_ITEM_TYPE_GTP:
5189                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
5190                                                         error);
5191                         if (ret < 0)
5192                                 return ret;
5193                         last_item = MLX5_FLOW_LAYER_GTP;
5194                         break;
5195                 case RTE_FLOW_ITEM_TYPE_ECPRI:
5196                         /* Capacity will be checked in the translate stage. */
5197                         ret = mlx5_flow_validate_item_ecpri(items, item_flags,
5198                                                             last_item,
5199                                                             ether_type,
5200                                                             &nic_ecpri_mask,
5201                                                             error);
5202                         if (ret < 0)
5203                                 return ret;
5204                         last_item = MLX5_FLOW_LAYER_ECPRI;
5205                         break;
5206                 default:
5207                         return rte_flow_error_set(error, ENOTSUP,
5208                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5209                                                   NULL, "item not supported");
5210                 }
5211                 item_flags |= last_item;
5212         }
5213         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5214                 int type = actions->type;
5215
5216                 if (!mlx5_flow_os_action_supported(type))
5217                         return rte_flow_error_set(error, ENOTSUP,
5218                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5219                                                   actions,
5220                                                   "action not supported");
5221                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5222                         return rte_flow_error_set(error, ENOTSUP,
5223                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5224                                                   actions, "too many actions");
5225                 switch (type) {
5226                 case RTE_FLOW_ACTION_TYPE_VOID:
5227                         break;
5228                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5229                         ret = flow_dv_validate_action_port_id(dev,
5230                                                               action_flags,
5231                                                               actions,
5232                                                               attr,
5233                                                               error);
5234                         if (ret)
5235                                 return ret;
5236                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5237                         ++actions_n;
5238                         break;
5239                 case RTE_FLOW_ACTION_TYPE_FLAG:
5240                         ret = flow_dv_validate_action_flag(dev, action_flags,
5241                                                            attr, error);
5242                         if (ret < 0)
5243                                 return ret;
5244                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5245                                 /* Count all modify-header actions as one. */
5246                                 if (!(action_flags &
5247                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5248                                         ++actions_n;
5249                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
5250                                                 MLX5_FLOW_ACTION_MARK_EXT;
5251                         } else {
5252                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
5253                                 ++actions_n;
5254                         }
5255                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5256                         break;
5257                 case RTE_FLOW_ACTION_TYPE_MARK:
5258                         ret = flow_dv_validate_action_mark(dev, actions,
5259                                                            action_flags,
5260                                                            attr, error);
5261                         if (ret < 0)
5262                                 return ret;
5263                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5264                                 /* Count all modify-header actions as one. */
5265                                 if (!(action_flags &
5266                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5267                                         ++actions_n;
5268                                 action_flags |= MLX5_FLOW_ACTION_MARK |
5269                                                 MLX5_FLOW_ACTION_MARK_EXT;
5270                         } else {
5271                                 action_flags |= MLX5_FLOW_ACTION_MARK;
5272                                 ++actions_n;
5273                         }
5274                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5275                         break;
5276                 case RTE_FLOW_ACTION_TYPE_SET_META:
5277                         ret = flow_dv_validate_action_set_meta(dev, actions,
5278                                                                action_flags,
5279                                                                attr, error);
5280                         if (ret < 0)
5281                                 return ret;
5282                         /* Count all modify-header actions as one action. */
5283                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5284                                 ++actions_n;
5285                         action_flags |= MLX5_FLOW_ACTION_SET_META;
5286                         rw_act_num += MLX5_ACT_NUM_SET_META;
5287                         break;
5288                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5289                         ret = flow_dv_validate_action_set_tag(dev, actions,
5290                                                               action_flags,
5291                                                               attr, error);
5292                         if (ret < 0)
5293                                 return ret;
5294                         /* Count all modify-header actions as one action. */
5295                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5296                                 ++actions_n;
5297                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
5298                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5299                         break;
5300                 case RTE_FLOW_ACTION_TYPE_DROP:
5301                         ret = mlx5_flow_validate_action_drop(action_flags,
5302                                                              attr, error);
5303                         if (ret < 0)
5304                                 return ret;
5305                         action_flags |= MLX5_FLOW_ACTION_DROP;
5306                         ++actions_n;
5307                         break;
5308                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5309                         ret = mlx5_flow_validate_action_queue(actions,
5310                                                               action_flags, dev,
5311                                                               attr, error);
5312                         if (ret < 0)
5313                                 return ret;
5314                         queue_index = ((const struct rte_flow_action_queue *)
5315                                                         (actions->conf))->index;
5316                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
5317                         ++actions_n;
5318                         break;
5319                 case RTE_FLOW_ACTION_TYPE_RSS:
5320                         rss = actions->conf;
5321                         ret = mlx5_flow_validate_action_rss(actions,
5322                                                             action_flags, dev,
5323                                                             attr, item_flags,
5324                                                             error);
5325                         if (ret < 0)
5326                                 return ret;
5327                         if (rss != NULL && rss->queue_num)
5328                                 queue_index = rss->queue[0];
5329                         action_flags |= MLX5_FLOW_ACTION_RSS;
5330                         ++actions_n;
5331                         break;
5332                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
5333                         ret =
5334                         mlx5_flow_validate_action_default_miss(action_flags,
5335                                         attr, error);
5336                         if (ret < 0)
5337                                 return ret;
5338                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
5339                         ++actions_n;
5340                         break;
5341                 case RTE_FLOW_ACTION_TYPE_COUNT:
5342                         ret = flow_dv_validate_action_count(dev, error);
5343                         if (ret < 0)
5344                                 return ret;
5345                         action_flags |= MLX5_FLOW_ACTION_COUNT;
5346                         ++actions_n;
5347                         break;
5348                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5349                         if (flow_dv_validate_action_pop_vlan(dev,
5350                                                              action_flags,
5351                                                              actions,
5352                                                              item_flags, attr,
5353                                                              error))
5354                                 return -rte_errno;
5355                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5356                         ++actions_n;
5357                         break;
5358                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5359                         ret = flow_dv_validate_action_push_vlan(dev,
5360                                                                 action_flags,
5361                                                                 vlan_m,
5362                                                                 actions, attr,
5363                                                                 error);
5364                         if (ret < 0)
5365                                 return ret;
5366                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5367                         ++actions_n;
5368                         break;
5369                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5370                         ret = flow_dv_validate_action_set_vlan_pcp
5371                                                 (action_flags, actions, error);
5372                         if (ret < 0)
5373                                 return ret;
5374                         /* Count PCP with push_vlan command. */
5375                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
5376                         break;
5377                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5378                         ret = flow_dv_validate_action_set_vlan_vid
5379                                                 (item_flags, action_flags,
5380                                                  actions, error);
5381                         if (ret < 0)
5382                                 return ret;
5383                         /* Count VID with push_vlan command. */
5384                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
5385                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
5386                         break;
5387                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5388                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5389                         ret = flow_dv_validate_action_l2_encap(dev,
5390                                                                action_flags,
5391                                                                actions, attr,
5392                                                                error);
5393                         if (ret < 0)
5394                                 return ret;
5395                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
5396                         ++actions_n;
5397                         break;
5398                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5399                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5400                         ret = flow_dv_validate_action_decap(dev, action_flags,
5401                                                             attr, error);
5402                         if (ret < 0)
5403                                 return ret;
5404                         action_flags |= MLX5_FLOW_ACTION_DECAP;
5405                         ++actions_n;
5406                         break;
5407                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5408                         ret = flow_dv_validate_action_raw_encap_decap
5409                                 (dev, NULL, actions->conf, attr, &action_flags,
5410                                  &actions_n, error);
5411                         if (ret < 0)
5412                                 return ret;
5413                         break;
5414                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5415                         decap = actions->conf;
5416                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5417                                 ;
5418                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5419                                 encap = NULL;
5420                                 actions--;
5421                         } else {
5422                                 encap = actions->conf;
5423                         }
5424                         ret = flow_dv_validate_action_raw_encap_decap
5425                                            (dev,
5426                                             decap ? decap : &empty_decap, encap,
5427                                             attr, &action_flags, &actions_n,
5428                                             error);
5429                         if (ret < 0)
5430                                 return ret;
5431                         break;
5432                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5433                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5434                         ret = flow_dv_validate_action_modify_mac(action_flags,
5435                                                                  actions,
5436                                                                  item_flags,
5437                                                                  error);
5438                         if (ret < 0)
5439                                 return ret;
5440                         /* Count all modify-header actions as one action. */
5441                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5442                                 ++actions_n;
5443                         action_flags |= actions->type ==
5444                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5445                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
5446                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
5447                         /*
5448                          * Even if the source and destination MAC addresses have
5449                          * overlap in the header with 4B alignment, the convert
5450                          * function will handle them separately and 4 SW actions
5451                          * will be created. And 2 actions will be added each
5452                          * time no matter how many bytes of address will be set.
5453                          */
5454                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
5455                         break;
5456                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5457                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5458                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
5459                                                                   actions,
5460                                                                   item_flags,
5461                                                                   error);
5462                         if (ret < 0)
5463                                 return ret;
5464                         /* Count all modify-header actions as one action. */
5465                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5466                                 ++actions_n;
5467                         action_flags |= actions->type ==
5468                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5469                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
5470                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
5471                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
5472                         break;
5473                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5474                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5475                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
5476                                                                   actions,
5477                                                                   item_flags,
5478                                                                   error);
5479                         if (ret < 0)
5480                                 return ret;
5481                         if (item_ipv6_proto == IPPROTO_ICMPV6)
5482                                 return rte_flow_error_set(error, ENOTSUP,
5483                                         RTE_FLOW_ERROR_TYPE_ACTION,
5484                                         actions,
5485                                         "Can't change header "
5486                                         "with ICMPv6 proto");
5487                         /* Count all modify-header actions as one action. */
5488                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5489                                 ++actions_n;
5490                         action_flags |= actions->type ==
5491                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5492                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
5493                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
5494                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
5495                         break;
5496                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5497                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5498                         ret = flow_dv_validate_action_modify_tp(action_flags,
5499                                                                 actions,
5500                                                                 item_flags,
5501                                                                 error);
5502                         if (ret < 0)
5503                                 return ret;
5504                         /* Count all modify-header actions as one action. */
5505                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5506                                 ++actions_n;
5507                         action_flags |= actions->type ==
5508                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5509                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
5510                                                 MLX5_FLOW_ACTION_SET_TP_DST;
5511                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
5512                         break;
5513                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5514                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5515                         ret = flow_dv_validate_action_modify_ttl(action_flags,
5516                                                                  actions,
5517                                                                  item_flags,
5518                                                                  error);
5519                         if (ret < 0)
5520                                 return ret;
5521                         /* Count all modify-header actions as one action. */
5522                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5523                                 ++actions_n;
5524                         action_flags |= actions->type ==
5525                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
5526                                                 MLX5_FLOW_ACTION_SET_TTL :
5527                                                 MLX5_FLOW_ACTION_DEC_TTL;
5528                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
5529                         break;
5530                 case RTE_FLOW_ACTION_TYPE_JUMP:
5531                         ret = flow_dv_validate_action_jump(actions,
5532                                                            action_flags,
5533                                                            attr, external,
5534                                                            error);
5535                         if (ret)
5536                                 return ret;
5537                         ++actions_n;
5538                         action_flags |= MLX5_FLOW_ACTION_JUMP;
5539                         break;
5540                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5541                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5542                         ret = flow_dv_validate_action_modify_tcp_seq
5543                                                                 (action_flags,
5544                                                                  actions,
5545                                                                  item_flags,
5546                                                                  error);
5547                         if (ret < 0)
5548                                 return ret;
5549                         /* Count all modify-header actions as one action. */
5550                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5551                                 ++actions_n;
5552                         action_flags |= actions->type ==
5553                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5554                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
5555                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5556                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
5557                         break;
5558                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5559                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5560                         ret = flow_dv_validate_action_modify_tcp_ack
5561                                                                 (action_flags,
5562                                                                  actions,
5563                                                                  item_flags,
5564                                                                  error);
5565                         if (ret < 0)
5566                                 return ret;
5567                         /* Count all modify-header actions as one action. */
5568                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5569                                 ++actions_n;
5570                         action_flags |= actions->type ==
5571                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
5572                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
5573                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
5574                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
5575                         break;
5576                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
5577                         break;
5578                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
5579                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
5580                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5581                         break;
5582                 case RTE_FLOW_ACTION_TYPE_METER:
5583                         ret = mlx5_flow_validate_action_meter(dev,
5584                                                               action_flags,
5585                                                               actions, attr,
5586                                                               error);
5587                         if (ret < 0)
5588                                 return ret;
5589                         action_flags |= MLX5_FLOW_ACTION_METER;
5590                         ++actions_n;
5591                         /* Meter action will add one more TAG action. */
5592                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5593                         break;
5594                 case RTE_FLOW_ACTION_TYPE_AGE:
5595                         ret = flow_dv_validate_action_age(action_flags,
5596                                                           actions, dev,
5597                                                           error);
5598                         if (ret < 0)
5599                                 return ret;
5600                         action_flags |= MLX5_FLOW_ACTION_AGE;
5601                         ++actions_n;
5602                         break;
5603                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5604                         ret = flow_dv_validate_action_modify_ipv4_dscp
5605                                                          (action_flags,
5606                                                           actions,
5607                                                           item_flags,
5608                                                           error);
5609                         if (ret < 0)
5610                                 return ret;
5611                         /* Count all modify-header actions as one action. */
5612                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5613                                 ++actions_n;
5614                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
5615                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5616                         break;
5617                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5618                         ret = flow_dv_validate_action_modify_ipv6_dscp
5619                                                                 (action_flags,
5620                                                                  actions,
5621                                                                  item_flags,
5622                                                                  error);
5623                         if (ret < 0)
5624                                 return ret;
5625                         /* Count all modify-header actions as one action. */
5626                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5627                                 ++actions_n;
5628                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
5629                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5630                         break;
5631                 default:
5632                         return rte_flow_error_set(error, ENOTSUP,
5633                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5634                                                   actions,
5635                                                   "action not supported");
5636                 }
5637         }
5638         /*
5639          * Validate the drop action mutual exclusion with other actions.
5640          * Drop action is mutually-exclusive with any other action, except for
5641          * Count action.
5642          */
5643         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
5644             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
5645                 return rte_flow_error_set(error, EINVAL,
5646                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5647                                           "Drop action is mutually-exclusive "
5648                                           "with any other action, except for "
5649                                           "Count action");
5650         /* Eswitch has few restrictions on using items and actions */
5651         if (attr->transfer) {
5652                 if (!mlx5_flow_ext_mreg_supported(dev) &&
5653                     action_flags & MLX5_FLOW_ACTION_FLAG)
5654                         return rte_flow_error_set(error, ENOTSUP,
5655                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5656                                                   NULL,
5657                                                   "unsupported action FLAG");
5658                 if (!mlx5_flow_ext_mreg_supported(dev) &&
5659                     action_flags & MLX5_FLOW_ACTION_MARK)
5660                         return rte_flow_error_set(error, ENOTSUP,
5661                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5662                                                   NULL,
5663                                                   "unsupported action MARK");
5664                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
5665                         return rte_flow_error_set(error, ENOTSUP,
5666                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5667                                                   NULL,
5668                                                   "unsupported action QUEUE");
5669                 if (action_flags & MLX5_FLOW_ACTION_RSS)
5670                         return rte_flow_error_set(error, ENOTSUP,
5671                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5672                                                   NULL,
5673                                                   "unsupported action RSS");
5674                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5675                         return rte_flow_error_set(error, EINVAL,
5676                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5677                                                   actions,
5678                                                   "no fate action is found");
5679         } else {
5680                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
5681                         return rte_flow_error_set(error, EINVAL,
5682                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5683                                                   actions,
5684                                                   "no fate action is found");
5685         }
5686         /* Continue validation for Xcap actions.*/
5687         if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) && (queue_index == 0xFFFF ||
5688             mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
5689                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
5690                     MLX5_FLOW_XCAP_ACTIONS)
5691                         return rte_flow_error_set(error, ENOTSUP,
5692                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5693                                                   NULL, "encap and decap "
5694                                                   "combination aren't supported");
5695                 if (!attr->transfer && attr->ingress && (action_flags &
5696                                                         MLX5_FLOW_ACTION_ENCAP))
5697                         return rte_flow_error_set(error, ENOTSUP,
5698                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5699                                                   NULL, "encap is not supported"
5700                                                   " for ingress traffic");
5701         }
5702         /* Hairpin flow will add one more TAG action. */
5703         if (hairpin > 0)
5704                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5705         /* extra metadata enabled: one more TAG action will be add. */
5706         if (dev_conf->dv_flow_en &&
5707             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
5708             mlx5_flow_ext_mreg_supported(dev))
5709                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5710         if ((uint32_t)rw_act_num >
5711                         flow_dv_modify_hdr_action_max(dev, is_root)) {
5712                 return rte_flow_error_set(error, ENOTSUP,
5713                                           RTE_FLOW_ERROR_TYPE_ACTION,
5714                                           NULL, "too many header modify"
5715                                           " actions to support");
5716         }
5717         return 0;
5718 }
5719
5720 /**
5721  * Internal preparation function. Allocates the DV flow size,
5722  * this size is constant.
5723  *
5724  * @param[in] dev
5725  *   Pointer to the rte_eth_dev structure.
5726  * @param[in] attr
5727  *   Pointer to the flow attributes.
5728  * @param[in] items
5729  *   Pointer to the list of items.
5730  * @param[in] actions
5731  *   Pointer to the list of actions.
5732  * @param[out] error
5733  *   Pointer to the error structure.
5734  *
5735  * @return
5736  *   Pointer to mlx5_flow object on success,
5737  *   otherwise NULL and rte_errno is set.
5738  */
5739 static struct mlx5_flow *
5740 flow_dv_prepare(struct rte_eth_dev *dev,
5741                 const struct rte_flow_attr *attr __rte_unused,
5742                 const struct rte_flow_item items[] __rte_unused,
5743                 const struct rte_flow_action actions[] __rte_unused,
5744                 struct rte_flow_error *error)
5745 {
5746         uint32_t handle_idx = 0;
5747         struct mlx5_flow *dev_flow;
5748         struct mlx5_flow_handle *dev_handle;
5749         struct mlx5_priv *priv = dev->data->dev_private;
5750
5751         /* In case of corrupting the memory. */
5752         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
5753                 rte_flow_error_set(error, ENOSPC,
5754                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5755                                    "not free temporary device flow");
5756                 return NULL;
5757         }
5758         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
5759                                    &handle_idx);
5760         if (!dev_handle) {
5761                 rte_flow_error_set(error, ENOMEM,
5762                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5763                                    "not enough memory to create flow handle");
5764                 return NULL;
5765         }
5766         /* No multi-thread supporting. */
5767         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
5768         dev_flow->handle = dev_handle;
5769         dev_flow->handle_idx = handle_idx;
5770         /*
5771          * In some old rdma-core releases, before continuing, a check of the
5772          * length of matching parameter will be done at first. It needs to use
5773          * the length without misc4 param. If the flow has misc4 support, then
5774          * the length needs to be adjusted accordingly. Each param member is
5775          * aligned with a 64B boundary naturally.
5776          */
5777         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param) -
5778                                   MLX5_ST_SZ_BYTES(fte_match_set_misc4);
5779         /*
5780          * The matching value needs to be cleared to 0 before using. In the
5781          * past, it will be automatically cleared when using rte_*alloc
5782          * API. The time consumption will be almost the same as before.
5783          */
5784         memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
5785         dev_flow->ingress = attr->ingress;
5786         dev_flow->dv.transfer = attr->transfer;
5787         return dev_flow;
5788 }
5789
5790 #ifdef RTE_LIBRTE_MLX5_DEBUG
5791 /**
5792  * Sanity check for match mask and value. Similar to check_valid_spec() in
5793  * kernel driver. If unmasked bit is present in value, it returns failure.
5794  *
5795  * @param match_mask
5796  *   pointer to match mask buffer.
5797  * @param match_value
5798  *   pointer to match value buffer.
5799  *
5800  * @return
5801  *   0 if valid, -EINVAL otherwise.
5802  */
5803 static int
5804 flow_dv_check_valid_spec(void *match_mask, void *match_value)
5805 {
5806         uint8_t *m = match_mask;
5807         uint8_t *v = match_value;
5808         unsigned int i;
5809
5810         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
5811                 if (v[i] & ~m[i]) {
5812                         DRV_LOG(ERR,
5813                                 "match_value differs from match_criteria"
5814                                 " %p[%u] != %p[%u]",
5815                                 match_value, i, match_mask, i);
5816                         return -EINVAL;
5817                 }
5818         }
5819         return 0;
5820 }
5821 #endif
5822
5823 /**
5824  * Add match of ip_version.
5825  *
5826  * @param[in] group
5827  *   Flow group.
5828  * @param[in] headers_v
5829  *   Values header pointer.
5830  * @param[in] headers_m
5831  *   Masks header pointer.
5832  * @param[in] ip_version
5833  *   The IP version to set.
5834  */
5835 static inline void
5836 flow_dv_set_match_ip_version(uint32_t group,
5837                              void *headers_v,
5838                              void *headers_m,
5839                              uint8_t ip_version)
5840 {
5841         if (group == 0)
5842                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
5843         else
5844                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
5845                          ip_version);
5846         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
5847         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
5848         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
5849 }
5850
5851 /**
5852  * Add Ethernet item to matcher and to the value.
5853  *
5854  * @param[in, out] matcher
5855  *   Flow matcher.
5856  * @param[in, out] key
5857  *   Flow matcher value.
5858  * @param[in] item
5859  *   Flow pattern to translate.
5860  * @param[in] inner
5861  *   Item is inner pattern.
5862  */
5863 static void
5864 flow_dv_translate_item_eth(void *matcher, void *key,
5865                            const struct rte_flow_item *item, int inner,
5866                            uint32_t group)
5867 {
5868         const struct rte_flow_item_eth *eth_m = item->mask;
5869         const struct rte_flow_item_eth *eth_v = item->spec;
5870         const struct rte_flow_item_eth nic_mask = {
5871                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
5872                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
5873                 .type = RTE_BE16(0xffff),
5874         };
5875         void *headers_m;
5876         void *headers_v;
5877         char *l24_v;
5878         unsigned int i;
5879
5880         if (!eth_v)
5881                 return;
5882         if (!eth_m)
5883                 eth_m = &nic_mask;
5884         if (inner) {
5885                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5886                                          inner_headers);
5887                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5888         } else {
5889                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5890                                          outer_headers);
5891                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5892         }
5893         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
5894                &eth_m->dst, sizeof(eth_m->dst));
5895         /* The value must be in the range of the mask. */
5896         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
5897         for (i = 0; i < sizeof(eth_m->dst); ++i)
5898                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
5899         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
5900                &eth_m->src, sizeof(eth_m->src));
5901         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
5902         /* The value must be in the range of the mask. */
5903         for (i = 0; i < sizeof(eth_m->dst); ++i)
5904                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
5905         if (eth_v->type) {
5906                 /* When ethertype is present set mask for tagged VLAN. */
5907                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
5908                 /* Set value for tagged VLAN if ethertype is 802.1Q. */
5909                 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
5910                     eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
5911                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag,
5912                                  1);
5913                         /* Return here to avoid setting match on ethertype. */
5914                         return;
5915                 }
5916         }
5917         /*
5918          * HW supports match on one Ethertype, the Ethertype following the last
5919          * VLAN tag of the packet (see PRM).
5920          * Set match on ethertype only if ETH header is not followed by VLAN.
5921          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
5922          * ethertype, and use ip_version field instead.
5923          * eCPRI over Ether layer will use type value 0xAEFE.
5924          */
5925         if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
5926             eth_m->type == 0xFFFF) {
5927                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
5928         } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
5929                    eth_m->type == 0xFFFF) {
5930                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
5931         } else {
5932                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
5933                          rte_be_to_cpu_16(eth_m->type));
5934                 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
5935                                      ethertype);
5936                 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
5937         }
5938 }
5939
5940 /**
5941  * Add VLAN item to matcher and to the value.
5942  *
5943  * @param[in, out] dev_flow
5944  *   Flow descriptor.
5945  * @param[in, out] matcher
5946  *   Flow matcher.
5947  * @param[in, out] key
5948  *   Flow matcher value.
5949  * @param[in] item
5950  *   Flow pattern to translate.
5951  * @param[in] inner
5952  *   Item is inner pattern.
5953  */
5954 static void
5955 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
5956                             void *matcher, void *key,
5957                             const struct rte_flow_item *item,
5958                             int inner, uint32_t group)
5959 {
5960         const struct rte_flow_item_vlan *vlan_m = item->mask;
5961         const struct rte_flow_item_vlan *vlan_v = item->spec;
5962         void *headers_m;
5963         void *headers_v;
5964         uint16_t tci_m;
5965         uint16_t tci_v;
5966
5967         if (inner) {
5968                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5969                                          inner_headers);
5970                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5971         } else {
5972                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5973                                          outer_headers);
5974                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5975                 /*
5976                  * This is workaround, masks are not supported,
5977                  * and pre-validated.
5978                  */
5979                 if (vlan_v)
5980                         dev_flow->handle->vf_vlan.tag =
5981                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
5982         }
5983         /*
5984          * When VLAN item exists in flow, mark packet as tagged,
5985          * even if TCI is not specified.
5986          */
5987         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
5988         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
5989         if (!vlan_v)
5990                 return;
5991         if (!vlan_m)
5992                 vlan_m = &rte_flow_item_vlan_mask;
5993         tci_m = rte_be_to_cpu_16(vlan_m->tci);
5994         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
5995         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
5996         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
5997         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
5998         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
5999         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
6000         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
6001         /*
6002          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
6003          * ethertype, and use ip_version field instead.
6004          */
6005         if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
6006             vlan_m->inner_type == 0xFFFF) {
6007                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6008         } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
6009                    vlan_m->inner_type == 0xFFFF) {
6010                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6011         } else {
6012                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
6013                          rte_be_to_cpu_16(vlan_m->inner_type));
6014                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
6015                          rte_be_to_cpu_16(vlan_m->inner_type &
6016                                           vlan_v->inner_type));
6017         }
6018 }
6019
6020 /**
6021  * Add IPV4 item to matcher and to the value.
6022  *
6023  * @param[in, out] matcher
6024  *   Flow matcher.
6025  * @param[in, out] key
6026  *   Flow matcher value.
6027  * @param[in] item
6028  *   Flow pattern to translate.
6029  * @param[in] item_flags
6030  *   Bit-fields that holds the items detected until now.
6031  * @param[in] inner
6032  *   Item is inner pattern.
6033  * @param[in] group
6034  *   The group to insert the rule.
6035  */
6036 static void
6037 flow_dv_translate_item_ipv4(void *matcher, void *key,
6038                             const struct rte_flow_item *item,
6039                             const uint64_t item_flags,
6040                             int inner, uint32_t group)
6041 {
6042         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
6043         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
6044         const struct rte_flow_item_ipv4 nic_mask = {
6045                 .hdr = {
6046                         .src_addr = RTE_BE32(0xffffffff),
6047                         .dst_addr = RTE_BE32(0xffffffff),
6048                         .type_of_service = 0xff,
6049                         .next_proto_id = 0xff,
6050                         .time_to_live = 0xff,
6051                 },
6052         };
6053         void *headers_m;
6054         void *headers_v;
6055         char *l24_m;
6056         char *l24_v;
6057         uint8_t tos;
6058
6059         if (inner) {
6060                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6061                                          inner_headers);
6062                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6063         } else {
6064                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6065                                          outer_headers);
6066                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6067         }
6068         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6069         /*
6070          * On outer header (which must contains L2), or inner header with L2,
6071          * set cvlan_tag mask bit to mark this packet as untagged.
6072          * This should be done even if item->spec is empty.
6073          */
6074         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6075                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6076         if (!ipv4_v)
6077                 return;
6078         if (!ipv4_m)
6079                 ipv4_m = &nic_mask;
6080         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6081                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6082         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6083                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6084         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
6085         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
6086         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6087                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6088         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6089                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6090         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
6091         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
6092         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
6093         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
6094                  ipv4_m->hdr.type_of_service);
6095         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
6096         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
6097                  ipv4_m->hdr.type_of_service >> 2);
6098         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
6099         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6100                  ipv4_m->hdr.next_proto_id);
6101         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6102                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
6103         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6104                  ipv4_m->hdr.time_to_live);
6105         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6106                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
6107 }
6108
6109 /**
6110  * Add IPV6 item to matcher and to the value.
6111  *
6112  * @param[in, out] matcher
6113  *   Flow matcher.
6114  * @param[in, out] key
6115  *   Flow matcher value.
6116  * @param[in] item
6117  *   Flow pattern to translate.
6118  * @param[in] item_flags
6119  *   Bit-fields that holds the items detected until now.
6120  * @param[in] inner
6121  *   Item is inner pattern.
6122  * @param[in] group
6123  *   The group to insert the rule.
6124  */
6125 static void
6126 flow_dv_translate_item_ipv6(void *matcher, void *key,
6127                             const struct rte_flow_item *item,
6128                             const uint64_t item_flags,
6129                             int inner, uint32_t group)
6130 {
6131         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
6132         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
6133         const struct rte_flow_item_ipv6 nic_mask = {
6134                 .hdr = {
6135                         .src_addr =
6136                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6137                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6138                         .dst_addr =
6139                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6140                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6141                         .vtc_flow = RTE_BE32(0xffffffff),
6142                         .proto = 0xff,
6143                         .hop_limits = 0xff,
6144                 },
6145         };
6146         void *headers_m;
6147         void *headers_v;
6148         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6149         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6150         char *l24_m;
6151         char *l24_v;
6152         uint32_t vtc_m;
6153         uint32_t vtc_v;
6154         int i;
6155         int size;
6156
6157         if (inner) {
6158                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6159                                          inner_headers);
6160                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6161         } else {
6162                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6163                                          outer_headers);
6164                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6165         }
6166         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6167         /*
6168          * On outer header (which must contains L2), or inner header with L2,
6169          * set cvlan_tag mask bit to mark this packet as untagged.
6170          * This should be done even if item->spec is empty.
6171          */
6172         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6173                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6174         if (!ipv6_v)
6175                 return;
6176         if (!ipv6_m)
6177                 ipv6_m = &nic_mask;
6178         size = sizeof(ipv6_m->hdr.dst_addr);
6179         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6180                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6181         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6182                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6183         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
6184         for (i = 0; i < size; ++i)
6185                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
6186         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6187                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6188         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6189                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6190         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
6191         for (i = 0; i < size; ++i)
6192                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
6193         /* TOS. */
6194         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
6195         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
6196         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
6197         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
6198         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
6199         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
6200         /* Label. */
6201         if (inner) {
6202                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
6203                          vtc_m);
6204                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
6205                          vtc_v);
6206         } else {
6207                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
6208                          vtc_m);
6209                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
6210                          vtc_v);
6211         }
6212         /* Protocol. */
6213         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6214                  ipv6_m->hdr.proto);
6215         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6216                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
6217         /* Hop limit. */
6218         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6219                  ipv6_m->hdr.hop_limits);
6220         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6221                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
6222 }
6223
6224 /**
6225  * Add TCP item to matcher and to the value.
6226  *
6227  * @param[in, out] matcher
6228  *   Flow matcher.
6229  * @param[in, out] key
6230  *   Flow matcher value.
6231  * @param[in] item
6232  *   Flow pattern to translate.
6233  * @param[in] inner
6234  *   Item is inner pattern.
6235  */
6236 static void
6237 flow_dv_translate_item_tcp(void *matcher, void *key,
6238                            const struct rte_flow_item *item,
6239                            int inner)
6240 {
6241         const struct rte_flow_item_tcp *tcp_m = item->mask;
6242         const struct rte_flow_item_tcp *tcp_v = item->spec;
6243         void *headers_m;
6244         void *headers_v;
6245
6246         if (inner) {
6247                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6248                                          inner_headers);
6249                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6250         } else {
6251                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6252                                          outer_headers);
6253                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6254         }
6255         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6256         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
6257         if (!tcp_v)
6258                 return;
6259         if (!tcp_m)
6260                 tcp_m = &rte_flow_item_tcp_mask;
6261         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
6262                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
6263         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
6264                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
6265         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
6266                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
6267         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
6268                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
6269         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
6270                  tcp_m->hdr.tcp_flags);
6271         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
6272                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
6273 }
6274
6275 /**
6276  * Add UDP item to matcher and to the value.
6277  *
6278  * @param[in, out] matcher
6279  *   Flow matcher.
6280  * @param[in, out] key
6281  *   Flow matcher value.
6282  * @param[in] item
6283  *   Flow pattern to translate.
6284  * @param[in] inner
6285  *   Item is inner pattern.
6286  */
6287 static void
6288 flow_dv_translate_item_udp(void *matcher, void *key,
6289                            const struct rte_flow_item *item,
6290                            int inner)
6291 {
6292         const struct rte_flow_item_udp *udp_m = item->mask;
6293         const struct rte_flow_item_udp *udp_v = item->spec;
6294         void *headers_m;
6295         void *headers_v;
6296
6297         if (inner) {
6298                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6299                                          inner_headers);
6300                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6301         } else {
6302                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6303                                          outer_headers);
6304                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6305         }
6306         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6307         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
6308         if (!udp_v)
6309                 return;
6310         if (!udp_m)
6311                 udp_m = &rte_flow_item_udp_mask;
6312         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
6313                  rte_be_to_cpu_16(udp_m->hdr.src_port));
6314         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
6315                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
6316         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
6317                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
6318         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6319                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
6320 }
6321
6322 /**
6323  * Add GRE optional Key item to matcher and to the value.
6324  *
6325  * @param[in, out] matcher
6326  *   Flow matcher.
6327  * @param[in, out] key
6328  *   Flow matcher value.
6329  * @param[in] item
6330  *   Flow pattern to translate.
6331  * @param[in] inner
6332  *   Item is inner pattern.
6333  */
6334 static void
6335 flow_dv_translate_item_gre_key(void *matcher, void *key,
6336                                    const struct rte_flow_item *item)
6337 {
6338         const rte_be32_t *key_m = item->mask;
6339         const rte_be32_t *key_v = item->spec;
6340         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6341         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6342         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
6343
6344         /* GRE K bit must be on and should already be validated */
6345         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
6346         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
6347         if (!key_v)
6348                 return;
6349         if (!key_m)
6350                 key_m = &gre_key_default_mask;
6351         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
6352                  rte_be_to_cpu_32(*key_m) >> 8);
6353         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
6354                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
6355         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
6356                  rte_be_to_cpu_32(*key_m) & 0xFF);
6357         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
6358                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
6359 }
6360
6361 /**
6362  * Add GRE item to matcher and to the value.
6363  *
6364  * @param[in, out] matcher
6365  *   Flow matcher.
6366  * @param[in, out] key
6367  *   Flow matcher value.
6368  * @param[in] item
6369  *   Flow pattern to translate.
6370  * @param[in] inner
6371  *   Item is inner pattern.
6372  */
6373 static void
6374 flow_dv_translate_item_gre(void *matcher, void *key,
6375                            const struct rte_flow_item *item,
6376                            int inner)
6377 {
6378         const struct rte_flow_item_gre *gre_m = item->mask;
6379         const struct rte_flow_item_gre *gre_v = item->spec;
6380         void *headers_m;
6381         void *headers_v;
6382         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6383         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6384         struct {
6385                 union {
6386                         __extension__
6387                         struct {
6388                                 uint16_t version:3;
6389                                 uint16_t rsvd0:9;
6390                                 uint16_t s_present:1;
6391                                 uint16_t k_present:1;
6392                                 uint16_t rsvd_bit1:1;
6393                                 uint16_t c_present:1;
6394                         };
6395                         uint16_t value;
6396                 };
6397         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
6398
6399         if (inner) {
6400                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6401                                          inner_headers);
6402                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6403         } else {
6404                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6405                                          outer_headers);
6406                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6407         }
6408         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6409         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
6410         if (!gre_v)
6411                 return;
6412         if (!gre_m)
6413                 gre_m = &rte_flow_item_gre_mask;
6414         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
6415                  rte_be_to_cpu_16(gre_m->protocol));
6416         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6417                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
6418         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
6419         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
6420         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
6421                  gre_crks_rsvd0_ver_m.c_present);
6422         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
6423                  gre_crks_rsvd0_ver_v.c_present &
6424                  gre_crks_rsvd0_ver_m.c_present);
6425         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
6426                  gre_crks_rsvd0_ver_m.k_present);
6427         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
6428                  gre_crks_rsvd0_ver_v.k_present &
6429                  gre_crks_rsvd0_ver_m.k_present);
6430         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
6431                  gre_crks_rsvd0_ver_m.s_present);
6432         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
6433                  gre_crks_rsvd0_ver_v.s_present &
6434                  gre_crks_rsvd0_ver_m.s_present);
6435 }
6436
6437 /**
6438  * Add NVGRE item to matcher and to the value.
6439  *
6440  * @param[in, out] matcher
6441  *   Flow matcher.
6442  * @param[in, out] key
6443  *   Flow matcher value.
6444  * @param[in] item
6445  *   Flow pattern to translate.
6446  * @param[in] inner
6447  *   Item is inner pattern.
6448  */
6449 static void
6450 flow_dv_translate_item_nvgre(void *matcher, void *key,
6451                              const struct rte_flow_item *item,
6452                              int inner)
6453 {
6454         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
6455         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
6456         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6457         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6458         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
6459         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
6460         char *gre_key_m;
6461         char *gre_key_v;
6462         int size;
6463         int i;
6464
6465         /* For NVGRE, GRE header fields must be set with defined values. */
6466         const struct rte_flow_item_gre gre_spec = {
6467                 .c_rsvd0_ver = RTE_BE16(0x2000),
6468                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
6469         };
6470         const struct rte_flow_item_gre gre_mask = {
6471                 .c_rsvd0_ver = RTE_BE16(0xB000),
6472                 .protocol = RTE_BE16(UINT16_MAX),
6473         };
6474         const struct rte_flow_item gre_item = {
6475                 .spec = &gre_spec,
6476                 .mask = &gre_mask,
6477                 .last = NULL,
6478         };
6479         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
6480         if (!nvgre_v)
6481                 return;
6482         if (!nvgre_m)
6483                 nvgre_m = &rte_flow_item_nvgre_mask;
6484         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
6485         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
6486         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
6487         memcpy(gre_key_m, tni_flow_id_m, size);
6488         for (i = 0; i < size; ++i)
6489                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
6490 }
6491
6492 /**
6493  * Add VXLAN item to matcher and to the value.
6494  *
6495  * @param[in, out] matcher
6496  *   Flow matcher.
6497  * @param[in, out] key
6498  *   Flow matcher value.
6499  * @param[in] item
6500  *   Flow pattern to translate.
6501  * @param[in] inner
6502  *   Item is inner pattern.
6503  */
6504 static void
6505 flow_dv_translate_item_vxlan(void *matcher, void *key,
6506                              const struct rte_flow_item *item,
6507                              int inner)
6508 {
6509         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
6510         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
6511         void *headers_m;
6512         void *headers_v;
6513         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6514         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6515         char *vni_m;
6516         char *vni_v;
6517         uint16_t dport;
6518         int size;
6519         int i;
6520
6521         if (inner) {
6522                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6523                                          inner_headers);
6524                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6525         } else {
6526                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6527                                          outer_headers);
6528                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6529         }
6530         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6531                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6532         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6533                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6534                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6535         }
6536         if (!vxlan_v)
6537                 return;
6538         if (!vxlan_m)
6539                 vxlan_m = &rte_flow_item_vxlan_mask;
6540         size = sizeof(vxlan_m->vni);
6541         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
6542         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
6543         memcpy(vni_m, vxlan_m->vni, size);
6544         for (i = 0; i < size; ++i)
6545                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6546 }
6547
6548 /**
6549  * Add VXLAN-GPE item to matcher and to the value.
6550  *
6551  * @param[in, out] matcher
6552  *   Flow matcher.
6553  * @param[in, out] key
6554  *   Flow matcher value.
6555  * @param[in] item
6556  *   Flow pattern to translate.
6557  * @param[in] inner
6558  *   Item is inner pattern.
6559  */
6560
6561 static void
6562 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
6563                                  const struct rte_flow_item *item, int inner)
6564 {
6565         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
6566         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
6567         void *headers_m;
6568         void *headers_v;
6569         void *misc_m =
6570                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
6571         void *misc_v =
6572                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
6573         char *vni_m;
6574         char *vni_v;
6575         uint16_t dport;
6576         int size;
6577         int i;
6578         uint8_t flags_m = 0xff;
6579         uint8_t flags_v = 0xc;
6580
6581         if (inner) {
6582                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6583                                          inner_headers);
6584                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6585         } else {
6586                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6587                                          outer_headers);
6588                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6589         }
6590         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6591                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6592         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6593                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6594                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6595         }
6596         if (!vxlan_v)
6597                 return;
6598         if (!vxlan_m)
6599                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
6600         size = sizeof(vxlan_m->vni);
6601         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
6602         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
6603         memcpy(vni_m, vxlan_m->vni, size);
6604         for (i = 0; i < size; ++i)
6605                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6606         if (vxlan_m->flags) {
6607                 flags_m = vxlan_m->flags;
6608                 flags_v = vxlan_v->flags;
6609         }
6610         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
6611         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
6612         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
6613                  vxlan_m->protocol);
6614         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
6615                  vxlan_v->protocol);
6616 }
6617
6618 /**
6619  * Add Geneve item to matcher and to the value.
6620  *
6621  * @param[in, out] matcher
6622  *   Flow matcher.
6623  * @param[in, out] key
6624  *   Flow matcher value.
6625  * @param[in] item
6626  *   Flow pattern to translate.
6627  * @param[in] inner
6628  *   Item is inner pattern.
6629  */
6630
6631 static void
6632 flow_dv_translate_item_geneve(void *matcher, void *key,
6633                               const struct rte_flow_item *item, int inner)
6634 {
6635         const struct rte_flow_item_geneve *geneve_m = item->mask;
6636         const struct rte_flow_item_geneve *geneve_v = item->spec;
6637         void *headers_m;
6638         void *headers_v;
6639         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6640         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6641         uint16_t dport;
6642         uint16_t gbhdr_m;
6643         uint16_t gbhdr_v;
6644         char *vni_m;
6645         char *vni_v;
6646         size_t size, i;
6647
6648         if (inner) {
6649                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6650                                          inner_headers);
6651                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6652         } else {
6653                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6654                                          outer_headers);
6655                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6656         }
6657         dport = MLX5_UDP_PORT_GENEVE;
6658         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6659                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6660                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6661         }
6662         if (!geneve_v)
6663                 return;
6664         if (!geneve_m)
6665                 geneve_m = &rte_flow_item_geneve_mask;
6666         size = sizeof(geneve_m->vni);
6667         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
6668         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
6669         memcpy(vni_m, geneve_m->vni, size);
6670         for (i = 0; i < size; ++i)
6671                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
6672         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
6673                  rte_be_to_cpu_16(geneve_m->protocol));
6674         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
6675                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
6676         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
6677         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
6678         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
6679                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
6680         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
6681                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
6682         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
6683                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
6684         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
6685                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
6686                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
6687 }
6688
6689 /**
6690  * Add MPLS item to matcher and to the value.
6691  *
6692  * @param[in, out] matcher
6693  *   Flow matcher.
6694  * @param[in, out] key
6695  *   Flow matcher value.
6696  * @param[in] item
6697  *   Flow pattern to translate.
6698  * @param[in] prev_layer
6699  *   The protocol layer indicated in previous item.
6700  * @param[in] inner
6701  *   Item is inner pattern.
6702  */
6703 static void
6704 flow_dv_translate_item_mpls(void *matcher, void *key,
6705                             const struct rte_flow_item *item,
6706                             uint64_t prev_layer,
6707                             int inner)
6708 {
6709         const uint32_t *in_mpls_m = item->mask;
6710         const uint32_t *in_mpls_v = item->spec;
6711         uint32_t *out_mpls_m = 0;
6712         uint32_t *out_mpls_v = 0;
6713         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6714         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6715         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
6716                                      misc_parameters_2);
6717         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
6718         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
6719         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6720
6721         switch (prev_layer) {
6722         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
6723                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
6724                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6725                          MLX5_UDP_PORT_MPLS);
6726                 break;
6727         case MLX5_FLOW_LAYER_GRE:
6728                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
6729                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6730                          RTE_ETHER_TYPE_MPLS);
6731                 break;
6732         default:
6733                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6734                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6735                          IPPROTO_MPLS);
6736                 break;
6737         }
6738         if (!in_mpls_v)
6739                 return;
6740         if (!in_mpls_m)
6741                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
6742         switch (prev_layer) {
6743         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
6744                 out_mpls_m =
6745                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
6746                                                  outer_first_mpls_over_udp);
6747                 out_mpls_v =
6748                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
6749                                                  outer_first_mpls_over_udp);
6750                 break;
6751         case MLX5_FLOW_LAYER_GRE:
6752                 out_mpls_m =
6753                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
6754                                                  outer_first_mpls_over_gre);
6755                 out_mpls_v =
6756                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
6757                                                  outer_first_mpls_over_gre);
6758                 break;
6759         default:
6760                 /* Inner MPLS not over GRE is not supported. */
6761                 if (!inner) {
6762                         out_mpls_m =
6763                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
6764                                                          misc2_m,
6765                                                          outer_first_mpls);
6766                         out_mpls_v =
6767                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
6768                                                          misc2_v,
6769                                                          outer_first_mpls);
6770                 }
6771                 break;
6772         }
6773         if (out_mpls_m && out_mpls_v) {
6774                 *out_mpls_m = *in_mpls_m;
6775                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
6776         }
6777 }
6778
6779 /**
6780  * Add metadata register item to matcher
6781  *
6782  * @param[in, out] matcher
6783  *   Flow matcher.
6784  * @param[in, out] key
6785  *   Flow matcher value.
6786  * @param[in] reg_type
6787  *   Type of device metadata register
6788  * @param[in] value
6789  *   Register value
6790  * @param[in] mask
6791  *   Register mask
6792  */
6793 static void
6794 flow_dv_match_meta_reg(void *matcher, void *key,
6795                        enum modify_reg reg_type,
6796                        uint32_t data, uint32_t mask)
6797 {
6798         void *misc2_m =
6799                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
6800         void *misc2_v =
6801                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
6802         uint32_t temp;
6803
6804         data &= mask;
6805         switch (reg_type) {
6806         case REG_A:
6807                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
6808                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
6809                 break;
6810         case REG_B:
6811                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
6812                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
6813                 break;
6814         case REG_C_0:
6815                 /*
6816                  * The metadata register C0 field might be divided into
6817                  * source vport index and META item value, we should set
6818                  * this field according to specified mask, not as whole one.
6819                  */
6820                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
6821                 temp |= mask;
6822                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
6823                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
6824                 temp &= ~mask;
6825                 temp |= data;
6826                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
6827                 break;
6828         case REG_C_1:
6829                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
6830                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
6831                 break;
6832         case REG_C_2:
6833                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
6834                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
6835                 break;
6836         case REG_C_3:
6837                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
6838                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
6839                 break;
6840         case REG_C_4:
6841                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
6842                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
6843                 break;
6844         case REG_C_5:
6845                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
6846                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
6847                 break;
6848         case REG_C_6:
6849                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
6850                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
6851                 break;
6852         case REG_C_7:
6853                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
6854                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
6855                 break;
6856         default:
6857                 MLX5_ASSERT(false);
6858                 break;
6859         }
6860 }
6861
6862 /**
6863  * Add MARK item to matcher
6864  *
6865  * @param[in] dev
6866  *   The device to configure through.
6867  * @param[in, out] matcher
6868  *   Flow matcher.
6869  * @param[in, out] key
6870  *   Flow matcher value.
6871  * @param[in] item
6872  *   Flow pattern to translate.
6873  */
6874 static void
6875 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
6876                             void *matcher, void *key,
6877                             const struct rte_flow_item *item)
6878 {
6879         struct mlx5_priv *priv = dev->data->dev_private;
6880         const struct rte_flow_item_mark *mark;
6881         uint32_t value;
6882         uint32_t mask;
6883
6884         mark = item->mask ? (const void *)item->mask :
6885                             &rte_flow_item_mark_mask;
6886         mask = mark->id & priv->sh->dv_mark_mask;
6887         mark = (const void *)item->spec;
6888         MLX5_ASSERT(mark);
6889         value = mark->id & priv->sh->dv_mark_mask & mask;
6890         if (mask) {
6891                 enum modify_reg reg;
6892
6893                 /* Get the metadata register index for the mark. */
6894                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
6895                 MLX5_ASSERT(reg > 0);
6896                 if (reg == REG_C_0) {
6897                         struct mlx5_priv *priv = dev->data->dev_private;
6898                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6899                         uint32_t shl_c0 = rte_bsf32(msk_c0);
6900
6901                         mask &= msk_c0;
6902                         mask <<= shl_c0;
6903                         value <<= shl_c0;
6904                 }
6905                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
6906         }
6907 }
6908
6909 /**
6910  * Add META item to matcher
6911  *
6912  * @param[in] dev
6913  *   The devich to configure through.
6914  * @param[in, out] matcher
6915  *   Flow matcher.
6916  * @param[in, out] key
6917  *   Flow matcher value.
6918  * @param[in] attr
6919  *   Attributes of flow that includes this item.
6920  * @param[in] item
6921  *   Flow pattern to translate.
6922  */
6923 static void
6924 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
6925                             void *matcher, void *key,
6926                             const struct rte_flow_attr *attr,
6927                             const struct rte_flow_item *item)
6928 {
6929         const struct rte_flow_item_meta *meta_m;
6930         const struct rte_flow_item_meta *meta_v;
6931
6932         meta_m = (const void *)item->mask;
6933         if (!meta_m)
6934                 meta_m = &rte_flow_item_meta_mask;
6935         meta_v = (const void *)item->spec;
6936         if (meta_v) {
6937                 int reg;
6938                 uint32_t value = meta_v->data;
6939                 uint32_t mask = meta_m->data;
6940
6941                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
6942                 if (reg < 0)
6943                         return;
6944                 /*
6945                  * In datapath code there is no endianness
6946                  * coversions for perfromance reasons, all
6947                  * pattern conversions are done in rte_flow.
6948                  */
6949                 value = rte_cpu_to_be_32(value);
6950                 mask = rte_cpu_to_be_32(mask);
6951                 if (reg == REG_C_0) {
6952                         struct mlx5_priv *priv = dev->data->dev_private;
6953                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6954                         uint32_t shl_c0 = rte_bsf32(msk_c0);
6955 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6956                         uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
6957
6958                         value >>= shr_c0;
6959                         mask >>= shr_c0;
6960 #endif
6961                         value <<= shl_c0;
6962                         mask <<= shl_c0;
6963                         MLX5_ASSERT(msk_c0);
6964                         MLX5_ASSERT(!(~msk_c0 & mask));
6965                 }
6966                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
6967         }
6968 }
6969
6970 /**
6971  * Add vport metadata Reg C0 item to matcher
6972  *
6973  * @param[in, out] matcher
6974  *   Flow matcher.
6975  * @param[in, out] key
6976  *   Flow matcher value.
6977  * @param[in] reg
6978  *   Flow pattern to translate.
6979  */
6980 static void
6981 flow_dv_translate_item_meta_vport(void *matcher, void *key,
6982                                   uint32_t value, uint32_t mask)
6983 {
6984         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
6985 }
6986
6987 /**
6988  * Add tag item to matcher
6989  *
6990  * @param[in] dev
6991  *   The devich to configure through.
6992  * @param[in, out] matcher
6993  *   Flow matcher.
6994  * @param[in, out] key
6995  *   Flow matcher value.
6996  * @param[in] item
6997  *   Flow pattern to translate.
6998  */
6999 static void
7000 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
7001                                 void *matcher, void *key,
7002                                 const struct rte_flow_item *item)
7003 {
7004         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
7005         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
7006         uint32_t mask, value;
7007
7008         MLX5_ASSERT(tag_v);
7009         value = tag_v->data;
7010         mask = tag_m ? tag_m->data : UINT32_MAX;
7011         if (tag_v->id == REG_C_0) {
7012                 struct mlx5_priv *priv = dev->data->dev_private;
7013                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
7014                 uint32_t shl_c0 = rte_bsf32(msk_c0);
7015
7016                 mask &= msk_c0;
7017                 mask <<= shl_c0;
7018                 value <<= shl_c0;
7019         }
7020         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
7021 }
7022
7023 /**
7024  * Add TAG item to matcher
7025  *
7026  * @param[in] dev
7027  *   The devich to configure through.
7028  * @param[in, out] matcher
7029  *   Flow matcher.
7030  * @param[in, out] key
7031  *   Flow matcher value.
7032  * @param[in] item
7033  *   Flow pattern to translate.
7034  */
7035 static void
7036 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
7037                            void *matcher, void *key,
7038                            const struct rte_flow_item *item)
7039 {
7040         const struct rte_flow_item_tag *tag_v = item->spec;
7041         const struct rte_flow_item_tag *tag_m = item->mask;
7042         enum modify_reg reg;
7043
7044         MLX5_ASSERT(tag_v);
7045         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
7046         /* Get the metadata register index for the tag. */
7047         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
7048         MLX5_ASSERT(reg > 0);
7049         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
7050 }
7051
7052 /**
7053  * Add source vport match to the specified matcher.
7054  *
7055  * @param[in, out] matcher
7056  *   Flow matcher.
7057  * @param[in, out] key
7058  *   Flow matcher value.
7059  * @param[in] port
7060  *   Source vport value to match
7061  * @param[in] mask
7062  *   Mask
7063  */
7064 static void
7065 flow_dv_translate_item_source_vport(void *matcher, void *key,
7066                                     int16_t port, uint16_t mask)
7067 {
7068         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7069         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7070
7071         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
7072         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
7073 }
7074
7075 /**
7076  * Translate port-id item to eswitch match on  port-id.
7077  *
7078  * @param[in] dev
7079  *   The devich to configure through.
7080  * @param[in, out] matcher
7081  *   Flow matcher.
7082  * @param[in, out] key
7083  *   Flow matcher value.
7084  * @param[in] item
7085  *   Flow pattern to translate.
7086  *
7087  * @return
7088  *   0 on success, a negative errno value otherwise.
7089  */
7090 static int
7091 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
7092                                void *key, const struct rte_flow_item *item)
7093 {
7094         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
7095         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
7096         struct mlx5_priv *priv;
7097         uint16_t mask, id;
7098
7099         mask = pid_m ? pid_m->id : 0xffff;
7100         id = pid_v ? pid_v->id : dev->data->port_id;
7101         priv = mlx5_port_to_eswitch_info(id, item == NULL);
7102         if (!priv)
7103                 return -rte_errno;
7104         /* Translate to vport field or to metadata, depending on mode. */
7105         if (priv->vport_meta_mask)
7106                 flow_dv_translate_item_meta_vport(matcher, key,
7107                                                   priv->vport_meta_tag,
7108                                                   priv->vport_meta_mask);
7109         else
7110                 flow_dv_translate_item_source_vport(matcher, key,
7111                                                     priv->vport_id, mask);
7112         return 0;
7113 }
7114
7115 /**
7116  * Add ICMP6 item to matcher and to the value.
7117  *
7118  * @param[in, out] matcher
7119  *   Flow matcher.
7120  * @param[in, out] key
7121  *   Flow matcher value.
7122  * @param[in] item
7123  *   Flow pattern to translate.
7124  * @param[in] inner
7125  *   Item is inner pattern.
7126  */
7127 static void
7128 flow_dv_translate_item_icmp6(void *matcher, void *key,
7129                               const struct rte_flow_item *item,
7130                               int inner)
7131 {
7132         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
7133         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
7134         void *headers_m;
7135         void *headers_v;
7136         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7137                                      misc_parameters_3);
7138         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7139         if (inner) {
7140                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7141                                          inner_headers);
7142                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7143         } else {
7144                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7145                                          outer_headers);
7146                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7147         }
7148         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7149         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
7150         if (!icmp6_v)
7151                 return;
7152         if (!icmp6_m)
7153                 icmp6_m = &rte_flow_item_icmp6_mask;
7154         /*
7155          * Force flow only to match the non-fragmented IPv6 ICMPv6 packets.
7156          * If only the protocol is specified, no need to match the frag.
7157          */
7158         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
7159         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
7160         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
7161         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
7162                  icmp6_v->type & icmp6_m->type);
7163         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
7164         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
7165                  icmp6_v->code & icmp6_m->code);
7166 }
7167
7168 /**
7169  * Add ICMP item to matcher and to the value.
7170  *
7171  * @param[in, out] matcher
7172  *   Flow matcher.
7173  * @param[in, out] key
7174  *   Flow matcher value.
7175  * @param[in] item
7176  *   Flow pattern to translate.
7177  * @param[in] inner
7178  *   Item is inner pattern.
7179  */
7180 static void
7181 flow_dv_translate_item_icmp(void *matcher, void *key,
7182                             const struct rte_flow_item *item,
7183                             int inner)
7184 {
7185         const struct rte_flow_item_icmp *icmp_m = item->mask;
7186         const struct rte_flow_item_icmp *icmp_v = item->spec;
7187         void *headers_m;
7188         void *headers_v;
7189         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7190                                      misc_parameters_3);
7191         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7192         if (inner) {
7193                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7194                                          inner_headers);
7195                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7196         } else {
7197                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7198                                          outer_headers);
7199                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7200         }
7201         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7202         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
7203         if (!icmp_v)
7204                 return;
7205         if (!icmp_m)
7206                 icmp_m = &rte_flow_item_icmp_mask;
7207         /*
7208          * Force flow only to match the non-fragmented IPv4 ICMP packets.
7209          * If only the protocol is specified, no need to match the frag.
7210          */
7211         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
7212         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
7213         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
7214                  icmp_m->hdr.icmp_type);
7215         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
7216                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
7217         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
7218                  icmp_m->hdr.icmp_code);
7219         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
7220                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
7221 }
7222
7223 /**
7224  * Add GTP item to matcher and to the value.
7225  *
7226  * @param[in, out] matcher
7227  *   Flow matcher.
7228  * @param[in, out] key
7229  *   Flow matcher value.
7230  * @param[in] item
7231  *   Flow pattern to translate.
7232  * @param[in] inner
7233  *   Item is inner pattern.
7234  */
7235 static void
7236 flow_dv_translate_item_gtp(void *matcher, void *key,
7237                            const struct rte_flow_item *item, int inner)
7238 {
7239         const struct rte_flow_item_gtp *gtp_m = item->mask;
7240         const struct rte_flow_item_gtp *gtp_v = item->spec;
7241         void *headers_m;
7242         void *headers_v;
7243         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7244                                      misc_parameters_3);
7245         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7246         uint16_t dport = RTE_GTPU_UDP_PORT;
7247
7248         if (inner) {
7249                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7250                                          inner_headers);
7251                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7252         } else {
7253                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7254                                          outer_headers);
7255                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7256         }
7257         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7258                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7259                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7260         }
7261         if (!gtp_v)
7262                 return;
7263         if (!gtp_m)
7264                 gtp_m = &rte_flow_item_gtp_mask;
7265         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
7266                  gtp_m->v_pt_rsv_flags);
7267         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
7268                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
7269         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
7270         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
7271                  gtp_v->msg_type & gtp_m->msg_type);
7272         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
7273                  rte_be_to_cpu_32(gtp_m->teid));
7274         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
7275                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
7276 }
7277
7278 /**
7279  * Add eCPRI item to matcher and to the value.
7280  *
7281  * @param[in] dev
7282  *   The devich to configure through.
7283  * @param[in, out] matcher
7284  *   Flow matcher.
7285  * @param[in, out] key
7286  *   Flow matcher value.
7287  * @param[in] item
7288  *   Flow pattern to translate.
7289  * @param[in] samples
7290  *   Sample IDs to be used in the matching.
7291  */
7292 static void
7293 flow_dv_translate_item_ecpri(struct rte_eth_dev *dev, void *matcher,
7294                              void *key, const struct rte_flow_item *item)
7295 {
7296         struct mlx5_priv *priv = dev->data->dev_private;
7297         const struct rte_flow_item_ecpri *ecpri_m = item->mask;
7298         const struct rte_flow_item_ecpri *ecpri_v = item->spec;
7299         void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher,
7300                                      misc_parameters_4);
7301         void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4);
7302         uint32_t *samples;
7303         void *dw_m;
7304         void *dw_v;
7305
7306         if (!ecpri_v)
7307                 return;
7308         if (!ecpri_m)
7309                 ecpri_m = &rte_flow_item_ecpri_mask;
7310         /*
7311          * Maximal four DW samples are supported in a single matching now.
7312          * Two are used now for a eCPRI matching:
7313          * 1. Type: one byte, mask should be 0x00ff0000 in network order
7314          * 2. ID of a message: one or two bytes, mask 0xffff0000 or 0xff000000
7315          *    if any.
7316          */
7317         if (!ecpri_m->hdr.common.u32)
7318                 return;
7319         samples = priv->sh->fp[MLX5_FLEX_PARSER_ECPRI_0].ids;
7320         /* Need to take the whole DW as the mask to fill the entry. */
7321         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
7322                             prog_sample_field_value_0);
7323         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
7324                             prog_sample_field_value_0);
7325         /* Already big endian (network order) in the header. */
7326         *(uint32_t *)dw_m = ecpri_m->hdr.common.u32;
7327         *(uint32_t *)dw_v = ecpri_v->hdr.common.u32;
7328         /* Sample#0, used for matching type, offset 0. */
7329         MLX5_SET(fte_match_set_misc4, misc4_m,
7330                  prog_sample_field_id_0, samples[0]);
7331         /* It makes no sense to set the sample ID in the mask field. */
7332         MLX5_SET(fte_match_set_misc4, misc4_v,
7333                  prog_sample_field_id_0, samples[0]);
7334         /*
7335          * Checking if message body part needs to be matched.
7336          * Some wildcard rules only matching type field should be supported.
7337          */
7338         if (ecpri_m->hdr.dummy[0]) {
7339                 switch (ecpri_v->hdr.common.type) {
7340                 case RTE_ECPRI_MSG_TYPE_IQ_DATA:
7341                 case RTE_ECPRI_MSG_TYPE_RTC_CTRL:
7342                 case RTE_ECPRI_MSG_TYPE_DLY_MSR:
7343                         dw_m = MLX5_ADDR_OF(fte_match_set_misc4, misc4_m,
7344                                             prog_sample_field_value_1);
7345                         dw_v = MLX5_ADDR_OF(fte_match_set_misc4, misc4_v,
7346                                             prog_sample_field_value_1);
7347                         *(uint32_t *)dw_m = ecpri_m->hdr.dummy[0];
7348                         *(uint32_t *)dw_v = ecpri_v->hdr.dummy[0];
7349                         /* Sample#1, to match message body, offset 4. */
7350                         MLX5_SET(fte_match_set_misc4, misc4_m,
7351                                  prog_sample_field_id_1, samples[1]);
7352                         MLX5_SET(fte_match_set_misc4, misc4_v,
7353                                  prog_sample_field_id_1, samples[1]);
7354                         break;
7355                 default:
7356                         /* Others, do not match any sample ID. */
7357                         break;
7358                 }
7359         }
7360 }
7361
7362 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
7363
7364 #define HEADER_IS_ZERO(match_criteria, headers)                              \
7365         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
7366                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
7367
7368 /**
7369  * Calculate flow matcher enable bitmap.
7370  *
7371  * @param match_criteria
7372  *   Pointer to flow matcher criteria.
7373  *
7374  * @return
7375  *   Bitmap of enabled fields.
7376  */
7377 static uint8_t
7378 flow_dv_matcher_enable(uint32_t *match_criteria)
7379 {
7380         uint8_t match_criteria_enable;
7381
7382         match_criteria_enable =
7383                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
7384                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
7385         match_criteria_enable |=
7386                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
7387                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
7388         match_criteria_enable |=
7389                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
7390                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
7391         match_criteria_enable |=
7392                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
7393                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
7394         match_criteria_enable |=
7395                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
7396                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
7397         match_criteria_enable |=
7398                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_4)) <<
7399                 MLX5_MATCH_CRITERIA_ENABLE_MISC4_BIT;
7400         return match_criteria_enable;
7401 }
7402
7403
7404 /**
7405  * Get a flow table.
7406  *
7407  * @param[in, out] dev
7408  *   Pointer to rte_eth_dev structure.
7409  * @param[in] table_id
7410  *   Table id to use.
7411  * @param[in] egress
7412  *   Direction of the table.
7413  * @param[in] transfer
7414  *   E-Switch or NIC flow.
7415  * @param[out] error
7416  *   pointer to error structure.
7417  *
7418  * @return
7419  *   Returns tables resource based on the index, NULL in case of failed.
7420  */
7421 static struct mlx5_flow_tbl_resource *
7422 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
7423                          uint32_t table_id, uint8_t egress,
7424                          uint8_t transfer,
7425                          struct rte_flow_error *error)
7426 {
7427         struct mlx5_priv *priv = dev->data->dev_private;
7428         struct mlx5_dev_ctx_shared *sh = priv->sh;
7429         struct mlx5_flow_tbl_resource *tbl;
7430         union mlx5_flow_tbl_key table_key = {
7431                 {
7432                         .table_id = table_id,
7433                         .reserved = 0,
7434                         .domain = !!transfer,
7435                         .direction = !!egress,
7436                 }
7437         };
7438         struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
7439                                                          table_key.v64);
7440         struct mlx5_flow_tbl_data_entry *tbl_data;
7441         uint32_t idx = 0;
7442         int ret;
7443         void *domain;
7444
7445         if (pos) {
7446                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
7447                                         entry);
7448                 tbl = &tbl_data->tbl;
7449                 rte_atomic32_inc(&tbl->refcnt);
7450                 return tbl;
7451         }
7452         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
7453         if (!tbl_data) {
7454                 rte_flow_error_set(error, ENOMEM,
7455                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7456                                    NULL,
7457                                    "cannot allocate flow table data entry");
7458                 return NULL;
7459         }
7460         tbl_data->idx = idx;
7461         tbl = &tbl_data->tbl;
7462         pos = &tbl_data->entry;
7463         if (transfer)
7464                 domain = sh->fdb_domain;
7465         else if (egress)
7466                 domain = sh->tx_domain;
7467         else
7468                 domain = sh->rx_domain;
7469         ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj);
7470         if (ret) {
7471                 rte_flow_error_set(error, ENOMEM,
7472                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7473                                    NULL, "cannot create flow table object");
7474                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7475                 return NULL;
7476         }
7477         /*
7478          * No multi-threads now, but still better to initialize the reference
7479          * count before insert it into the hash list.
7480          */
7481         rte_atomic32_init(&tbl->refcnt);
7482         /* Jump action reference count is initialized here. */
7483         rte_atomic32_init(&tbl_data->jump.refcnt);
7484         pos->key = table_key.v64;
7485         ret = mlx5_hlist_insert(sh->flow_tbls, pos);
7486         if (ret < 0) {
7487                 rte_flow_error_set(error, -ret,
7488                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7489                                    "cannot insert flow table data entry");
7490                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
7491                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7492         }
7493         rte_atomic32_inc(&tbl->refcnt);
7494         return tbl;
7495 }
7496
7497 /**
7498  * Release a flow table.
7499  *
7500  * @param[in] dev
7501  *   Pointer to rte_eth_dev structure.
7502  * @param[in] tbl
7503  *   Table resource to be released.
7504  *
7505  * @return
7506  *   Returns 0 if table was released, else return 1;
7507  */
7508 static int
7509 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
7510                              struct mlx5_flow_tbl_resource *tbl)
7511 {
7512         struct mlx5_priv *priv = dev->data->dev_private;
7513         struct mlx5_dev_ctx_shared *sh = priv->sh;
7514         struct mlx5_flow_tbl_data_entry *tbl_data =
7515                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7516
7517         if (!tbl)
7518                 return 0;
7519         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
7520                 struct mlx5_hlist_entry *pos = &tbl_data->entry;
7521
7522                 mlx5_flow_os_destroy_flow_tbl(tbl->obj);
7523                 tbl->obj = NULL;
7524                 /* remove the entry from the hash list and free memory. */
7525                 mlx5_hlist_remove(sh->flow_tbls, pos);
7526                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
7527                                 tbl_data->idx);
7528                 return 0;
7529         }
7530         return 1;
7531 }
7532
7533 /**
7534  * Register the flow matcher.
7535  *
7536  * @param[in, out] dev
7537  *   Pointer to rte_eth_dev structure.
7538  * @param[in, out] matcher
7539  *   Pointer to flow matcher.
7540  * @param[in, out] key
7541  *   Pointer to flow table key.
7542  * @parm[in, out] dev_flow
7543  *   Pointer to the dev_flow.
7544  * @param[out] error
7545  *   pointer to error structure.
7546  *
7547  * @return
7548  *   0 on success otherwise -errno and errno is set.
7549  */
7550 static int
7551 flow_dv_matcher_register(struct rte_eth_dev *dev,
7552                          struct mlx5_flow_dv_matcher *matcher,
7553                          union mlx5_flow_tbl_key *key,
7554                          struct mlx5_flow *dev_flow,
7555                          struct rte_flow_error *error)
7556 {
7557         struct mlx5_priv *priv = dev->data->dev_private;
7558         struct mlx5_dev_ctx_shared *sh = priv->sh;
7559         struct mlx5_flow_dv_matcher *cache_matcher;
7560         struct mlx5dv_flow_matcher_attr dv_attr = {
7561                 .type = IBV_FLOW_ATTR_NORMAL,
7562                 .match_mask = (void *)&matcher->mask,
7563         };
7564         struct mlx5_flow_tbl_resource *tbl;
7565         struct mlx5_flow_tbl_data_entry *tbl_data;
7566         int ret;
7567
7568         tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
7569                                        key->domain, error);
7570         if (!tbl)
7571                 return -rte_errno;      /* No need to refill the error info */
7572         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7573         /* Lookup from cache. */
7574         LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
7575                 if (matcher->crc == cache_matcher->crc &&
7576                     matcher->priority == cache_matcher->priority &&
7577                     !memcmp((const void *)matcher->mask.buf,
7578                             (const void *)cache_matcher->mask.buf,
7579                             cache_matcher->mask.size)) {
7580                         DRV_LOG(DEBUG,
7581                                 "%s group %u priority %hd use %s "
7582                                 "matcher %p: refcnt %d++",
7583                                 key->domain ? "FDB" : "NIC", key->table_id,
7584                                 cache_matcher->priority,
7585                                 key->direction ? "tx" : "rx",
7586                                 (void *)cache_matcher,
7587                                 rte_atomic32_read(&cache_matcher->refcnt));
7588                         rte_atomic32_inc(&cache_matcher->refcnt);
7589                         dev_flow->handle->dvh.matcher = cache_matcher;
7590                         /* old matcher should not make the table ref++. */
7591                         flow_dv_tbl_resource_release(dev, tbl);
7592                         return 0;
7593                 }
7594         }
7595         /* Register new matcher. */
7596         cache_matcher = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache_matcher), 0,
7597                                     SOCKET_ID_ANY);
7598         if (!cache_matcher) {
7599                 flow_dv_tbl_resource_release(dev, tbl);
7600                 return rte_flow_error_set(error, ENOMEM,
7601                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7602                                           "cannot allocate matcher memory");
7603         }
7604         *cache_matcher = *matcher;
7605         dv_attr.match_criteria_enable =
7606                 flow_dv_matcher_enable(cache_matcher->mask.buf);
7607         dv_attr.priority = matcher->priority;
7608         if (key->direction)
7609                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
7610         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj,
7611                                                &cache_matcher->matcher_object);
7612         if (ret) {
7613                 mlx5_free(cache_matcher);
7614 #ifdef HAVE_MLX5DV_DR
7615                 flow_dv_tbl_resource_release(dev, tbl);
7616 #endif
7617                 return rte_flow_error_set(error, ENOMEM,
7618                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7619                                           NULL, "cannot create matcher");
7620         }
7621         /* Save the table information */
7622         cache_matcher->tbl = tbl;
7623         rte_atomic32_init(&cache_matcher->refcnt);
7624         /* only matcher ref++, table ref++ already done above in get API. */
7625         rte_atomic32_inc(&cache_matcher->refcnt);
7626         LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
7627         dev_flow->handle->dvh.matcher = cache_matcher;
7628         DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
7629                 key->domain ? "FDB" : "NIC", key->table_id,
7630                 cache_matcher->priority,
7631                 key->direction ? "tx" : "rx", (void *)cache_matcher,
7632                 rte_atomic32_read(&cache_matcher->refcnt));
7633         return 0;
7634 }
7635
7636 /**
7637  * Find existing tag resource or create and register a new one.
7638  *
7639  * @param dev[in, out]
7640  *   Pointer to rte_eth_dev structure.
7641  * @param[in, out] tag_be24
7642  *   Tag value in big endian then R-shift 8.
7643  * @parm[in, out] dev_flow
7644  *   Pointer to the dev_flow.
7645  * @param[out] error
7646  *   pointer to error structure.
7647  *
7648  * @return
7649  *   0 on success otherwise -errno and errno is set.
7650  */
7651 static int
7652 flow_dv_tag_resource_register
7653                         (struct rte_eth_dev *dev,
7654                          uint32_t tag_be24,
7655                          struct mlx5_flow *dev_flow,
7656                          struct rte_flow_error *error)
7657 {
7658         struct mlx5_priv *priv = dev->data->dev_private;
7659         struct mlx5_dev_ctx_shared *sh = priv->sh;
7660         struct mlx5_flow_dv_tag_resource *cache_resource;
7661         struct mlx5_hlist_entry *entry;
7662         int ret;
7663
7664         /* Lookup a matching resource from cache. */
7665         entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
7666         if (entry) {
7667                 cache_resource = container_of
7668                         (entry, struct mlx5_flow_dv_tag_resource, entry);
7669                 rte_atomic32_inc(&cache_resource->refcnt);
7670                 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
7671                 dev_flow->dv.tag_resource = cache_resource;
7672                 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
7673                         (void *)cache_resource,
7674                         rte_atomic32_read(&cache_resource->refcnt));
7675                 return 0;
7676         }
7677         /* Register new resource. */
7678         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG],
7679                                        &dev_flow->handle->dvh.rix_tag);
7680         if (!cache_resource)
7681                 return rte_flow_error_set(error, ENOMEM,
7682                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7683                                           "cannot allocate resource memory");
7684         cache_resource->entry.key = (uint64_t)tag_be24;
7685         ret = mlx5_flow_os_create_flow_action_tag(tag_be24,
7686                                                   &cache_resource->action);
7687         if (ret) {
7688                 mlx5_free(cache_resource);
7689                 return rte_flow_error_set(error, ENOMEM,
7690                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7691                                           NULL, "cannot create action");
7692         }
7693         rte_atomic32_init(&cache_resource->refcnt);
7694         rte_atomic32_inc(&cache_resource->refcnt);
7695         if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
7696                 mlx5_flow_os_destroy_flow_action(cache_resource->action);
7697                 mlx5_free(cache_resource);
7698                 return rte_flow_error_set(error, EEXIST,
7699                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7700                                           NULL, "cannot insert tag");
7701         }
7702         dev_flow->dv.tag_resource = cache_resource;
7703         DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
7704                 (void *)cache_resource,
7705                 rte_atomic32_read(&cache_resource->refcnt));
7706         return 0;
7707 }
7708
7709 /**
7710  * Release the tag.
7711  *
7712  * @param dev
7713  *   Pointer to Ethernet device.
7714  * @param tag_idx
7715  *   Tag index.
7716  *
7717  * @return
7718  *   1 while a reference on it exists, 0 when freed.
7719  */
7720 static int
7721 flow_dv_tag_release(struct rte_eth_dev *dev,
7722                     uint32_t tag_idx)
7723 {
7724         struct mlx5_priv *priv = dev->data->dev_private;
7725         struct mlx5_dev_ctx_shared *sh = priv->sh;
7726         struct mlx5_flow_dv_tag_resource *tag;
7727
7728         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
7729         if (!tag)
7730                 return 0;
7731         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
7732                 dev->data->port_id, (void *)tag,
7733                 rte_atomic32_read(&tag->refcnt));
7734         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
7735                 claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
7736                 mlx5_hlist_remove(sh->tag_table, &tag->entry);
7737                 DRV_LOG(DEBUG, "port %u tag %p: removed",
7738                         dev->data->port_id, (void *)tag);
7739                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
7740                 return 0;
7741         }
7742         return 1;
7743 }
7744
7745 /**
7746  * Translate port ID action to vport.
7747  *
7748  * @param[in] dev
7749  *   Pointer to rte_eth_dev structure.
7750  * @param[in] action
7751  *   Pointer to the port ID action.
7752  * @param[out] dst_port_id
7753  *   The target port ID.
7754  * @param[out] error
7755  *   Pointer to the error structure.
7756  *
7757  * @return
7758  *   0 on success, a negative errno value otherwise and rte_errno is set.
7759  */
7760 static int
7761 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
7762                                  const struct rte_flow_action *action,
7763                                  uint32_t *dst_port_id,
7764                                  struct rte_flow_error *error)
7765 {
7766         uint32_t port;
7767         struct mlx5_priv *priv;
7768         const struct rte_flow_action_port_id *conf =
7769                         (const struct rte_flow_action_port_id *)action->conf;
7770
7771         port = conf->original ? dev->data->port_id : conf->id;
7772         priv = mlx5_port_to_eswitch_info(port, false);
7773         if (!priv)
7774                 return rte_flow_error_set(error, -rte_errno,
7775                                           RTE_FLOW_ERROR_TYPE_ACTION,
7776                                           NULL,
7777                                           "No eswitch info was found for port");
7778 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
7779         /*
7780          * This parameter is transferred to
7781          * mlx5dv_dr_action_create_dest_ib_port().
7782          */
7783         *dst_port_id = priv->dev_port;
7784 #else
7785         /*
7786          * Legacy mode, no LAG configurations is supported.
7787          * This parameter is transferred to
7788          * mlx5dv_dr_action_create_dest_vport().
7789          */
7790         *dst_port_id = priv->vport_id;
7791 #endif
7792         return 0;
7793 }
7794
7795 /**
7796  * Create a counter with aging configuration.
7797  *
7798  * @param[in] dev
7799  *   Pointer to rte_eth_dev structure.
7800  * @param[out] count
7801  *   Pointer to the counter action configuration.
7802  * @param[in] age
7803  *   Pointer to the aging action configuration.
7804  *
7805  * @return
7806  *   Index to flow counter on success, 0 otherwise.
7807  */
7808 static uint32_t
7809 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
7810                                 struct mlx5_flow *dev_flow,
7811                                 const struct rte_flow_action_count *count,
7812                                 const struct rte_flow_action_age *age)
7813 {
7814         uint32_t counter;
7815         struct mlx5_age_param *age_param;
7816
7817         counter = flow_dv_counter_alloc(dev,
7818                                 count ? count->shared : 0,
7819                                 count ? count->id : 0,
7820                                 dev_flow->dv.group, !!age);
7821         if (!counter || age == NULL)
7822                 return counter;
7823         age_param  = flow_dv_counter_idx_get_age(dev, counter);
7824         /*
7825          * The counter age accuracy may have a bit delay. Have 3/4
7826          * second bias on the timeount in order to let it age in time.
7827          */
7828         age_param->context = age->context ? age->context :
7829                 (void *)(uintptr_t)(dev_flow->flow_idx);
7830         /*
7831          * The counter age accuracy may have a bit delay. Have 3/4
7832          * second bias on the timeount in order to let it age in time.
7833          */
7834         age_param->timeout = age->timeout * 10 - MLX5_AGING_TIME_DELAY;
7835         /* Set expire time in unit of 0.1 sec. */
7836         age_param->port_id = dev->data->port_id;
7837         age_param->expire = age_param->timeout +
7838                         rte_rdtsc() / (rte_get_tsc_hz() / 10);
7839         rte_atomic16_set(&age_param->state, AGE_CANDIDATE);
7840         return counter;
7841 }
7842 /**
7843  * Add Tx queue matcher
7844  *
7845  * @param[in] dev
7846  *   Pointer to the dev struct.
7847  * @param[in, out] matcher
7848  *   Flow matcher.
7849  * @param[in, out] key
7850  *   Flow matcher value.
7851  * @param[in] item
7852  *   Flow pattern to translate.
7853  * @param[in] inner
7854  *   Item is inner pattern.
7855  */
7856 static void
7857 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
7858                                 void *matcher, void *key,
7859                                 const struct rte_flow_item *item)
7860 {
7861         const struct mlx5_rte_flow_item_tx_queue *queue_m;
7862         const struct mlx5_rte_flow_item_tx_queue *queue_v;
7863         void *misc_m =
7864                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7865         void *misc_v =
7866                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7867         struct mlx5_txq_ctrl *txq;
7868         uint32_t queue;
7869
7870
7871         queue_m = (const void *)item->mask;
7872         if (!queue_m)
7873                 return;
7874         queue_v = (const void *)item->spec;
7875         if (!queue_v)
7876                 return;
7877         txq = mlx5_txq_get(dev, queue_v->queue);
7878         if (!txq)
7879                 return;
7880         queue = txq->obj->sq->id;
7881         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
7882         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
7883                  queue & queue_m->queue);
7884         mlx5_txq_release(dev, queue_v->queue);
7885 }
7886
7887 /**
7888  * Set the hash fields according to the @p flow information.
7889  *
7890  * @param[in] dev_flow
7891  *   Pointer to the mlx5_flow.
7892  * @param[in] rss_desc
7893  *   Pointer to the mlx5_flow_rss_desc.
7894  */
7895 static void
7896 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
7897                        struct mlx5_flow_rss_desc *rss_desc)
7898 {
7899         uint64_t items = dev_flow->handle->layers;
7900         int rss_inner = 0;
7901         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
7902
7903         dev_flow->hash_fields = 0;
7904 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
7905         if (rss_desc->level >= 2) {
7906                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
7907                 rss_inner = 1;
7908         }
7909 #endif
7910         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
7911             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
7912                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
7913                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
7914                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
7915                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
7916                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
7917                         else
7918                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
7919                 }
7920         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
7921                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
7922                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
7923                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
7924                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
7925                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
7926                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
7927                         else
7928                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
7929                 }
7930         }
7931         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
7932             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
7933                 if (rss_types & ETH_RSS_UDP) {
7934                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
7935                                 dev_flow->hash_fields |=
7936                                                 IBV_RX_HASH_SRC_PORT_UDP;
7937                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
7938                                 dev_flow->hash_fields |=
7939                                                 IBV_RX_HASH_DST_PORT_UDP;
7940                         else
7941                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
7942                 }
7943         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
7944                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
7945                 if (rss_types & ETH_RSS_TCP) {
7946                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
7947                                 dev_flow->hash_fields |=
7948                                                 IBV_RX_HASH_SRC_PORT_TCP;
7949                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
7950                                 dev_flow->hash_fields |=
7951                                                 IBV_RX_HASH_DST_PORT_TCP;
7952                         else
7953                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
7954                 }
7955         }
7956 }
7957
7958 /**
7959  * Fill the flow with DV spec, lock free
7960  * (mutex should be acquired by caller).
7961  *
7962  * @param[in] dev
7963  *   Pointer to rte_eth_dev structure.
7964  * @param[in, out] dev_flow
7965  *   Pointer to the sub flow.
7966  * @param[in] attr
7967  *   Pointer to the flow attributes.
7968  * @param[in] items
7969  *   Pointer to the list of items.
7970  * @param[in] actions
7971  *   Pointer to the list of actions.
7972  * @param[out] error
7973  *   Pointer to the error structure.
7974  *
7975  * @return
7976  *   0 on success, a negative errno value otherwise and rte_errno is set.
7977  */
7978 static int
7979 __flow_dv_translate(struct rte_eth_dev *dev,
7980                     struct mlx5_flow *dev_flow,
7981                     const struct rte_flow_attr *attr,
7982                     const struct rte_flow_item items[],
7983                     const struct rte_flow_action actions[],
7984                     struct rte_flow_error *error)
7985 {
7986         struct mlx5_priv *priv = dev->data->dev_private;
7987         struct mlx5_dev_config *dev_conf = &priv->config;
7988         struct rte_flow *flow = dev_flow->flow;
7989         struct mlx5_flow_handle *handle = dev_flow->handle;
7990         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
7991                                               priv->rss_desc)
7992                                               [!!priv->flow_nested_idx];
7993         uint64_t item_flags = 0;
7994         uint64_t last_item = 0;
7995         uint64_t action_flags = 0;
7996         uint64_t priority = attr->priority;
7997         struct mlx5_flow_dv_matcher matcher = {
7998                 .mask = {
7999                         .size = sizeof(matcher.mask.buf) -
8000                                 MLX5_ST_SZ_BYTES(fte_match_set_misc4),
8001                 },
8002         };
8003         int actions_n = 0;
8004         bool actions_end = false;
8005         union {
8006                 struct mlx5_flow_dv_modify_hdr_resource res;
8007                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
8008                             sizeof(struct mlx5_modification_cmd) *
8009                             (MLX5_MAX_MODIFY_NUM + 1)];
8010         } mhdr_dummy;
8011         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
8012         const struct rte_flow_action_count *count = NULL;
8013         const struct rte_flow_action_age *age = NULL;
8014         union flow_dv_attr flow_attr = { .attr = 0 };
8015         uint32_t tag_be;
8016         union mlx5_flow_tbl_key tbl_key;
8017         uint32_t modify_action_position = UINT32_MAX;
8018         void *match_mask = matcher.mask.buf;
8019         void *match_value = dev_flow->dv.value.buf;
8020         uint8_t next_protocol = 0xff;
8021         struct rte_vlan_hdr vlan = { 0 };
8022         uint32_t table;
8023         int ret = 0;
8024
8025         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
8026                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
8027         ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
8028                                        !!priv->fdb_def_rule, &table, error);
8029         if (ret)
8030                 return ret;
8031         dev_flow->dv.group = table;
8032         if (attr->transfer)
8033                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
8034         if (priority == MLX5_FLOW_PRIO_RSVD)
8035                 priority = dev_conf->flow_prio - 1;
8036         /* number of actions must be set to 0 in case of dirty stack. */
8037         mhdr_res->actions_num = 0;
8038         for (; !actions_end ; actions++) {
8039                 const struct rte_flow_action_queue *queue;
8040                 const struct rte_flow_action_rss *rss;
8041                 const struct rte_flow_action *action = actions;
8042                 const uint8_t *rss_key;
8043                 const struct rte_flow_action_jump *jump_data;
8044                 const struct rte_flow_action_meter *mtr;
8045                 struct mlx5_flow_tbl_resource *tbl;
8046                 uint32_t port_id = 0;
8047                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
8048                 int action_type = actions->type;
8049                 const struct rte_flow_action *found_action = NULL;
8050                 struct mlx5_flow_meter *fm = NULL;
8051
8052                 if (!mlx5_flow_os_action_supported(action_type))
8053                         return rte_flow_error_set(error, ENOTSUP,
8054                                                   RTE_FLOW_ERROR_TYPE_ACTION,
8055                                                   actions,
8056                                                   "action not supported");
8057                 switch (action_type) {
8058                 case RTE_FLOW_ACTION_TYPE_VOID:
8059                         break;
8060                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
8061                         if (flow_dv_translate_action_port_id(dev, action,
8062                                                              &port_id, error))
8063                                 return -rte_errno;
8064                         port_id_resource.port_id = port_id;
8065                         MLX5_ASSERT(!handle->rix_port_id_action);
8066                         if (flow_dv_port_id_action_resource_register
8067                             (dev, &port_id_resource, dev_flow, error))
8068                                 return -rte_errno;
8069                         dev_flow->dv.actions[actions_n++] =
8070                                         dev_flow->dv.port_id_action->action;
8071                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
8072                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
8073                         break;
8074                 case RTE_FLOW_ACTION_TYPE_FLAG:
8075                         action_flags |= MLX5_FLOW_ACTION_FLAG;
8076                         dev_flow->handle->mark = 1;
8077                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
8078                                 struct rte_flow_action_mark mark = {
8079                                         .id = MLX5_FLOW_MARK_DEFAULT,
8080                                 };
8081
8082                                 if (flow_dv_convert_action_mark(dev, &mark,
8083                                                                 mhdr_res,
8084                                                                 error))
8085                                         return -rte_errno;
8086                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
8087                                 break;
8088                         }
8089                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
8090                         /*
8091                          * Only one FLAG or MARK is supported per device flow
8092                          * right now. So the pointer to the tag resource must be
8093                          * zero before the register process.
8094                          */
8095                         MLX5_ASSERT(!handle->dvh.rix_tag);
8096                         if (flow_dv_tag_resource_register(dev, tag_be,
8097                                                           dev_flow, error))
8098                                 return -rte_errno;
8099                         MLX5_ASSERT(dev_flow->dv.tag_resource);
8100                         dev_flow->dv.actions[actions_n++] =
8101                                         dev_flow->dv.tag_resource->action;
8102                         break;
8103                 case RTE_FLOW_ACTION_TYPE_MARK:
8104                         action_flags |= MLX5_FLOW_ACTION_MARK;
8105                         dev_flow->handle->mark = 1;
8106                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
8107                                 const struct rte_flow_action_mark *mark =
8108                                         (const struct rte_flow_action_mark *)
8109                                                 actions->conf;
8110
8111                                 if (flow_dv_convert_action_mark(dev, mark,
8112                                                                 mhdr_res,
8113                                                                 error))
8114                                         return -rte_errno;
8115                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
8116                                 break;
8117                         }
8118                         /* Fall-through */
8119                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
8120                         /* Legacy (non-extensive) MARK action. */
8121                         tag_be = mlx5_flow_mark_set
8122                               (((const struct rte_flow_action_mark *)
8123                                (actions->conf))->id);
8124                         MLX5_ASSERT(!handle->dvh.rix_tag);
8125                         if (flow_dv_tag_resource_register(dev, tag_be,
8126                                                           dev_flow, error))
8127                                 return -rte_errno;
8128                         MLX5_ASSERT(dev_flow->dv.tag_resource);
8129                         dev_flow->dv.actions[actions_n++] =
8130                                         dev_flow->dv.tag_resource->action;
8131                         break;
8132                 case RTE_FLOW_ACTION_TYPE_SET_META:
8133                         if (flow_dv_convert_action_set_meta
8134                                 (dev, mhdr_res, attr,
8135                                  (const struct rte_flow_action_set_meta *)
8136                                   actions->conf, error))
8137                                 return -rte_errno;
8138                         action_flags |= MLX5_FLOW_ACTION_SET_META;
8139                         break;
8140                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
8141                         if (flow_dv_convert_action_set_tag
8142                                 (dev, mhdr_res,
8143                                  (const struct rte_flow_action_set_tag *)
8144                                   actions->conf, error))
8145                                 return -rte_errno;
8146                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8147                         break;
8148                 case RTE_FLOW_ACTION_TYPE_DROP:
8149                         action_flags |= MLX5_FLOW_ACTION_DROP;
8150                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
8151                         break;
8152                 case RTE_FLOW_ACTION_TYPE_QUEUE:
8153                         queue = actions->conf;
8154                         rss_desc->queue_num = 1;
8155                         rss_desc->queue[0] = queue->index;
8156                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
8157                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
8158                         break;
8159                 case RTE_FLOW_ACTION_TYPE_RSS:
8160                         rss = actions->conf;
8161                         memcpy(rss_desc->queue, rss->queue,
8162                                rss->queue_num * sizeof(uint16_t));
8163                         rss_desc->queue_num = rss->queue_num;
8164                         /* NULL RSS key indicates default RSS key. */
8165                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
8166                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
8167                         /*
8168                          * rss->level and rss.types should be set in advance
8169                          * when expanding items for RSS.
8170                          */
8171                         action_flags |= MLX5_FLOW_ACTION_RSS;
8172                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
8173                         break;
8174                 case RTE_FLOW_ACTION_TYPE_AGE:
8175                 case RTE_FLOW_ACTION_TYPE_COUNT:
8176                         if (!dev_conf->devx) {
8177                                 return rte_flow_error_set
8178                                               (error, ENOTSUP,
8179                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8180                                                NULL,
8181                                                "count action not supported");
8182                         }
8183                         /* Save information first, will apply later. */
8184                         if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT)
8185                                 count = action->conf;
8186                         else
8187                                 age = action->conf;
8188                         action_flags |= MLX5_FLOW_ACTION_COUNT;
8189                         break;
8190                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
8191                         dev_flow->dv.actions[actions_n++] =
8192                                                 priv->sh->pop_vlan_action;
8193                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
8194                         break;
8195                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
8196                         if (!(action_flags &
8197                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
8198                                 flow_dev_get_vlan_info_from_items(items, &vlan);
8199                         vlan.eth_proto = rte_be_to_cpu_16
8200                              ((((const struct rte_flow_action_of_push_vlan *)
8201                                                    actions->conf)->ethertype));
8202                         found_action = mlx5_flow_find_action
8203                                         (actions + 1,
8204                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
8205                         if (found_action)
8206                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
8207                         found_action = mlx5_flow_find_action
8208                                         (actions + 1,
8209                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
8210                         if (found_action)
8211                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
8212                         if (flow_dv_create_action_push_vlan
8213                                             (dev, attr, &vlan, dev_flow, error))
8214                                 return -rte_errno;
8215                         dev_flow->dv.actions[actions_n++] =
8216                                         dev_flow->dv.push_vlan_res->action;
8217                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
8218                         break;
8219                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
8220                         /* of_vlan_push action handled this action */
8221                         MLX5_ASSERT(action_flags &
8222                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
8223                         break;
8224                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
8225                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
8226                                 break;
8227                         flow_dev_get_vlan_info_from_items(items, &vlan);
8228                         mlx5_update_vlan_vid_pcp(actions, &vlan);
8229                         /* If no VLAN push - this is a modify header action */
8230                         if (flow_dv_convert_action_modify_vlan_vid
8231                                                 (mhdr_res, actions, error))
8232                                 return -rte_errno;
8233                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
8234                         break;
8235                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
8236                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
8237                         if (flow_dv_create_action_l2_encap(dev, actions,
8238                                                            dev_flow,
8239                                                            attr->transfer,
8240                                                            error))
8241                                 return -rte_errno;
8242                         dev_flow->dv.actions[actions_n++] =
8243                                         dev_flow->dv.encap_decap->action;
8244                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
8245                         break;
8246                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
8247                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
8248                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
8249                                                            attr->transfer,
8250                                                            error))
8251                                 return -rte_errno;
8252                         dev_flow->dv.actions[actions_n++] =
8253                                         dev_flow->dv.encap_decap->action;
8254                         action_flags |= MLX5_FLOW_ACTION_DECAP;
8255                         break;
8256                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
8257                         /* Handle encap with preceding decap. */
8258                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
8259                                 if (flow_dv_create_action_raw_encap
8260                                         (dev, actions, dev_flow, attr, error))
8261                                         return -rte_errno;
8262                                 dev_flow->dv.actions[actions_n++] =
8263                                         dev_flow->dv.encap_decap->action;
8264                         } else {
8265                                 /* Handle encap without preceding decap. */
8266                                 if (flow_dv_create_action_l2_encap
8267                                     (dev, actions, dev_flow, attr->transfer,
8268                                      error))
8269                                         return -rte_errno;
8270                                 dev_flow->dv.actions[actions_n++] =
8271                                         dev_flow->dv.encap_decap->action;
8272                         }
8273                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
8274                         break;
8275                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
8276                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
8277                                 ;
8278                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
8279                                 if (flow_dv_create_action_l2_decap
8280                                     (dev, dev_flow, attr->transfer, error))
8281                                         return -rte_errno;
8282                                 dev_flow->dv.actions[actions_n++] =
8283                                         dev_flow->dv.encap_decap->action;
8284                         }
8285                         /* If decap is followed by encap, handle it at encap. */
8286                         action_flags |= MLX5_FLOW_ACTION_DECAP;
8287                         break;
8288                 case RTE_FLOW_ACTION_TYPE_JUMP:
8289                         jump_data = action->conf;
8290                         ret = mlx5_flow_group_to_table(attr, dev_flow->external,
8291                                                        jump_data->group,
8292                                                        !!priv->fdb_def_rule,
8293                                                        &table, error);
8294                         if (ret)
8295                                 return ret;
8296                         tbl = flow_dv_tbl_resource_get(dev, table,
8297                                                        attr->egress,
8298                                                        attr->transfer, error);
8299                         if (!tbl)
8300                                 return rte_flow_error_set
8301                                                 (error, errno,
8302                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8303                                                  NULL,
8304                                                  "cannot create jump action.");
8305                         if (flow_dv_jump_tbl_resource_register
8306                             (dev, tbl, dev_flow, error)) {
8307                                 flow_dv_tbl_resource_release(dev, tbl);
8308                                 return rte_flow_error_set
8309                                                 (error, errno,
8310                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8311                                                  NULL,
8312                                                  "cannot create jump action.");
8313                         }
8314                         dev_flow->dv.actions[actions_n++] =
8315                                         dev_flow->dv.jump->action;
8316                         action_flags |= MLX5_FLOW_ACTION_JUMP;
8317                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
8318                         break;
8319                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
8320                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
8321                         if (flow_dv_convert_action_modify_mac
8322                                         (mhdr_res, actions, error))
8323                                 return -rte_errno;
8324                         action_flags |= actions->type ==
8325                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
8326                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
8327                                         MLX5_FLOW_ACTION_SET_MAC_DST;
8328                         break;
8329                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
8330                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
8331                         if (flow_dv_convert_action_modify_ipv4
8332                                         (mhdr_res, actions, error))
8333                                 return -rte_errno;
8334                         action_flags |= actions->type ==
8335                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
8336                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
8337                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
8338                         break;
8339                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
8340                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
8341                         if (flow_dv_convert_action_modify_ipv6
8342                                         (mhdr_res, actions, error))
8343                                 return -rte_errno;
8344                         action_flags |= actions->type ==
8345                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
8346                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
8347                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
8348                         break;
8349                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
8350                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
8351                         if (flow_dv_convert_action_modify_tp
8352                                         (mhdr_res, actions, items,
8353                                          &flow_attr, dev_flow, !!(action_flags &
8354                                          MLX5_FLOW_ACTION_DECAP), error))
8355                                 return -rte_errno;
8356                         action_flags |= actions->type ==
8357                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
8358                                         MLX5_FLOW_ACTION_SET_TP_SRC :
8359                                         MLX5_FLOW_ACTION_SET_TP_DST;
8360                         break;
8361                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
8362                         if (flow_dv_convert_action_modify_dec_ttl
8363                                         (mhdr_res, items, &flow_attr, dev_flow,
8364                                          !!(action_flags &
8365                                          MLX5_FLOW_ACTION_DECAP), error))
8366                                 return -rte_errno;
8367                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
8368                         break;
8369                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
8370                         if (flow_dv_convert_action_modify_ttl
8371                                         (mhdr_res, actions, items, &flow_attr,
8372                                          dev_flow, !!(action_flags &
8373                                          MLX5_FLOW_ACTION_DECAP), error))
8374                                 return -rte_errno;
8375                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
8376                         break;
8377                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
8378                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
8379                         if (flow_dv_convert_action_modify_tcp_seq
8380                                         (mhdr_res, actions, error))
8381                                 return -rte_errno;
8382                         action_flags |= actions->type ==
8383                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
8384                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
8385                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
8386                         break;
8387
8388                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
8389                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
8390                         if (flow_dv_convert_action_modify_tcp_ack
8391                                         (mhdr_res, actions, error))
8392                                 return -rte_errno;
8393                         action_flags |= actions->type ==
8394                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
8395                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
8396                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
8397                         break;
8398                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
8399                         if (flow_dv_convert_action_set_reg
8400                                         (mhdr_res, actions, error))
8401                                 return -rte_errno;
8402                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8403                         break;
8404                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
8405                         if (flow_dv_convert_action_copy_mreg
8406                                         (dev, mhdr_res, actions, error))
8407                                 return -rte_errno;
8408                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8409                         break;
8410                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
8411                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
8412                         dev_flow->handle->fate_action =
8413                                         MLX5_FLOW_FATE_DEFAULT_MISS;
8414                         break;
8415                 case RTE_FLOW_ACTION_TYPE_METER:
8416                         mtr = actions->conf;
8417                         if (!flow->meter) {
8418                                 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
8419                                                             attr, error);
8420                                 if (!fm)
8421                                         return rte_flow_error_set(error,
8422                                                 rte_errno,
8423                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8424                                                 NULL,
8425                                                 "meter not found "
8426                                                 "or invalid parameters");
8427                                 flow->meter = fm->idx;
8428                         }
8429                         /* Set the meter action. */
8430                         if (!fm) {
8431                                 fm = mlx5_ipool_get(priv->sh->ipool
8432                                                 [MLX5_IPOOL_MTR], flow->meter);
8433                                 if (!fm)
8434                                         return rte_flow_error_set(error,
8435                                                 rte_errno,
8436                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8437                                                 NULL,
8438                                                 "meter not found "
8439                                                 "or invalid parameters");
8440                         }
8441                         dev_flow->dv.actions[actions_n++] =
8442                                 fm->mfts->meter_action;
8443                         action_flags |= MLX5_FLOW_ACTION_METER;
8444                         break;
8445                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
8446                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
8447                                                               actions, error))
8448                                 return -rte_errno;
8449                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
8450                         break;
8451                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
8452                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
8453                                                               actions, error))
8454                                 return -rte_errno;
8455                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
8456                         break;
8457                 case RTE_FLOW_ACTION_TYPE_END:
8458                         actions_end = true;
8459                         if (mhdr_res->actions_num) {
8460                                 /* create modify action if needed. */
8461                                 if (flow_dv_modify_hdr_resource_register
8462                                         (dev, mhdr_res, dev_flow, error))
8463                                         return -rte_errno;
8464                                 dev_flow->dv.actions[modify_action_position] =
8465                                         handle->dvh.modify_hdr->action;
8466                         }
8467                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
8468                                 flow->counter =
8469                                         flow_dv_translate_create_counter(dev,
8470                                                 dev_flow, count, age);
8471
8472                                 if (!flow->counter)
8473                                         return rte_flow_error_set
8474                                                 (error, rte_errno,
8475                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8476                                                 NULL,
8477                                                 "cannot create counter"
8478                                                 " object.");
8479                                 dev_flow->dv.actions[actions_n++] =
8480                                           (flow_dv_counter_get_by_idx(dev,
8481                                           flow->counter, NULL))->action;
8482                         }
8483                         break;
8484                 default:
8485                         break;
8486                 }
8487                 if (mhdr_res->actions_num &&
8488                     modify_action_position == UINT32_MAX)
8489                         modify_action_position = actions_n++;
8490         }
8491         dev_flow->dv.actions_n = actions_n;
8492         dev_flow->act_flags = action_flags;
8493         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
8494                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
8495                 int item_type = items->type;
8496
8497                 if (!mlx5_flow_os_item_supported(item_type))
8498                         return rte_flow_error_set(error, ENOTSUP,
8499                                                   RTE_FLOW_ERROR_TYPE_ITEM,
8500                                                   NULL, "item not supported");
8501                 switch (item_type) {
8502                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
8503                         flow_dv_translate_item_port_id(dev, match_mask,
8504                                                        match_value, items);
8505                         last_item = MLX5_FLOW_ITEM_PORT_ID;
8506                         break;
8507                 case RTE_FLOW_ITEM_TYPE_ETH:
8508                         flow_dv_translate_item_eth(match_mask, match_value,
8509                                                    items, tunnel,
8510                                                    dev_flow->dv.group);
8511                         matcher.priority = action_flags &
8512                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
8513                                         !dev_flow->external ?
8514                                         MLX5_PRIORITY_MAP_L3 :
8515                                         MLX5_PRIORITY_MAP_L2;
8516                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
8517                                              MLX5_FLOW_LAYER_OUTER_L2;
8518                         break;
8519                 case RTE_FLOW_ITEM_TYPE_VLAN:
8520                         flow_dv_translate_item_vlan(dev_flow,
8521                                                     match_mask, match_value,
8522                                                     items, tunnel,
8523                                                     dev_flow->dv.group);
8524                         matcher.priority = MLX5_PRIORITY_MAP_L2;
8525                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
8526                                               MLX5_FLOW_LAYER_INNER_VLAN) :
8527                                              (MLX5_FLOW_LAYER_OUTER_L2 |
8528                                               MLX5_FLOW_LAYER_OUTER_VLAN);
8529                         break;
8530                 case RTE_FLOW_ITEM_TYPE_IPV4:
8531                         mlx5_flow_tunnel_ip_check(items, next_protocol,
8532                                                   &item_flags, &tunnel);
8533                         flow_dv_translate_item_ipv4(match_mask, match_value,
8534                                                     items, item_flags, tunnel,
8535                                                     dev_flow->dv.group);
8536                         matcher.priority = MLX5_PRIORITY_MAP_L3;
8537                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
8538                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
8539                         if (items->mask != NULL &&
8540                             ((const struct rte_flow_item_ipv4 *)
8541                              items->mask)->hdr.next_proto_id) {
8542                                 next_protocol =
8543                                         ((const struct rte_flow_item_ipv4 *)
8544                                          (items->spec))->hdr.next_proto_id;
8545                                 next_protocol &=
8546                                         ((const struct rte_flow_item_ipv4 *)
8547                                          (items->mask))->hdr.next_proto_id;
8548                         } else {
8549                                 /* Reset for inner layer. */
8550                                 next_protocol = 0xff;
8551                         }
8552                         break;
8553                 case RTE_FLOW_ITEM_TYPE_IPV6:
8554                         mlx5_flow_tunnel_ip_check(items, next_protocol,
8555                                                   &item_flags, &tunnel);
8556                         flow_dv_translate_item_ipv6(match_mask, match_value,
8557                                                     items, item_flags, tunnel,
8558                                                     dev_flow->dv.group);
8559                         matcher.priority = MLX5_PRIORITY_MAP_L3;
8560                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
8561                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
8562                         if (items->mask != NULL &&
8563                             ((const struct rte_flow_item_ipv6 *)
8564                              items->mask)->hdr.proto) {
8565                                 next_protocol =
8566                                         ((const struct rte_flow_item_ipv6 *)
8567                                          items->spec)->hdr.proto;
8568                                 next_protocol &=
8569                                         ((const struct rte_flow_item_ipv6 *)
8570                                          items->mask)->hdr.proto;
8571                         } else {
8572                                 /* Reset for inner layer. */
8573                                 next_protocol = 0xff;
8574                         }
8575                         break;
8576                 case RTE_FLOW_ITEM_TYPE_TCP:
8577                         flow_dv_translate_item_tcp(match_mask, match_value,
8578                                                    items, tunnel);
8579                         matcher.priority = MLX5_PRIORITY_MAP_L4;
8580                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
8581                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
8582                         break;
8583                 case RTE_FLOW_ITEM_TYPE_UDP:
8584                         flow_dv_translate_item_udp(match_mask, match_value,
8585                                                    items, tunnel);
8586                         matcher.priority = MLX5_PRIORITY_MAP_L4;
8587                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
8588                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
8589                         break;
8590                 case RTE_FLOW_ITEM_TYPE_GRE:
8591                         flow_dv_translate_item_gre(match_mask, match_value,
8592                                                    items, tunnel);
8593                         matcher.priority = rss_desc->level >= 2 ?
8594                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8595                         last_item = MLX5_FLOW_LAYER_GRE;
8596                         break;
8597                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
8598                         flow_dv_translate_item_gre_key(match_mask,
8599                                                        match_value, items);
8600                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
8601                         break;
8602                 case RTE_FLOW_ITEM_TYPE_NVGRE:
8603                         flow_dv_translate_item_nvgre(match_mask, match_value,
8604                                                      items, tunnel);
8605                         matcher.priority = rss_desc->level >= 2 ?
8606                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8607                         last_item = MLX5_FLOW_LAYER_GRE;
8608                         break;
8609                 case RTE_FLOW_ITEM_TYPE_VXLAN:
8610                         flow_dv_translate_item_vxlan(match_mask, match_value,
8611                                                      items, tunnel);
8612                         matcher.priority = rss_desc->level >= 2 ?
8613                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8614                         last_item = MLX5_FLOW_LAYER_VXLAN;
8615                         break;
8616                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
8617                         flow_dv_translate_item_vxlan_gpe(match_mask,
8618                                                          match_value, items,
8619                                                          tunnel);
8620                         matcher.priority = rss_desc->level >= 2 ?
8621                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8622                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
8623                         break;
8624                 case RTE_FLOW_ITEM_TYPE_GENEVE:
8625                         flow_dv_translate_item_geneve(match_mask, match_value,
8626                                                       items, tunnel);
8627                         matcher.priority = rss_desc->level >= 2 ?
8628                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8629                         last_item = MLX5_FLOW_LAYER_GENEVE;
8630                         break;
8631                 case RTE_FLOW_ITEM_TYPE_MPLS:
8632                         flow_dv_translate_item_mpls(match_mask, match_value,
8633                                                     items, last_item, tunnel);
8634                         matcher.priority = rss_desc->level >= 2 ?
8635                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8636                         last_item = MLX5_FLOW_LAYER_MPLS;
8637                         break;
8638                 case RTE_FLOW_ITEM_TYPE_MARK:
8639                         flow_dv_translate_item_mark(dev, match_mask,
8640                                                     match_value, items);
8641                         last_item = MLX5_FLOW_ITEM_MARK;
8642                         break;
8643                 case RTE_FLOW_ITEM_TYPE_META:
8644                         flow_dv_translate_item_meta(dev, match_mask,
8645                                                     match_value, attr, items);
8646                         last_item = MLX5_FLOW_ITEM_METADATA;
8647                         break;
8648                 case RTE_FLOW_ITEM_TYPE_ICMP:
8649                         flow_dv_translate_item_icmp(match_mask, match_value,
8650                                                     items, tunnel);
8651                         last_item = MLX5_FLOW_LAYER_ICMP;
8652                         break;
8653                 case RTE_FLOW_ITEM_TYPE_ICMP6:
8654                         flow_dv_translate_item_icmp6(match_mask, match_value,
8655                                                       items, tunnel);
8656                         last_item = MLX5_FLOW_LAYER_ICMP6;
8657                         break;
8658                 case RTE_FLOW_ITEM_TYPE_TAG:
8659                         flow_dv_translate_item_tag(dev, match_mask,
8660                                                    match_value, items);
8661                         last_item = MLX5_FLOW_ITEM_TAG;
8662                         break;
8663                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
8664                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
8665                                                         match_value, items);
8666                         last_item = MLX5_FLOW_ITEM_TAG;
8667                         break;
8668                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
8669                         flow_dv_translate_item_tx_queue(dev, match_mask,
8670                                                         match_value,
8671                                                         items);
8672                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
8673                         break;
8674                 case RTE_FLOW_ITEM_TYPE_GTP:
8675                         flow_dv_translate_item_gtp(match_mask, match_value,
8676                                                    items, tunnel);
8677                         matcher.priority = rss_desc->level >= 2 ?
8678                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8679                         last_item = MLX5_FLOW_LAYER_GTP;
8680                         break;
8681                 case RTE_FLOW_ITEM_TYPE_ECPRI:
8682                         if (!mlx5_flex_parser_ecpri_exist(dev)) {
8683                                 /* Create it only the first time to be used. */
8684                                 ret = mlx5_flex_parser_ecpri_alloc(dev);
8685                                 if (ret)
8686                                         return rte_flow_error_set
8687                                                 (error, -ret,
8688                                                 RTE_FLOW_ERROR_TYPE_ITEM,
8689                                                 NULL,
8690                                                 "cannot create eCPRI parser");
8691                         }
8692                         /* Adjust the length matcher and device flow value. */
8693                         matcher.mask.size = MLX5_ST_SZ_BYTES(fte_match_param);
8694                         dev_flow->dv.value.size =
8695                                         MLX5_ST_SZ_BYTES(fte_match_param);
8696                         flow_dv_translate_item_ecpri(dev, match_mask,
8697                                                      match_value, items);
8698                         /* No other protocol should follow eCPRI layer. */
8699                         last_item = MLX5_FLOW_LAYER_ECPRI;
8700                         break;
8701                 default:
8702                         break;
8703                 }
8704                 item_flags |= last_item;
8705         }
8706         /*
8707          * When E-Switch mode is enabled, we have two cases where we need to
8708          * set the source port manually.
8709          * The first one, is in case of Nic steering rule, and the second is
8710          * E-Switch rule where no port_id item was found. In both cases
8711          * the source port is set according the current port in use.
8712          */
8713         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
8714             (priv->representor || priv->master)) {
8715                 if (flow_dv_translate_item_port_id(dev, match_mask,
8716                                                    match_value, NULL))
8717                         return -rte_errno;
8718         }
8719 #ifdef RTE_LIBRTE_MLX5_DEBUG
8720         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
8721                                               dev_flow->dv.value.buf));
8722 #endif
8723         /*
8724          * Layers may be already initialized from prefix flow if this dev_flow
8725          * is the suffix flow.
8726          */
8727         handle->layers |= item_flags;
8728         if (action_flags & MLX5_FLOW_ACTION_RSS)
8729                 flow_dv_hashfields_set(dev_flow, rss_desc);
8730         /* Register matcher. */
8731         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
8732                                     matcher.mask.size);
8733         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
8734                                                      matcher.priority);
8735         /* reserved field no needs to be set to 0 here. */
8736         tbl_key.domain = attr->transfer;
8737         tbl_key.direction = attr->egress;
8738         tbl_key.table_id = dev_flow->dv.group;
8739         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
8740                 return -rte_errno;
8741         return 0;
8742 }
8743
8744 /**
8745  * Apply the flow to the NIC, lock free,
8746  * (mutex should be acquired by caller).
8747  *
8748  * @param[in] dev
8749  *   Pointer to the Ethernet device structure.
8750  * @param[in, out] flow
8751  *   Pointer to flow structure.
8752  * @param[out] error
8753  *   Pointer to error structure.
8754  *
8755  * @return
8756  *   0 on success, a negative errno value otherwise and rte_errno is set.
8757  */
8758 static int
8759 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
8760                 struct rte_flow_error *error)
8761 {
8762         struct mlx5_flow_dv_workspace *dv;
8763         struct mlx5_flow_handle *dh;
8764         struct mlx5_flow_handle_dv *dv_h;
8765         struct mlx5_flow *dev_flow;
8766         struct mlx5_priv *priv = dev->data->dev_private;
8767         uint32_t handle_idx;
8768         int n;
8769         int err;
8770         int idx;
8771
8772         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
8773                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
8774                 dv = &dev_flow->dv;
8775                 dh = dev_flow->handle;
8776                 dv_h = &dh->dvh;
8777                 n = dv->actions_n;
8778                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
8779                         if (dv->transfer) {
8780                                 dv->actions[n++] = priv->sh->esw_drop_action;
8781                         } else {
8782                                 struct mlx5_hrxq *drop_hrxq;
8783                                 drop_hrxq = mlx5_hrxq_drop_new(dev);
8784                                 if (!drop_hrxq) {
8785                                         rte_flow_error_set
8786                                                 (error, errno,
8787                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8788                                                  NULL,
8789                                                  "cannot get drop hash queue");
8790                                         goto error;
8791                                 }
8792                                 /*
8793                                  * Drop queues will be released by the specify
8794                                  * mlx5_hrxq_drop_release() function. Assign
8795                                  * the special index to hrxq to mark the queue
8796                                  * has been allocated.
8797                                  */
8798                                 dh->rix_hrxq = UINT32_MAX;
8799                                 dv->actions[n++] = drop_hrxq->action;
8800                         }
8801                 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
8802                         struct mlx5_hrxq *hrxq;
8803                         uint32_t hrxq_idx;
8804                         struct mlx5_flow_rss_desc *rss_desc =
8805                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
8806                                 [!!priv->flow_nested_idx];
8807
8808                         MLX5_ASSERT(rss_desc->queue_num);
8809                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
8810                                                  MLX5_RSS_HASH_KEY_LEN,
8811                                                  dev_flow->hash_fields,
8812                                                  rss_desc->queue,
8813                                                  rss_desc->queue_num);
8814                         if (!hrxq_idx) {
8815                                 hrxq_idx = mlx5_hrxq_new
8816                                                 (dev, rss_desc->key,
8817                                                 MLX5_RSS_HASH_KEY_LEN,
8818                                                 dev_flow->hash_fields,
8819                                                 rss_desc->queue,
8820                                                 rss_desc->queue_num,
8821                                                 !!(dh->layers &
8822                                                 MLX5_FLOW_LAYER_TUNNEL));
8823                         }
8824                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
8825                                               hrxq_idx);
8826                         if (!hrxq) {
8827                                 rte_flow_error_set
8828                                         (error, rte_errno,
8829                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8830                                          "cannot get hash queue");
8831                                 goto error;
8832                         }
8833                         dh->rix_hrxq = hrxq_idx;
8834                         dv->actions[n++] = hrxq->action;
8835                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
8836                         if (flow_dv_default_miss_resource_register
8837                                         (dev, error)) {
8838                                 rte_flow_error_set
8839                                         (error, rte_errno,
8840                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8841                                          "cannot create default miss resource");
8842                                 goto error_default_miss;
8843                         }
8844                         dh->rix_default_fate =  MLX5_FLOW_FATE_DEFAULT_MISS;
8845                         dv->actions[n++] = priv->sh->default_miss.action;
8846                 }
8847                 err = mlx5_flow_os_create_flow(dv_h->matcher->matcher_object,
8848                                                (void *)&dv->value, n,
8849                                                dv->actions, &dh->drv_flow);
8850                 if (err) {
8851                         rte_flow_error_set(error, errno,
8852                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8853                                            NULL,
8854                                            "hardware refuses to create flow");
8855                         goto error;
8856                 }
8857                 if (priv->vmwa_context &&
8858                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
8859                         /*
8860                          * The rule contains the VLAN pattern.
8861                          * For VF we are going to create VLAN
8862                          * interface to make hypervisor set correct
8863                          * e-Switch vport context.
8864                          */
8865                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
8866                 }
8867         }
8868         return 0;
8869 error:
8870         if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
8871                 flow_dv_default_miss_resource_release(dev);
8872 error_default_miss:
8873         err = rte_errno; /* Save rte_errno before cleanup. */
8874         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
8875                        handle_idx, dh, next) {
8876                 /* hrxq is union, don't clear it if the flag is not set. */
8877                 if (dh->rix_hrxq) {
8878                         if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
8879                                 mlx5_hrxq_drop_release(dev);
8880                                 dh->rix_hrxq = 0;
8881                         } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
8882                                 mlx5_hrxq_release(dev, dh->rix_hrxq);
8883                                 dh->rix_hrxq = 0;
8884                         }
8885                 }
8886                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
8887                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
8888         }
8889         rte_errno = err; /* Restore rte_errno. */
8890         return -rte_errno;
8891 }
8892
8893 /**
8894  * Release the flow matcher.
8895  *
8896  * @param dev
8897  *   Pointer to Ethernet device.
8898  * @param handle
8899  *   Pointer to mlx5_flow_handle.
8900  *
8901  * @return
8902  *   1 while a reference on it exists, 0 when freed.
8903  */
8904 static int
8905 flow_dv_matcher_release(struct rte_eth_dev *dev,
8906                         struct mlx5_flow_handle *handle)
8907 {
8908         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
8909
8910         MLX5_ASSERT(matcher->matcher_object);
8911         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
8912                 dev->data->port_id, (void *)matcher,
8913                 rte_atomic32_read(&matcher->refcnt));
8914         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
8915                 claim_zero(mlx5_flow_os_destroy_flow_matcher
8916                            (matcher->matcher_object));
8917                 LIST_REMOVE(matcher, next);
8918                 /* table ref-- in release interface. */
8919                 flow_dv_tbl_resource_release(dev, matcher->tbl);
8920                 mlx5_free(matcher);
8921                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
8922                         dev->data->port_id, (void *)matcher);
8923                 return 0;
8924         }
8925         return 1;
8926 }
8927
8928 /**
8929  * Release an encap/decap resource.
8930  *
8931  * @param dev
8932  *   Pointer to Ethernet device.
8933  * @param handle
8934  *   Pointer to mlx5_flow_handle.
8935  *
8936  * @return
8937  *   1 while a reference on it exists, 0 when freed.
8938  */
8939 static int
8940 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
8941                                      struct mlx5_flow_handle *handle)
8942 {
8943         struct mlx5_priv *priv = dev->data->dev_private;
8944         uint32_t idx = handle->dvh.rix_encap_decap;
8945         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
8946
8947         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
8948                          idx);
8949         if (!cache_resource)
8950                 return 0;
8951         MLX5_ASSERT(cache_resource->action);
8952         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
8953                 (void *)cache_resource,
8954                 rte_atomic32_read(&cache_resource->refcnt));
8955         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8956                 claim_zero(mlx5_flow_os_destroy_flow_action
8957                                                 (cache_resource->action));
8958                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
8959                              &priv->sh->encaps_decaps, idx,
8960                              cache_resource, next);
8961                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
8962                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
8963                         (void *)cache_resource);
8964                 return 0;
8965         }
8966         return 1;
8967 }
8968
8969 /**
8970  * Release an jump to table action resource.
8971  *
8972  * @param dev
8973  *   Pointer to Ethernet device.
8974  * @param handle
8975  *   Pointer to mlx5_flow_handle.
8976  *
8977  * @return
8978  *   1 while a reference on it exists, 0 when freed.
8979  */
8980 static int
8981 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
8982                                   struct mlx5_flow_handle *handle)
8983 {
8984         struct mlx5_priv *priv = dev->data->dev_private;
8985         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
8986         struct mlx5_flow_tbl_data_entry *tbl_data;
8987
8988         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
8989                              handle->rix_jump);
8990         if (!tbl_data)
8991                 return 0;
8992         cache_resource = &tbl_data->jump;
8993         MLX5_ASSERT(cache_resource->action);
8994         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
8995                 (void *)cache_resource,
8996                 rte_atomic32_read(&cache_resource->refcnt));
8997         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8998                 claim_zero(mlx5_flow_os_destroy_flow_action
8999                                                 (cache_resource->action));
9000                 /* jump action memory free is inside the table release. */
9001                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
9002                 DRV_LOG(DEBUG, "jump table resource %p: removed",
9003                         (void *)cache_resource);
9004                 return 0;
9005         }
9006         return 1;
9007 }
9008
9009 /**
9010  * Release a default miss resource.
9011  *
9012  * @param dev
9013  *   Pointer to Ethernet device.
9014  * @return
9015  *   1 while a reference on it exists, 0 when freed.
9016  */
9017 static int
9018 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
9019 {
9020         struct mlx5_priv *priv = dev->data->dev_private;
9021         struct mlx5_dev_ctx_shared *sh = priv->sh;
9022         struct mlx5_flow_default_miss_resource *cache_resource =
9023                         &sh->default_miss;
9024
9025         MLX5_ASSERT(cache_resource->action);
9026         DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--",
9027                         (void *)cache_resource->action,
9028                         rte_atomic32_read(&cache_resource->refcnt));
9029         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
9030                 claim_zero(mlx5_glue->destroy_flow_action
9031                                 (cache_resource->action));
9032                 DRV_LOG(DEBUG, "default miss resource %p: removed",
9033                                 (void *)cache_resource->action);
9034                 return 0;
9035         }
9036         return 1;
9037 }
9038
9039 /**
9040  * Release a modify-header resource.
9041  *
9042  * @param handle
9043  *   Pointer to mlx5_flow_handle.
9044  *
9045  * @return
9046  *   1 while a reference on it exists, 0 when freed.
9047  */
9048 static int
9049 flow_dv_modify_hdr_resource_release(struct mlx5_flow_handle *handle)
9050 {
9051         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
9052                                                         handle->dvh.modify_hdr;
9053
9054         MLX5_ASSERT(cache_resource->action);
9055         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
9056                 (void *)cache_resource,
9057                 rte_atomic32_read(&cache_resource->refcnt));
9058         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
9059                 claim_zero(mlx5_flow_os_destroy_flow_action
9060                                                 (cache_resource->action));
9061                 LIST_REMOVE(cache_resource, next);
9062                 mlx5_free(cache_resource);
9063                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
9064                         (void *)cache_resource);
9065                 return 0;
9066         }
9067         return 1;
9068 }
9069
9070 /**
9071  * Release port ID action resource.
9072  *
9073  * @param dev
9074  *   Pointer to Ethernet device.
9075  * @param handle
9076  *   Pointer to mlx5_flow_handle.
9077  *
9078  * @return
9079  *   1 while a reference on it exists, 0 when freed.
9080  */
9081 static int
9082 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
9083                                         struct mlx5_flow_handle *handle)
9084 {
9085         struct mlx5_priv *priv = dev->data->dev_private;
9086         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
9087         uint32_t idx = handle->rix_port_id_action;
9088
9089         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
9090                                         idx);
9091         if (!cache_resource)
9092                 return 0;
9093         MLX5_ASSERT(cache_resource->action);
9094         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
9095                 (void *)cache_resource,
9096                 rte_atomic32_read(&cache_resource->refcnt));
9097         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
9098                 claim_zero(mlx5_flow_os_destroy_flow_action
9099                                                 (cache_resource->action));
9100                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
9101                              &priv->sh->port_id_action_list, idx,
9102                              cache_resource, next);
9103                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx);
9104                 DRV_LOG(DEBUG, "port id action resource %p: removed",
9105                         (void *)cache_resource);
9106                 return 0;
9107         }
9108         return 1;
9109 }
9110
9111 /**
9112  * Release push vlan action resource.
9113  *
9114  * @param dev
9115  *   Pointer to Ethernet device.
9116  * @param handle
9117  *   Pointer to mlx5_flow_handle.
9118  *
9119  * @return
9120  *   1 while a reference on it exists, 0 when freed.
9121  */
9122 static int
9123 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
9124                                           struct mlx5_flow_handle *handle)
9125 {
9126         struct mlx5_priv *priv = dev->data->dev_private;
9127         uint32_t idx = handle->dvh.rix_push_vlan;
9128         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
9129
9130         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
9131                                         idx);
9132         if (!cache_resource)
9133                 return 0;
9134         MLX5_ASSERT(cache_resource->action);
9135         DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
9136                 (void *)cache_resource,
9137                 rte_atomic32_read(&cache_resource->refcnt));
9138         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
9139                 claim_zero(mlx5_flow_os_destroy_flow_action
9140                                                 (cache_resource->action));
9141                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
9142                              &priv->sh->push_vlan_action_list, idx,
9143                              cache_resource, next);
9144                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
9145                 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
9146                         (void *)cache_resource);
9147                 return 0;
9148         }
9149         return 1;
9150 }
9151
9152 /**
9153  * Release the fate resource.
9154  *
9155  * @param dev
9156  *   Pointer to Ethernet device.
9157  * @param handle
9158  *   Pointer to mlx5_flow_handle.
9159  */
9160 static void
9161 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
9162                                struct mlx5_flow_handle *handle)
9163 {
9164         if (!handle->rix_fate)
9165                 return;
9166         switch (handle->fate_action) {
9167         case MLX5_FLOW_FATE_DROP:
9168                 mlx5_hrxq_drop_release(dev);
9169                 break;
9170         case MLX5_FLOW_FATE_QUEUE:
9171                 mlx5_hrxq_release(dev, handle->rix_hrxq);
9172                 break;
9173         case MLX5_FLOW_FATE_JUMP:
9174                 flow_dv_jump_tbl_resource_release(dev, handle);
9175                 break;
9176         case MLX5_FLOW_FATE_PORT_ID:
9177                 flow_dv_port_id_action_resource_release(dev, handle);
9178                 break;
9179         case MLX5_FLOW_FATE_DEFAULT_MISS:
9180                 flow_dv_default_miss_resource_release(dev);
9181                 break;
9182         default:
9183                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
9184                 break;
9185         }
9186         handle->rix_fate = 0;
9187 }
9188
9189 /**
9190  * Remove the flow from the NIC but keeps it in memory.
9191  * Lock free, (mutex should be acquired by caller).
9192  *
9193  * @param[in] dev
9194  *   Pointer to Ethernet device.
9195  * @param[in, out] flow
9196  *   Pointer to flow structure.
9197  */
9198 static void
9199 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
9200 {
9201         struct mlx5_flow_handle *dh;
9202         uint32_t handle_idx;
9203         struct mlx5_priv *priv = dev->data->dev_private;
9204
9205         if (!flow)
9206                 return;
9207         handle_idx = flow->dev_handles;
9208         while (handle_idx) {
9209                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9210                                     handle_idx);
9211                 if (!dh)
9212                         return;
9213                 if (dh->drv_flow) {
9214                         claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
9215                         dh->drv_flow = NULL;
9216                 }
9217                 if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
9218                     dh->fate_action == MLX5_FLOW_FATE_QUEUE ||
9219                     dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
9220                         flow_dv_fate_resource_release(dev, dh);
9221                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
9222                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
9223                 handle_idx = dh->next.next;
9224         }
9225 }
9226
9227 /**
9228  * Remove the flow from the NIC and the memory.
9229  * Lock free, (mutex should be acquired by caller).
9230  *
9231  * @param[in] dev
9232  *   Pointer to the Ethernet device structure.
9233  * @param[in, out] flow
9234  *   Pointer to flow structure.
9235  */
9236 static void
9237 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
9238 {
9239         struct mlx5_flow_handle *dev_handle;
9240         struct mlx5_priv *priv = dev->data->dev_private;
9241
9242         if (!flow)
9243                 return;
9244         __flow_dv_remove(dev, flow);
9245         if (flow->counter) {
9246                 flow_dv_counter_release(dev, flow->counter);
9247                 flow->counter = 0;
9248         }
9249         if (flow->meter) {
9250                 struct mlx5_flow_meter *fm;
9251
9252                 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
9253                                     flow->meter);
9254                 if (fm)
9255                         mlx5_flow_meter_detach(fm);
9256                 flow->meter = 0;
9257         }
9258         while (flow->dev_handles) {
9259                 uint32_t tmp_idx = flow->dev_handles;
9260
9261                 dev_handle = mlx5_ipool_get(priv->sh->ipool
9262                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
9263                 if (!dev_handle)
9264                         return;
9265                 flow->dev_handles = dev_handle->next.next;
9266                 if (dev_handle->dvh.matcher)
9267                         flow_dv_matcher_release(dev, dev_handle);
9268                 if (dev_handle->dvh.rix_encap_decap)
9269                         flow_dv_encap_decap_resource_release(dev, dev_handle);
9270                 if (dev_handle->dvh.modify_hdr)
9271                         flow_dv_modify_hdr_resource_release(dev_handle);
9272                 if (dev_handle->dvh.rix_push_vlan)
9273                         flow_dv_push_vlan_action_resource_release(dev,
9274                                                                   dev_handle);
9275                 if (dev_handle->dvh.rix_tag)
9276                         flow_dv_tag_release(dev,
9277                                             dev_handle->dvh.rix_tag);
9278                 flow_dv_fate_resource_release(dev, dev_handle);
9279                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9280                            tmp_idx);
9281         }
9282 }
9283
9284 /**
9285  * Query a dv flow  rule for its statistics via devx.
9286  *
9287  * @param[in] dev
9288  *   Pointer to Ethernet device.
9289  * @param[in] flow
9290  *   Pointer to the sub flow.
9291  * @param[out] data
9292  *   data retrieved by the query.
9293  * @param[out] error
9294  *   Perform verbose error reporting if not NULL.
9295  *
9296  * @return
9297  *   0 on success, a negative errno value otherwise and rte_errno is set.
9298  */
9299 static int
9300 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
9301                     void *data, struct rte_flow_error *error)
9302 {
9303         struct mlx5_priv *priv = dev->data->dev_private;
9304         struct rte_flow_query_count *qc = data;
9305
9306         if (!priv->config.devx)
9307                 return rte_flow_error_set(error, ENOTSUP,
9308                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9309                                           NULL,
9310                                           "counters are not supported");
9311         if (flow->counter) {
9312                 uint64_t pkts, bytes;
9313                 struct mlx5_flow_counter *cnt;
9314
9315                 cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
9316                                                  NULL);
9317                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
9318                                                &bytes);
9319
9320                 if (err)
9321                         return rte_flow_error_set(error, -err,
9322                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9323                                         NULL, "cannot read counters");
9324                 qc->hits_set = 1;
9325                 qc->bytes_set = 1;
9326                 qc->hits = pkts - cnt->hits;
9327                 qc->bytes = bytes - cnt->bytes;
9328                 if (qc->reset) {
9329                         cnt->hits = pkts;
9330                         cnt->bytes = bytes;
9331                 }
9332                 return 0;
9333         }
9334         return rte_flow_error_set(error, EINVAL,
9335                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9336                                   NULL,
9337                                   "counters are not available");
9338 }
9339
9340 /**
9341  * Query a flow.
9342  *
9343  * @see rte_flow_query()
9344  * @see rte_flow_ops
9345  */
9346 static int
9347 flow_dv_query(struct rte_eth_dev *dev,
9348               struct rte_flow *flow __rte_unused,
9349               const struct rte_flow_action *actions __rte_unused,
9350               void *data __rte_unused,
9351               struct rte_flow_error *error __rte_unused)
9352 {
9353         int ret = -EINVAL;
9354
9355         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
9356                 switch (actions->type) {
9357                 case RTE_FLOW_ACTION_TYPE_VOID:
9358                         break;
9359                 case RTE_FLOW_ACTION_TYPE_COUNT:
9360                         ret = flow_dv_query_count(dev, flow, data, error);
9361                         break;
9362                 default:
9363                         return rte_flow_error_set(error, ENOTSUP,
9364                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9365                                                   actions,
9366                                                   "action not supported");
9367                 }
9368         }
9369         return ret;
9370 }
9371
9372 /**
9373  * Destroy the meter table set.
9374  * Lock free, (mutex should be acquired by caller).
9375  *
9376  * @param[in] dev
9377  *   Pointer to Ethernet device.
9378  * @param[in] tbl
9379  *   Pointer to the meter table set.
9380  *
9381  * @return
9382  *   Always 0.
9383  */
9384 static int
9385 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
9386                         struct mlx5_meter_domains_infos *tbl)
9387 {
9388         struct mlx5_priv *priv = dev->data->dev_private;
9389         struct mlx5_meter_domains_infos *mtd =
9390                                 (struct mlx5_meter_domains_infos *)tbl;
9391
9392         if (!mtd || !priv->config.dv_flow_en)
9393                 return 0;
9394         if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
9395                 claim_zero(mlx5_flow_os_destroy_flow
9396                            (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
9397         if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
9398                 claim_zero(mlx5_flow_os_destroy_flow
9399                            (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
9400         if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
9401                 claim_zero(mlx5_flow_os_destroy_flow
9402                            (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
9403         if (mtd->egress.color_matcher)
9404                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9405                            (mtd->egress.color_matcher));
9406         if (mtd->egress.any_matcher)
9407                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9408                            (mtd->egress.any_matcher));
9409         if (mtd->egress.tbl)
9410                 flow_dv_tbl_resource_release(dev, mtd->egress.tbl);
9411         if (mtd->egress.sfx_tbl)
9412                 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl);
9413         if (mtd->ingress.color_matcher)
9414                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9415                            (mtd->ingress.color_matcher));
9416         if (mtd->ingress.any_matcher)
9417                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9418                            (mtd->ingress.any_matcher));
9419         if (mtd->ingress.tbl)
9420                 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl);
9421         if (mtd->ingress.sfx_tbl)
9422                 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl);
9423         if (mtd->transfer.color_matcher)
9424                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9425                            (mtd->transfer.color_matcher));
9426         if (mtd->transfer.any_matcher)
9427                 claim_zero(mlx5_flow_os_destroy_flow_matcher
9428                            (mtd->transfer.any_matcher));
9429         if (mtd->transfer.tbl)
9430                 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl);
9431         if (mtd->transfer.sfx_tbl)
9432                 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl);
9433         if (mtd->drop_actn)
9434                 claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn));
9435         mlx5_free(mtd);
9436         return 0;
9437 }
9438
9439 /* Number of meter flow actions, count and jump or count and drop. */
9440 #define METER_ACTIONS 2
9441
9442 /**
9443  * Create specify domain meter table and suffix table.
9444  *
9445  * @param[in] dev
9446  *   Pointer to Ethernet device.
9447  * @param[in,out] mtb
9448  *   Pointer to DV meter table set.
9449  * @param[in] egress
9450  *   Table attribute.
9451  * @param[in] transfer
9452  *   Table attribute.
9453  * @param[in] color_reg_c_idx
9454  *   Reg C index for color match.
9455  *
9456  * @return
9457  *   0 on success, -1 otherwise and rte_errno is set.
9458  */
9459 static int
9460 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
9461                            struct mlx5_meter_domains_infos *mtb,
9462                            uint8_t egress, uint8_t transfer,
9463                            uint32_t color_reg_c_idx)
9464 {
9465         struct mlx5_priv *priv = dev->data->dev_private;
9466         struct mlx5_dev_ctx_shared *sh = priv->sh;
9467         struct mlx5_flow_dv_match_params mask = {
9468                 .size = sizeof(mask.buf),
9469         };
9470         struct mlx5_flow_dv_match_params value = {
9471                 .size = sizeof(value.buf),
9472         };
9473         struct mlx5dv_flow_matcher_attr dv_attr = {
9474                 .type = IBV_FLOW_ATTR_NORMAL,
9475                 .priority = 0,
9476                 .match_criteria_enable = 0,
9477                 .match_mask = (void *)&mask,
9478         };
9479         void *actions[METER_ACTIONS];
9480         struct mlx5_meter_domain_info *dtb;
9481         struct rte_flow_error error;
9482         int i = 0;
9483         int ret;
9484
9485         if (transfer)
9486                 dtb = &mtb->transfer;
9487         else if (egress)
9488                 dtb = &mtb->egress;
9489         else
9490                 dtb = &mtb->ingress;
9491         /* Create the meter table with METER level. */
9492         dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
9493                                             egress, transfer, &error);
9494         if (!dtb->tbl) {
9495                 DRV_LOG(ERR, "Failed to create meter policer table.");
9496                 return -1;
9497         }
9498         /* Create the meter suffix table with SUFFIX level. */
9499         dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
9500                                             MLX5_FLOW_TABLE_LEVEL_SUFFIX,
9501                                             egress, transfer, &error);
9502         if (!dtb->sfx_tbl) {
9503                 DRV_LOG(ERR, "Failed to create meter suffix table.");
9504                 return -1;
9505         }
9506         /* Create matchers, Any and Color. */
9507         dv_attr.priority = 3;
9508         dv_attr.match_criteria_enable = 0;
9509         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
9510                                                &dtb->any_matcher);
9511         if (ret) {
9512                 DRV_LOG(ERR, "Failed to create meter"
9513                              " policer default matcher.");
9514                 goto error_exit;
9515         }
9516         dv_attr.priority = 0;
9517         dv_attr.match_criteria_enable =
9518                                 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
9519         flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
9520                                rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
9521         ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
9522                                                &dtb->color_matcher);
9523         if (ret) {
9524                 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
9525                 goto error_exit;
9526         }
9527         if (mtb->count_actns[RTE_MTR_DROPPED])
9528                 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
9529         actions[i++] = mtb->drop_actn;
9530         /* Default rule: lowest priority, match any, actions: drop. */
9531         ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i,
9532                                        actions,
9533                                        &dtb->policer_rules[RTE_MTR_DROPPED]);
9534         if (ret) {
9535                 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
9536                 goto error_exit;
9537         }
9538         return 0;
9539 error_exit:
9540         return -1;
9541 }
9542
9543 /**
9544  * Create the needed meter and suffix tables.
9545  * Lock free, (mutex should be acquired by caller).
9546  *
9547  * @param[in] dev
9548  *   Pointer to Ethernet device.
9549  * @param[in] fm
9550  *   Pointer to the flow meter.
9551  *
9552  * @return
9553  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
9554  */
9555 static struct mlx5_meter_domains_infos *
9556 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
9557                        const struct mlx5_flow_meter *fm)
9558 {
9559         struct mlx5_priv *priv = dev->data->dev_private;
9560         struct mlx5_meter_domains_infos *mtb;
9561         int ret;
9562         int i;
9563
9564         if (!priv->mtr_en) {
9565                 rte_errno = ENOTSUP;
9566                 return NULL;
9567         }
9568         mtb = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*mtb), 0, SOCKET_ID_ANY);
9569         if (!mtb) {
9570                 DRV_LOG(ERR, "Failed to allocate memory for meter.");
9571                 return NULL;
9572         }
9573         /* Create meter count actions */
9574         for (i = 0; i <= RTE_MTR_DROPPED; i++) {
9575                 struct mlx5_flow_counter *cnt;
9576                 if (!fm->policer_stats.cnt[i])
9577                         continue;
9578                 cnt = flow_dv_counter_get_by_idx(dev,
9579                       fm->policer_stats.cnt[i], NULL);
9580                 mtb->count_actns[i] = cnt->action;
9581         }
9582         /* Create drop action. */
9583         ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn);
9584         if (ret) {
9585                 DRV_LOG(ERR, "Failed to create drop action.");
9586                 goto error_exit;
9587         }
9588         /* Egress meter table. */
9589         ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
9590         if (ret) {
9591                 DRV_LOG(ERR, "Failed to prepare egress meter table.");
9592                 goto error_exit;
9593         }
9594         /* Ingress meter table. */
9595         ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
9596         if (ret) {
9597                 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
9598                 goto error_exit;
9599         }
9600         /* FDB meter table. */
9601         if (priv->config.dv_esw_en) {
9602                 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
9603                                                  priv->mtr_color_reg);
9604                 if (ret) {
9605                         DRV_LOG(ERR, "Failed to prepare fdb meter table.");
9606                         goto error_exit;
9607                 }
9608         }
9609         return mtb;
9610 error_exit:
9611         flow_dv_destroy_mtr_tbl(dev, mtb);
9612         return NULL;
9613 }
9614
9615 /**
9616  * Destroy domain policer rule.
9617  *
9618  * @param[in] dt
9619  *   Pointer to domain table.
9620  */
9621 static void
9622 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
9623 {
9624         int i;
9625
9626         for (i = 0; i < RTE_MTR_DROPPED; i++) {
9627                 if (dt->policer_rules[i]) {
9628                         claim_zero(mlx5_flow_os_destroy_flow
9629                                    (dt->policer_rules[i]));
9630                         dt->policer_rules[i] = NULL;
9631                 }
9632         }
9633         if (dt->jump_actn) {
9634                 claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn));
9635                 dt->jump_actn = NULL;
9636         }
9637 }
9638
9639 /**
9640  * Destroy policer rules.
9641  *
9642  * @param[in] dev
9643  *   Pointer to Ethernet device.
9644  * @param[in] fm
9645  *   Pointer to flow meter structure.
9646  * @param[in] attr
9647  *   Pointer to flow attributes.
9648  *
9649  * @return
9650  *   Always 0.
9651  */
9652 static int
9653 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
9654                               const struct mlx5_flow_meter *fm,
9655                               const struct rte_flow_attr *attr)
9656 {
9657         struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
9658
9659         if (!mtb)
9660                 return 0;
9661         if (attr->egress)
9662                 flow_dv_destroy_domain_policer_rule(&mtb->egress);
9663         if (attr->ingress)
9664                 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
9665         if (attr->transfer)
9666                 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
9667         return 0;
9668 }
9669
9670 /**
9671  * Create specify domain meter policer rule.
9672  *
9673  * @param[in] fm
9674  *   Pointer to flow meter structure.
9675  * @param[in] mtb
9676  *   Pointer to DV meter table set.
9677  * @param[in] mtr_reg_c
9678  *   Color match REG_C.
9679  *
9680  * @return
9681  *   0 on success, -1 otherwise.
9682  */
9683 static int
9684 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
9685                                     struct mlx5_meter_domain_info *dtb,
9686                                     uint8_t mtr_reg_c)
9687 {
9688         struct mlx5_flow_dv_match_params matcher = {
9689                 .size = sizeof(matcher.buf),
9690         };
9691         struct mlx5_flow_dv_match_params value = {
9692                 .size = sizeof(value.buf),
9693         };
9694         struct mlx5_meter_domains_infos *mtb = fm->mfts;
9695         void *actions[METER_ACTIONS];
9696         int i;
9697         int ret = 0;
9698
9699         /* Create jump action. */
9700         if (!dtb->jump_actn)
9701                 ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
9702                                 (dtb->sfx_tbl->obj, &dtb->jump_actn);
9703         if (ret) {
9704                 DRV_LOG(ERR, "Failed to create policer jump action.");
9705                 goto error;
9706         }
9707         for (i = 0; i < RTE_MTR_DROPPED; i++) {
9708                 int j = 0;
9709
9710                 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
9711                                        rte_col_2_mlx5_col(i), UINT8_MAX);
9712                 if (mtb->count_actns[i])
9713                         actions[j++] = mtb->count_actns[i];
9714                 if (fm->action[i] == MTR_POLICER_ACTION_DROP)
9715                         actions[j++] = mtb->drop_actn;
9716                 else
9717                         actions[j++] = dtb->jump_actn;
9718                 ret = mlx5_flow_os_create_flow(dtb->color_matcher,
9719                                                (void *)&value, j, actions,
9720                                                &dtb->policer_rules[i]);
9721                 if (ret) {
9722                         DRV_LOG(ERR, "Failed to create policer rule.");
9723                         goto error;
9724                 }
9725         }
9726         return 0;
9727 error:
9728         rte_errno = errno;
9729         return -1;
9730 }
9731
9732 /**
9733  * Create policer rules.
9734  *
9735  * @param[in] dev
9736  *   Pointer to Ethernet device.
9737  * @param[in] fm
9738  *   Pointer to flow meter structure.
9739  * @param[in] attr
9740  *   Pointer to flow attributes.
9741  *
9742  * @return
9743  *   0 on success, -1 otherwise.
9744  */
9745 static int
9746 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
9747                              struct mlx5_flow_meter *fm,
9748                              const struct rte_flow_attr *attr)
9749 {
9750         struct mlx5_priv *priv = dev->data->dev_private;
9751         struct mlx5_meter_domains_infos *mtb = fm->mfts;
9752         int ret;
9753
9754         if (attr->egress) {
9755                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
9756                                                 priv->mtr_color_reg);
9757                 if (ret) {
9758                         DRV_LOG(ERR, "Failed to create egress policer.");
9759                         goto error;
9760                 }
9761         }
9762         if (attr->ingress) {
9763                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
9764                                                 priv->mtr_color_reg);
9765                 if (ret) {
9766                         DRV_LOG(ERR, "Failed to create ingress policer.");
9767                         goto error;
9768                 }
9769         }
9770         if (attr->transfer) {
9771                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
9772                                                 priv->mtr_color_reg);
9773                 if (ret) {
9774                         DRV_LOG(ERR, "Failed to create transfer policer.");
9775                         goto error;
9776                 }
9777         }
9778         return 0;
9779 error:
9780         flow_dv_destroy_policer_rules(dev, fm, attr);
9781         return -1;
9782 }
9783
9784 /**
9785  * Query a devx counter.
9786  *
9787  * @param[in] dev
9788  *   Pointer to the Ethernet device structure.
9789  * @param[in] cnt
9790  *   Index to the flow counter.
9791  * @param[in] clear
9792  *   Set to clear the counter statistics.
9793  * @param[out] pkts
9794  *   The statistics value of packets.
9795  * @param[out] bytes
9796  *   The statistics value of bytes.
9797  *
9798  * @return
9799  *   0 on success, otherwise return -1.
9800  */
9801 static int
9802 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
9803                       uint64_t *pkts, uint64_t *bytes)
9804 {
9805         struct mlx5_priv *priv = dev->data->dev_private;
9806         struct mlx5_flow_counter *cnt;
9807         uint64_t inn_pkts, inn_bytes;
9808         int ret;
9809
9810         if (!priv->config.devx)
9811                 return -1;
9812
9813         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
9814         if (ret)
9815                 return -1;
9816         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
9817         *pkts = inn_pkts - cnt->hits;
9818         *bytes = inn_bytes - cnt->bytes;
9819         if (clear) {
9820                 cnt->hits = inn_pkts;
9821                 cnt->bytes = inn_bytes;
9822         }
9823         return 0;
9824 }
9825
9826 /**
9827  * Get aged-out flows.
9828  *
9829  * @param[in] dev
9830  *   Pointer to the Ethernet device structure.
9831  * @param[in] context
9832  *   The address of an array of pointers to the aged-out flows contexts.
9833  * @param[in] nb_contexts
9834  *   The length of context array pointers.
9835  * @param[out] error
9836  *   Perform verbose error reporting if not NULL. Initialized in case of
9837  *   error only.
9838  *
9839  * @return
9840  *   how many contexts get in success, otherwise negative errno value.
9841  *   if nb_contexts is 0, return the amount of all aged contexts.
9842  *   if nb_contexts is not 0 , return the amount of aged flows reported
9843  *   in the context array.
9844  * @note: only stub for now
9845  */
9846 static int
9847 flow_get_aged_flows(struct rte_eth_dev *dev,
9848                     void **context,
9849                     uint32_t nb_contexts,
9850                     struct rte_flow_error *error)
9851 {
9852         struct mlx5_priv *priv = dev->data->dev_private;
9853         struct mlx5_age_info *age_info;
9854         struct mlx5_age_param *age_param;
9855         struct mlx5_flow_counter *counter;
9856         int nb_flows = 0;
9857
9858         if (nb_contexts && !context)
9859                 return rte_flow_error_set(error, EINVAL,
9860                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9861                                           NULL,
9862                                           "Should assign at least one flow or"
9863                                           " context to get if nb_contexts != 0");
9864         age_info = GET_PORT_AGE_INFO(priv);
9865         rte_spinlock_lock(&age_info->aged_sl);
9866         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
9867                 nb_flows++;
9868                 if (nb_contexts) {
9869                         age_param = MLX5_CNT_TO_AGE(counter);
9870                         context[nb_flows - 1] = age_param->context;
9871                         if (!(--nb_contexts))
9872                                 break;
9873                 }
9874         }
9875         rte_spinlock_unlock(&age_info->aged_sl);
9876         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
9877         return nb_flows;
9878 }
9879
9880 /*
9881  * Mutex-protected thunk to lock-free  __flow_dv_translate().
9882  */
9883 static int
9884 flow_dv_translate(struct rte_eth_dev *dev,
9885                   struct mlx5_flow *dev_flow,
9886                   const struct rte_flow_attr *attr,
9887                   const struct rte_flow_item items[],
9888                   const struct rte_flow_action actions[],
9889                   struct rte_flow_error *error)
9890 {
9891         int ret;
9892
9893         flow_dv_shared_lock(dev);
9894         ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
9895         flow_dv_shared_unlock(dev);
9896         return ret;
9897 }
9898
9899 /*
9900  * Mutex-protected thunk to lock-free  __flow_dv_apply().
9901  */
9902 static int
9903 flow_dv_apply(struct rte_eth_dev *dev,
9904               struct rte_flow *flow,
9905               struct rte_flow_error *error)
9906 {
9907         int ret;
9908
9909         flow_dv_shared_lock(dev);
9910         ret = __flow_dv_apply(dev, flow, error);
9911         flow_dv_shared_unlock(dev);
9912         return ret;
9913 }
9914
9915 /*
9916  * Mutex-protected thunk to lock-free __flow_dv_remove().
9917  */
9918 static void
9919 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
9920 {
9921         flow_dv_shared_lock(dev);
9922         __flow_dv_remove(dev, flow);
9923         flow_dv_shared_unlock(dev);
9924 }
9925
9926 /*
9927  * Mutex-protected thunk to lock-free __flow_dv_destroy().
9928  */
9929 static void
9930 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
9931 {
9932         flow_dv_shared_lock(dev);
9933         __flow_dv_destroy(dev, flow);
9934         flow_dv_shared_unlock(dev);
9935 }
9936
9937 /*
9938  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
9939  */
9940 static uint32_t
9941 flow_dv_counter_allocate(struct rte_eth_dev *dev)
9942 {
9943         uint32_t cnt;
9944
9945         flow_dv_shared_lock(dev);
9946         cnt = flow_dv_counter_alloc(dev, 0, 0, 1, 0);
9947         flow_dv_shared_unlock(dev);
9948         return cnt;
9949 }
9950
9951 /*
9952  * Mutex-protected thunk to lock-free flow_dv_counter_release().
9953  */
9954 static void
9955 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
9956 {
9957         flow_dv_shared_lock(dev);
9958         flow_dv_counter_release(dev, cnt);
9959         flow_dv_shared_unlock(dev);
9960 }
9961
9962 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
9963         .validate = flow_dv_validate,
9964         .prepare = flow_dv_prepare,
9965         .translate = flow_dv_translate,
9966         .apply = flow_dv_apply,
9967         .remove = flow_dv_remove,
9968         .destroy = flow_dv_destroy,
9969         .query = flow_dv_query,
9970         .create_mtr_tbls = flow_dv_create_mtr_tbl,
9971         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
9972         .create_policer_rules = flow_dv_create_policer_rules,
9973         .destroy_policer_rules = flow_dv_destroy_policer_rules,
9974         .counter_alloc = flow_dv_counter_allocate,
9975         .counter_free = flow_dv_counter_free,
9976         .counter_query = flow_dv_counter_query,
9977         .get_aged_flows = flow_get_aged_flows,
9978 };
9979
9980 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */