drivers/net/mlx5/mlx5_flow_dv.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdalign.h>
   7 #include <stdint.h>
   8 #include <string.h>
   9 #include <unistd.h>
  10
  11 /* Verbs header. */
  12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
  13 #ifdef PEDANTIC
  14 #pragma GCC diagnostic ignored "-Wpedantic"
  15 #endif
  16 #include <infiniband/verbs.h>
  17 #ifdef PEDANTIC
  18 #pragma GCC diagnostic error "-Wpedantic"
  19 #endif
  20
  21 #include <rte_common.h>
  22 #include <rte_ether.h>
  23 #include <rte_ethdev_driver.h>
  24 #include <rte_flow.h>
  25 #include <rte_flow_driver.h>
  26 #include <rte_malloc.h>
  27 #include <rte_cycles.h>
  28 #include <rte_ip.h>
  29 #include <rte_gre.h>
  30 #include <rte_vxlan.h>
  31 #include <rte_gtp.h>
  32
  33 #include <mlx5_glue.h>
  34 #include <mlx5_devx_cmds.h>
  35 #include <mlx5_prm.h>
  36
  37 #include "mlx5_defs.h"
  38 #include "mlx5.h"
  39 #include "mlx5_common_os.h"
  40 #include "mlx5_flow.h"
  41 #include "mlx5_flow_os.h"
  42 #include "mlx5_rxtx.h"
  43
  44 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
  45
  46 #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
  47 #define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
  48 #endif
  49
  50 #ifndef HAVE_MLX5DV_DR_ESWITCH
  51 #ifndef MLX5DV_FLOW_TABLE_TYPE_FDB
  52 #define MLX5DV_FLOW_TABLE_TYPE_FDB 0
  53 #endif
  54 #endif
  55
  56 #ifndef HAVE_MLX5DV_DR
  57 #define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
  58 #endif
  59
  60 /* VLAN header definitions */
  61 #define MLX5DV_FLOW_VLAN_PCP_SHIFT 13
  62 #define MLX5DV_FLOW_VLAN_PCP_MASK (0x7 << MLX5DV_FLOW_VLAN_PCP_SHIFT)
  63 #define MLX5DV_FLOW_VLAN_VID_MASK 0x0fff
  64 #define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
  65 #define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
  66
  67 union flow_dv_attr {
  68         struct {
  69                 uint32_t valid:1;
  70                 uint32_t ipv4:1;
  71                 uint32_t ipv6:1;
  72                 uint32_t tcp:1;
  73                 uint32_t udp:1;
  74                 uint32_t reserved:27;
  75         };
  76         uint32_t attr;
  77 };
  78
  79 static int
  80 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
  81                              struct mlx5_flow_tbl_resource *tbl);
  82
  83 static int
  84 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev);
  85
  86 /**
  87  * Initialize flow attributes structure according to flow items' types.
  88  *
  89  * flow_dv_validate() avoids multiple L3/L4 layers cases other than tunnel
  90  * mode. For tunnel mode, the items to be modified are the outermost ones.
  91  *
  92  * @param[in] item
  93  *   Pointer to item specification.
  94  * @param[out] attr
  95  *   Pointer to flow attributes structure.
  96  * @param[in] dev_flow
  97  *   Pointer to the sub flow.
  98  * @param[in] tunnel_decap
  99  *   Whether action is after tunnel decapsulation.
 100  */
 101 static void
 102 flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 103                   struct mlx5_flow *dev_flow, bool tunnel_decap)
 104 {
 105         uint64_t layers = dev_flow->handle->layers;
 106
 107         /*
 108          * If layers is already initialized, it means this dev_flow is the
 109          * suffix flow, the layers flags is set by the prefix flow. Need to
 110          * use the layer flags from prefix flow as the suffix flow may not
 111          * have the user defined items as the flow is split.
 112          */
 113         if (layers) {
 114                 if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
 115                         attr->ipv4 = 1;
 116                 else if (layers & MLX5_FLOW_LAYER_OUTER_L3_IPV6)
 117                         attr->ipv6 = 1;
 118                 if (layers & MLX5_FLOW_LAYER_OUTER_L4_TCP)
 119                         attr->tcp = 1;
 120                 else if (layers & MLX5_FLOW_LAYER_OUTER_L4_UDP)
 121                         attr->udp = 1;
 122                 attr->valid = 1;
 123                 return;
 124         }
 125         for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
 126                 uint8_t next_protocol = 0xff;
 127                 switch (item->type) {
 128                 case RTE_FLOW_ITEM_TYPE_GRE:
 129                 case RTE_FLOW_ITEM_TYPE_NVGRE:
 130                 case RTE_FLOW_ITEM_TYPE_VXLAN:
 131                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 132                 case RTE_FLOW_ITEM_TYPE_GENEVE:
 133                 case RTE_FLOW_ITEM_TYPE_MPLS:
 134                         if (tunnel_decap)
 135                                 attr->attr = 0;
 136                         break;
 137                 case RTE_FLOW_ITEM_TYPE_IPV4:
 138                         if (!attr->ipv6)
 139                                 attr->ipv4 = 1;
 140                         if (item->mask != NULL &&
 141                             ((const struct rte_flow_item_ipv4 *)
 142                             item->mask)->hdr.next_proto_id)
 143                                 next_protocol =
 144                                     ((const struct rte_flow_item_ipv4 *)
 145                                       (item->spec))->hdr.next_proto_id &
 146                                     ((const struct rte_flow_item_ipv4 *)
 147                                       (item->mask))->hdr.next_proto_id;
 148                         if ((next_protocol == IPPROTO_IPIP ||
 149                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 150                                 attr->attr = 0;
 151                         break;
 152                 case RTE_FLOW_ITEM_TYPE_IPV6:
 153                         if (!attr->ipv4)
 154                                 attr->ipv6 = 1;
 155                         if (item->mask != NULL &&
 156                             ((const struct rte_flow_item_ipv6 *)
 157                             item->mask)->hdr.proto)
 158                                 next_protocol =
 159                                     ((const struct rte_flow_item_ipv6 *)
 160                                       (item->spec))->hdr.proto &
 161                                     ((const struct rte_flow_item_ipv6 *)
 162                                       (item->mask))->hdr.proto;
 163                         if ((next_protocol == IPPROTO_IPIP ||
 164                             next_protocol == IPPROTO_IPV6) && tunnel_decap)
 165                                 attr->attr = 0;
 166                         break;
 167                 case RTE_FLOW_ITEM_TYPE_UDP:
 168                         if (!attr->tcp)
 169                                 attr->udp = 1;
 170                         break;
 171                 case RTE_FLOW_ITEM_TYPE_TCP:
 172                         if (!attr->udp)
 173                                 attr->tcp = 1;
 174                         break;
 175                 default:
 176                         break;
 177                 }
 178         }
 179         attr->valid = 1;
 180 }
 181
 182 /**
 183  * Convert rte_mtr_color to mlx5 color.
 184  *
 185  * @param[in] rcol
 186  *   rte_mtr_color.
 187  *
 188  * @return
 189  *   mlx5 color.
 190  */
 191 static int
 192 rte_col_2_mlx5_col(enum rte_color rcol)
 193 {
 194         switch (rcol) {
 195         case RTE_COLOR_GREEN:
 196                 return MLX5_FLOW_COLOR_GREEN;
 197         case RTE_COLOR_YELLOW:
 198                 return MLX5_FLOW_COLOR_YELLOW;
 199         case RTE_COLOR_RED:
 200                 return MLX5_FLOW_COLOR_RED;
 201         default:
 202                 break;
 203         }
 204         return MLX5_FLOW_COLOR_UNDEFINED;
 205 }
 206
 207 struct field_modify_info {
 208         uint32_t size; /* Size of field in protocol header, in bytes. */
 209         uint32_t offset; /* Offset of field in protocol header, in bytes. */
 210         enum mlx5_modification_field id;
 211 };
 212
 213 struct field_modify_info modify_eth[] = {
 214         {4,  0, MLX5_MODI_OUT_DMAC_47_16},
 215         {2,  4, MLX5_MODI_OUT_DMAC_15_0},
 216         {4,  6, MLX5_MODI_OUT_SMAC_47_16},
 217         {2, 10, MLX5_MODI_OUT_SMAC_15_0},
 218         {0, 0, 0},
 219 };
 220
 221 struct field_modify_info modify_vlan_out_first_vid[] = {
 222         /* Size in bits !!! */
 223         {12, 0, MLX5_MODI_OUT_FIRST_VID},
 224         {0, 0, 0},
 225 };
 226
 227 struct field_modify_info modify_ipv4[] = {
 228         {1,  1, MLX5_MODI_OUT_IP_DSCP},
 229         {1,  8, MLX5_MODI_OUT_IPV4_TTL},
 230         {4, 12, MLX5_MODI_OUT_SIPV4},
 231         {4, 16, MLX5_MODI_OUT_DIPV4},
 232         {0, 0, 0},
 233 };
 234
 235 struct field_modify_info modify_ipv6[] = {
 236         {1,  0, MLX5_MODI_OUT_IP_DSCP},
 237         {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
 238         {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
 239         {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
 240         {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
 241         {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
 242         {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
 243         {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
 244         {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
 245         {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
 246         {0, 0, 0},
 247 };
 248
 249 struct field_modify_info modify_udp[] = {
 250         {2, 0, MLX5_MODI_OUT_UDP_SPORT},
 251         {2, 2, MLX5_MODI_OUT_UDP_DPORT},
 252         {0, 0, 0},
 253 };
 254
 255 struct field_modify_info modify_tcp[] = {
 256         {2, 0, MLX5_MODI_OUT_TCP_SPORT},
 257         {2, 2, MLX5_MODI_OUT_TCP_DPORT},
 258         {4, 4, MLX5_MODI_OUT_TCP_SEQ_NUM},
 259         {4, 8, MLX5_MODI_OUT_TCP_ACK_NUM},
 260         {0, 0, 0},
 261 };
 262
 263 static void
 264 mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 265                           uint8_t next_protocol, uint64_t *item_flags,
 266                           int *tunnel)
 267 {
 268         MLX5_ASSERT(item->type == RTE_FLOW_ITEM_TYPE_IPV4 ||
 269                     item->type == RTE_FLOW_ITEM_TYPE_IPV6);
 270         if (next_protocol == IPPROTO_IPIP) {
 271                 *item_flags |= MLX5_FLOW_LAYER_IPIP;
 272                 *tunnel = 1;
 273         }
 274         if (next_protocol == IPPROTO_IPV6) {
 275                 *item_flags |= MLX5_FLOW_LAYER_IPV6_ENCAP;
 276                 *tunnel = 1;
 277         }
 278 }
 279
 280 /**
 281  * Acquire the synchronizing object to protect multithreaded access
 282  * to shared dv context. Lock occurs only if context is actually
 283  * shared, i.e. we have multiport IB device and representors are
 284  * created.
 285  *
 286  * @param[in] dev
 287  *   Pointer to the rte_eth_dev structure.
 288  */
 289 static void
 290 flow_dv_shared_lock(struct rte_eth_dev *dev)
 291 {
 292         struct mlx5_priv *priv = dev->data->dev_private;
 293         struct mlx5_dev_ctx_shared *sh = priv->sh;
 294
 295         if (sh->dv_refcnt > 1) {
 296                 int ret;
 297
 298                 ret = pthread_mutex_lock(&sh->dv_mutex);
 299                 MLX5_ASSERT(!ret);
 300                 (void)ret;
 301         }
 302 }
 303
 304 static void
 305 flow_dv_shared_unlock(struct rte_eth_dev *dev)
 306 {
 307         struct mlx5_priv *priv = dev->data->dev_private;
 308         struct mlx5_dev_ctx_shared *sh = priv->sh;
 309
 310         if (sh->dv_refcnt > 1) {
 311                 int ret;
 312
 313                 ret = pthread_mutex_unlock(&sh->dv_mutex);
 314                 MLX5_ASSERT(!ret);
 315                 (void)ret;
 316         }
 317 }
 318
 319 /* Update VLAN's VID/PCP based on input rte_flow_action.
 320  *
 321  * @param[in] action
 322  *   Pointer to struct rte_flow_action.
 323  * @param[out] vlan
 324  *   Pointer to struct rte_vlan_hdr.
 325  */
 326 static void
 327 mlx5_update_vlan_vid_pcp(const struct rte_flow_action *action,
 328                          struct rte_vlan_hdr *vlan)
 329 {
 330         uint16_t vlan_tci;
 331         if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP) {
 332                 vlan_tci =
 333                     ((const struct rte_flow_action_of_set_vlan_pcp *)
 334                                                action->conf)->vlan_pcp;
 335                 vlan_tci = vlan_tci << MLX5DV_FLOW_VLAN_PCP_SHIFT;
 336                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
 337                 vlan->vlan_tci |= vlan_tci;
 338         } else if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID) {
 339                 vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
 340                 vlan->vlan_tci |= rte_be_to_cpu_16
 341                     (((const struct rte_flow_action_of_set_vlan_vid *)
 342                                              action->conf)->vlan_vid);
 343         }
 344 }
 345
 346 /**
 347  * Fetch 1, 2, 3 or 4 byte field from the byte array
 348  * and return as unsigned integer in host-endian format.
 349  *
 350  * @param[in] data
 351  *   Pointer to data array.
 352  * @param[in] size
 353  *   Size of field to extract.
 354  *
 355  * @return
 356  *   converted field in host endian format.
 357  */
 358 static inline uint32_t
 359 flow_dv_fetch_field(const uint8_t *data, uint32_t size)
 360 {
 361         uint32_t ret;
 362
 363         switch (size) {
 364         case 1:
 365                 ret = *data;
 366                 break;
 367         case 2:
 368                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 369                 break;
 370         case 3:
 371                 ret = rte_be_to_cpu_16(*(const unaligned_uint16_t *)data);
 372                 ret = (ret << 8) | *(data + sizeof(uint16_t));
 373                 break;
 374         case 4:
 375                 ret = rte_be_to_cpu_32(*(const unaligned_uint32_t *)data);
 376                 break;
 377         default:
 378                 MLX5_ASSERT(false);
 379                 ret = 0;
 380                 break;
 381         }
 382         return ret;
 383 }
 384
 385 /**
 386  * Convert modify-header action to DV specification.
 387  *
 388  * Data length of each action is determined by provided field description
 389  * and the item mask. Data bit offset and width of each action is determined
 390  * by provided item mask.
 391  *
 392  * @param[in] item
 393  *   Pointer to item specification.
 394  * @param[in] field
 395  *   Pointer to field modification information.
 396  *     For MLX5_MODIFICATION_TYPE_SET specifies destination field.
 397  *     For MLX5_MODIFICATION_TYPE_ADD specifies destination field.
 398  *     For MLX5_MODIFICATION_TYPE_COPY specifies source field.
 399  * @param[in] dcopy
 400  *   Destination field info for MLX5_MODIFICATION_TYPE_COPY in @type.
 401  *   Negative offset value sets the same offset as source offset.
 402  *   size field is ignored, value is taken from source field.
 403  * @param[in,out] resource
 404  *   Pointer to the modify-header resource.
 405  * @param[in] type
 406  *   Type of modification.
 407  * @param[out] error
 408  *   Pointer to the error structure.
 409  *
 410  * @return
 411  *   0 on success, a negative errno value otherwise and rte_errno is set.
 412  */
 413 static int
 414 flow_dv_convert_modify_action(struct rte_flow_item *item,
 415                               struct field_modify_info *field,
 416                               struct field_modify_info *dcopy,
 417                               struct mlx5_flow_dv_modify_hdr_resource *resource,
 418                               uint32_t type, struct rte_flow_error *error)
 419 {
 420         uint32_t i = resource->actions_num;
 421         struct mlx5_modification_cmd *actions = resource->actions;
 422
 423         /*
 424          * The item and mask are provided in big-endian format.
 425          * The fields should be presented as in big-endian format either.
 426          * Mask must be always present, it defines the actual field width.
 427          */
 428         MLX5_ASSERT(item->mask);
 429         MLX5_ASSERT(field->size);
 430         do {
 431                 unsigned int size_b;
 432                 unsigned int off_b;
 433                 uint32_t mask;
 434                 uint32_t data;
 435
 436                 if (i >= MLX5_MAX_MODIFY_NUM)
 437                         return rte_flow_error_set(error, EINVAL,
 438                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 439                                  "too many items to modify");
 440                 /* Fetch variable byte size mask from the array. */
 441                 mask = flow_dv_fetch_field((const uint8_t *)item->mask +
 442                                            field->offset, field->size);
 443                 if (!mask) {
 444                         ++field;
 445                         continue;
 446                 }
 447                 /* Deduce actual data width in bits from mask value. */
 448                 off_b = rte_bsf32(mask);
 449                 size_b = sizeof(uint32_t) * CHAR_BIT -
 450                          off_b - __builtin_clz(mask);
 451                 MLX5_ASSERT(size_b);
 452                 size_b = size_b == sizeof(uint32_t) * CHAR_BIT ? 0 : size_b;
 453                 actions[i] = (struct mlx5_modification_cmd) {
 454                         .action_type = type,
 455                         .field = field->id,
 456                         .offset = off_b,
 457                         .length = size_b,
 458                 };
 459                 /* Convert entire record to expected big-endian format. */
 460                 actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 461                 if (type == MLX5_MODIFICATION_TYPE_COPY) {
 462                         MLX5_ASSERT(dcopy);
 463                         actions[i].dst_field = dcopy->id;
 464                         actions[i].dst_offset =
 465                                 (int)dcopy->offset < 0 ? off_b : dcopy->offset;
 466                         /* Convert entire record to big-endian format. */
 467                         actions[i].data1 = rte_cpu_to_be_32(actions[i].data1);
 468                 } else {
 469                         MLX5_ASSERT(item->spec);
 470                         data = flow_dv_fetch_field((const uint8_t *)item->spec +
 471                                                    field->offset, field->size);
 472                         /* Shift out the trailing masked bits from data. */
 473                         data = (data & mask) >> off_b;
 474                         actions[i].data1 = rte_cpu_to_be_32(data);
 475                 }
 476                 ++i;
 477                 ++field;
 478         } while (field->size);
 479         if (resource->actions_num == i)
 480                 return rte_flow_error_set(error, EINVAL,
 481                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 482                                           "invalid modification flow item");
 483         resource->actions_num = i;
 484         return 0;
 485 }
 486
 487 /**
 488  * Convert modify-header set IPv4 address action to DV specification.
 489  *
 490  * @param[in,out] resource
 491  *   Pointer to the modify-header resource.
 492  * @param[in] action
 493  *   Pointer to action specification.
 494  * @param[out] error
 495  *   Pointer to the error structure.
 496  *
 497  * @return
 498  *   0 on success, a negative errno value otherwise and rte_errno is set.
 499  */
 500 static int
 501 flow_dv_convert_action_modify_ipv4
 502                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 503                          const struct rte_flow_action *action,
 504                          struct rte_flow_error *error)
 505 {
 506         const struct rte_flow_action_set_ipv4 *conf =
 507                 (const struct rte_flow_action_set_ipv4 *)(action->conf);
 508         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
 509         struct rte_flow_item_ipv4 ipv4;
 510         struct rte_flow_item_ipv4 ipv4_mask;
 511
 512         memset(&ipv4, 0, sizeof(ipv4));
 513         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 514         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
 515                 ipv4.hdr.src_addr = conf->ipv4_addr;
 516                 ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
 517         } else {
 518                 ipv4.hdr.dst_addr = conf->ipv4_addr;
 519                 ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
 520         }
 521         item.spec = &ipv4;
 522         item.mask = &ipv4_mask;
 523         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
 524                                              MLX5_MODIFICATION_TYPE_SET, error);
 525 }
 526
 527 /**
 528  * Convert modify-header set IPv6 address action to DV specification.
 529  *
 530  * @param[in,out] resource
 531  *   Pointer to the modify-header resource.
 532  * @param[in] action
 533  *   Pointer to action specification.
 534  * @param[out] error
 535  *   Pointer to the error structure.
 536  *
 537  * @return
 538  *   0 on success, a negative errno value otherwise and rte_errno is set.
 539  */
 540 static int
 541 flow_dv_convert_action_modify_ipv6
 542                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 543                          const struct rte_flow_action *action,
 544                          struct rte_flow_error *error)
 545 {
 546         const struct rte_flow_action_set_ipv6 *conf =
 547                 (const struct rte_flow_action_set_ipv6 *)(action->conf);
 548         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
 549         struct rte_flow_item_ipv6 ipv6;
 550         struct rte_flow_item_ipv6 ipv6_mask;
 551
 552         memset(&ipv6, 0, sizeof(ipv6));
 553         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 554         if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
 555                 memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
 556                        sizeof(ipv6.hdr.src_addr));
 557                 memcpy(&ipv6_mask.hdr.src_addr,
 558                        &rte_flow_item_ipv6_mask.hdr.src_addr,
 559                        sizeof(ipv6.hdr.src_addr));
 560         } else {
 561                 memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
 562                        sizeof(ipv6.hdr.dst_addr));
 563                 memcpy(&ipv6_mask.hdr.dst_addr,
 564                        &rte_flow_item_ipv6_mask.hdr.dst_addr,
 565                        sizeof(ipv6.hdr.dst_addr));
 566         }
 567         item.spec = &ipv6;
 568         item.mask = &ipv6_mask;
 569         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
 570                                              MLX5_MODIFICATION_TYPE_SET, error);
 571 }
 572
 573 /**
 574  * Convert modify-header set MAC address action to DV specification.
 575  *
 576  * @param[in,out] resource
 577  *   Pointer to the modify-header resource.
 578  * @param[in] action
 579  *   Pointer to action specification.
 580  * @param[out] error
 581  *   Pointer to the error structure.
 582  *
 583  * @return
 584  *   0 on success, a negative errno value otherwise and rte_errno is set.
 585  */
 586 static int
 587 flow_dv_convert_action_modify_mac
 588                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 589                          const struct rte_flow_action *action,
 590                          struct rte_flow_error *error)
 591 {
 592         const struct rte_flow_action_set_mac *conf =
 593                 (const struct rte_flow_action_set_mac *)(action->conf);
 594         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
 595         struct rte_flow_item_eth eth;
 596         struct rte_flow_item_eth eth_mask;
 597
 598         memset(&eth, 0, sizeof(eth));
 599         memset(&eth_mask, 0, sizeof(eth_mask));
 600         if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
 601                 memcpy(&eth.src.addr_bytes, &conf->mac_addr,
 602                        sizeof(eth.src.addr_bytes));
 603                 memcpy(&eth_mask.src.addr_bytes,
 604                        &rte_flow_item_eth_mask.src.addr_bytes,
 605                        sizeof(eth_mask.src.addr_bytes));
 606         } else {
 607                 memcpy(&eth.dst.addr_bytes, &conf->mac_addr,
 608                        sizeof(eth.dst.addr_bytes));
 609                 memcpy(&eth_mask.dst.addr_bytes,
 610                        &rte_flow_item_eth_mask.dst.addr_bytes,
 611                        sizeof(eth_mask.dst.addr_bytes));
 612         }
 613         item.spec = &eth;
 614         item.mask = &eth_mask;
 615         return flow_dv_convert_modify_action(&item, modify_eth, NULL, resource,
 616                                              MLX5_MODIFICATION_TYPE_SET, error);
 617 }
 618
 619 /**
 620  * Convert modify-header set VLAN VID action to DV specification.
 621  *
 622  * @param[in,out] resource
 623  *   Pointer to the modify-header resource.
 624  * @param[in] action
 625  *   Pointer to action specification.
 626  * @param[out] error
 627  *   Pointer to the error structure.
 628  *
 629  * @return
 630  *   0 on success, a negative errno value otherwise and rte_errno is set.
 631  */
 632 static int
 633 flow_dv_convert_action_modify_vlan_vid
 634                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 635                          const struct rte_flow_action *action,
 636                          struct rte_flow_error *error)
 637 {
 638         const struct rte_flow_action_of_set_vlan_vid *conf =
 639                 (const struct rte_flow_action_of_set_vlan_vid *)(action->conf);
 640         int i = resource->actions_num;
 641         struct mlx5_modification_cmd *actions = resource->actions;
 642         struct field_modify_info *field = modify_vlan_out_first_vid;
 643
 644         if (i >= MLX5_MAX_MODIFY_NUM)
 645                 return rte_flow_error_set(error, EINVAL,
 646                          RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 647                          "too many items to modify");
 648         actions[i] = (struct mlx5_modification_cmd) {
 649                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 650                 .field = field->id,
 651                 .length = field->size,
 652                 .offset = field->offset,
 653         };
 654         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
 655         actions[i].data1 = conf->vlan_vid;
 656         actions[i].data1 = actions[i].data1 << 16;
 657         resource->actions_num = ++i;
 658         return 0;
 659 }
 660
 661 /**
 662  * Convert modify-header set TP action to DV specification.
 663  *
 664  * @param[in,out] resource
 665  *   Pointer to the modify-header resource.
 666  * @param[in] action
 667  *   Pointer to action specification.
 668  * @param[in] items
 669  *   Pointer to rte_flow_item objects list.
 670  * @param[in] attr
 671  *   Pointer to flow attributes structure.
 672  * @param[in] dev_flow
 673  *   Pointer to the sub flow.
 674  * @param[in] tunnel_decap
 675  *   Whether action is after tunnel decapsulation.
 676  * @param[out] error
 677  *   Pointer to the error structure.
 678  *
 679  * @return
 680  *   0 on success, a negative errno value otherwise and rte_errno is set.
 681  */
 682 static int
 683 flow_dv_convert_action_modify_tp
 684                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 685                          const struct rte_flow_action *action,
 686                          const struct rte_flow_item *items,
 687                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 688                          bool tunnel_decap, struct rte_flow_error *error)
 689 {
 690         const struct rte_flow_action_set_tp *conf =
 691                 (const struct rte_flow_action_set_tp *)(action->conf);
 692         struct rte_flow_item item;
 693         struct rte_flow_item_udp udp;
 694         struct rte_flow_item_udp udp_mask;
 695         struct rte_flow_item_tcp tcp;
 696         struct rte_flow_item_tcp tcp_mask;
 697         struct field_modify_info *field;
 698
 699         if (!attr->valid)
 700                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 701         if (attr->udp) {
 702                 memset(&udp, 0, sizeof(udp));
 703                 memset(&udp_mask, 0, sizeof(udp_mask));
 704                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 705                         udp.hdr.src_port = conf->port;
 706                         udp_mask.hdr.src_port =
 707                                         rte_flow_item_udp_mask.hdr.src_port;
 708                 } else {
 709                         udp.hdr.dst_port = conf->port;
 710                         udp_mask.hdr.dst_port =
 711                                         rte_flow_item_udp_mask.hdr.dst_port;
 712                 }
 713                 item.type = RTE_FLOW_ITEM_TYPE_UDP;
 714                 item.spec = &udp;
 715                 item.mask = &udp_mask;
 716                 field = modify_udp;
 717         } else {
 718                 MLX5_ASSERT(attr->tcp);
 719                 memset(&tcp, 0, sizeof(tcp));
 720                 memset(&tcp_mask, 0, sizeof(tcp_mask));
 721                 if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
 722                         tcp.hdr.src_port = conf->port;
 723                         tcp_mask.hdr.src_port =
 724                                         rte_flow_item_tcp_mask.hdr.src_port;
 725                 } else {
 726                         tcp.hdr.dst_port = conf->port;
 727                         tcp_mask.hdr.dst_port =
 728                                         rte_flow_item_tcp_mask.hdr.dst_port;
 729                 }
 730                 item.type = RTE_FLOW_ITEM_TYPE_TCP;
 731                 item.spec = &tcp;
 732                 item.mask = &tcp_mask;
 733                 field = modify_tcp;
 734         }
 735         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 736                                              MLX5_MODIFICATION_TYPE_SET, error);
 737 }
 738
 739 /**
 740  * Convert modify-header set TTL action to DV specification.
 741  *
 742  * @param[in,out] resource
 743  *   Pointer to the modify-header resource.
 744  * @param[in] action
 745  *   Pointer to action specification.
 746  * @param[in] items
 747  *   Pointer to rte_flow_item objects list.
 748  * @param[in] attr
 749  *   Pointer to flow attributes structure.
 750  * @param[in] dev_flow
 751  *   Pointer to the sub flow.
 752  * @param[in] tunnel_decap
 753  *   Whether action is after tunnel decapsulation.
 754  * @param[out] error
 755  *   Pointer to the error structure.
 756  *
 757  * @return
 758  *   0 on success, a negative errno value otherwise and rte_errno is set.
 759  */
 760 static int
 761 flow_dv_convert_action_modify_ttl
 762                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 763                          const struct rte_flow_action *action,
 764                          const struct rte_flow_item *items,
 765                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 766                          bool tunnel_decap, struct rte_flow_error *error)
 767 {
 768         const struct rte_flow_action_set_ttl *conf =
 769                 (const struct rte_flow_action_set_ttl *)(action->conf);
 770         struct rte_flow_item item;
 771         struct rte_flow_item_ipv4 ipv4;
 772         struct rte_flow_item_ipv4 ipv4_mask;
 773         struct rte_flow_item_ipv6 ipv6;
 774         struct rte_flow_item_ipv6 ipv6_mask;
 775         struct field_modify_info *field;
 776
 777         if (!attr->valid)
 778                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 779         if (attr->ipv4) {
 780                 memset(&ipv4, 0, sizeof(ipv4));
 781                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 782                 ipv4.hdr.time_to_live = conf->ttl_value;
 783                 ipv4_mask.hdr.time_to_live = 0xFF;
 784                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 785                 item.spec = &ipv4;
 786                 item.mask = &ipv4_mask;
 787                 field = modify_ipv4;
 788         } else {
 789                 MLX5_ASSERT(attr->ipv6);
 790                 memset(&ipv6, 0, sizeof(ipv6));
 791                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 792                 ipv6.hdr.hop_limits = conf->ttl_value;
 793                 ipv6_mask.hdr.hop_limits = 0xFF;
 794                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 795                 item.spec = &ipv6;
 796                 item.mask = &ipv6_mask;
 797                 field = modify_ipv6;
 798         }
 799         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 800                                              MLX5_MODIFICATION_TYPE_SET, error);
 801 }
 802
 803 /**
 804  * Convert modify-header decrement TTL action to DV specification.
 805  *
 806  * @param[in,out] resource
 807  *   Pointer to the modify-header resource.
 808  * @param[in] action
 809  *   Pointer to action specification.
 810  * @param[in] items
 811  *   Pointer to rte_flow_item objects list.
 812  * @param[in] attr
 813  *   Pointer to flow attributes structure.
 814  * @param[in] dev_flow
 815  *   Pointer to the sub flow.
 816  * @param[in] tunnel_decap
 817  *   Whether action is after tunnel decapsulation.
 818  * @param[out] error
 819  *   Pointer to the error structure.
 820  *
 821  * @return
 822  *   0 on success, a negative errno value otherwise and rte_errno is set.
 823  */
 824 static int
 825 flow_dv_convert_action_modify_dec_ttl
 826                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 827                          const struct rte_flow_item *items,
 828                          union flow_dv_attr *attr, struct mlx5_flow *dev_flow,
 829                          bool tunnel_decap, struct rte_flow_error *error)
 830 {
 831         struct rte_flow_item item;
 832         struct rte_flow_item_ipv4 ipv4;
 833         struct rte_flow_item_ipv4 ipv4_mask;
 834         struct rte_flow_item_ipv6 ipv6;
 835         struct rte_flow_item_ipv6 ipv6_mask;
 836         struct field_modify_info *field;
 837
 838         if (!attr->valid)
 839                 flow_dv_attr_init(items, attr, dev_flow, tunnel_decap);
 840         if (attr->ipv4) {
 841                 memset(&ipv4, 0, sizeof(ipv4));
 842                 memset(&ipv4_mask, 0, sizeof(ipv4_mask));
 843                 ipv4.hdr.time_to_live = 0xFF;
 844                 ipv4_mask.hdr.time_to_live = 0xFF;
 845                 item.type = RTE_FLOW_ITEM_TYPE_IPV4;
 846                 item.spec = &ipv4;
 847                 item.mask = &ipv4_mask;
 848                 field = modify_ipv4;
 849         } else {
 850                 MLX5_ASSERT(attr->ipv6);
 851                 memset(&ipv6, 0, sizeof(ipv6));
 852                 memset(&ipv6_mask, 0, sizeof(ipv6_mask));
 853                 ipv6.hdr.hop_limits = 0xFF;
 854                 ipv6_mask.hdr.hop_limits = 0xFF;
 855                 item.type = RTE_FLOW_ITEM_TYPE_IPV6;
 856                 item.spec = &ipv6;
 857                 item.mask = &ipv6_mask;
 858                 field = modify_ipv6;
 859         }
 860         return flow_dv_convert_modify_action(&item, field, NULL, resource,
 861                                              MLX5_MODIFICATION_TYPE_ADD, error);
 862 }
 863
 864 /**
 865  * Convert modify-header increment/decrement TCP Sequence number
 866  * to DV specification.
 867  *
 868  * @param[in,out] resource
 869  *   Pointer to the modify-header resource.
 870  * @param[in] action
 871  *   Pointer to action specification.
 872  * @param[out] error
 873  *   Pointer to the error structure.
 874  *
 875  * @return
 876  *   0 on success, a negative errno value otherwise and rte_errno is set.
 877  */
 878 static int
 879 flow_dv_convert_action_modify_tcp_seq
 880                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 881                          const struct rte_flow_action *action,
 882                          struct rte_flow_error *error)
 883 {
 884         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 885         uint64_t value = rte_be_to_cpu_32(*conf);
 886         struct rte_flow_item item;
 887         struct rte_flow_item_tcp tcp;
 888         struct rte_flow_item_tcp tcp_mask;
 889
 890         memset(&tcp, 0, sizeof(tcp));
 891         memset(&tcp_mask, 0, sizeof(tcp_mask));
 892         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ)
 893                 /*
 894                  * The HW has no decrement operation, only increment operation.
 895                  * To simulate decrement X from Y using increment operation
 896                  * we need to add UINT32_MAX X times to Y.
 897                  * Each adding of UINT32_MAX decrements Y by 1.
 898                  */
 899                 value *= UINT32_MAX;
 900         tcp.hdr.sent_seq = rte_cpu_to_be_32((uint32_t)value);
 901         tcp_mask.hdr.sent_seq = RTE_BE32(UINT32_MAX);
 902         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 903         item.spec = &tcp;
 904         item.mask = &tcp_mask;
 905         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 906                                              MLX5_MODIFICATION_TYPE_ADD, error);
 907 }
 908
 909 /**
 910  * Convert modify-header increment/decrement TCP Acknowledgment number
 911  * to DV specification.
 912  *
 913  * @param[in,out] resource
 914  *   Pointer to the modify-header resource.
 915  * @param[in] action
 916  *   Pointer to action specification.
 917  * @param[out] error
 918  *   Pointer to the error structure.
 919  *
 920  * @return
 921  *   0 on success, a negative errno value otherwise and rte_errno is set.
 922  */
 923 static int
 924 flow_dv_convert_action_modify_tcp_ack
 925                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 926                          const struct rte_flow_action *action,
 927                          struct rte_flow_error *error)
 928 {
 929         const rte_be32_t *conf = (const rte_be32_t *)(action->conf);
 930         uint64_t value = rte_be_to_cpu_32(*conf);
 931         struct rte_flow_item item;
 932         struct rte_flow_item_tcp tcp;
 933         struct rte_flow_item_tcp tcp_mask;
 934
 935         memset(&tcp, 0, sizeof(tcp));
 936         memset(&tcp_mask, 0, sizeof(tcp_mask));
 937         if (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK)
 938                 /*
 939                  * The HW has no decrement operation, only increment operation.
 940                  * To simulate decrement X from Y using increment operation
 941                  * we need to add UINT32_MAX X times to Y.
 942                  * Each adding of UINT32_MAX decrements Y by 1.
 943                  */
 944                 value *= UINT32_MAX;
 945         tcp.hdr.recv_ack = rte_cpu_to_be_32((uint32_t)value);
 946         tcp_mask.hdr.recv_ack = RTE_BE32(UINT32_MAX);
 947         item.type = RTE_FLOW_ITEM_TYPE_TCP;
 948         item.spec = &tcp;
 949         item.mask = &tcp_mask;
 950         return flow_dv_convert_modify_action(&item, modify_tcp, NULL, resource,
 951                                              MLX5_MODIFICATION_TYPE_ADD, error);
 952 }
 953
 954 static enum mlx5_modification_field reg_to_field[] = {
 955         [REG_NONE] = MLX5_MODI_OUT_NONE,
 956         [REG_A] = MLX5_MODI_META_DATA_REG_A,
 957         [REG_B] = MLX5_MODI_META_DATA_REG_B,
 958         [REG_C_0] = MLX5_MODI_META_REG_C_0,
 959         [REG_C_1] = MLX5_MODI_META_REG_C_1,
 960         [REG_C_2] = MLX5_MODI_META_REG_C_2,
 961         [REG_C_3] = MLX5_MODI_META_REG_C_3,
 962         [REG_C_4] = MLX5_MODI_META_REG_C_4,
 963         [REG_C_5] = MLX5_MODI_META_REG_C_5,
 964         [REG_C_6] = MLX5_MODI_META_REG_C_6,
 965         [REG_C_7] = MLX5_MODI_META_REG_C_7,
 966 };
 967
 968 /**
 969  * Convert register set to DV specification.
 970  *
 971  * @param[in,out] resource
 972  *   Pointer to the modify-header resource.
 973  * @param[in] action
 974  *   Pointer to action specification.
 975  * @param[out] error
 976  *   Pointer to the error structure.
 977  *
 978  * @return
 979  *   0 on success, a negative errno value otherwise and rte_errno is set.
 980  */
 981 static int
 982 flow_dv_convert_action_set_reg
 983                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
 984                          const struct rte_flow_action *action,
 985                          struct rte_flow_error *error)
 986 {
 987         const struct mlx5_rte_flow_action_set_tag *conf = action->conf;
 988         struct mlx5_modification_cmd *actions = resource->actions;
 989         uint32_t i = resource->actions_num;
 990
 991         if (i >= MLX5_MAX_MODIFY_NUM)
 992                 return rte_flow_error_set(error, EINVAL,
 993                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 994                                           "too many items to modify");
 995         MLX5_ASSERT(conf->id != REG_NONE);
 996         MLX5_ASSERT(conf->id < RTE_DIM(reg_to_field));
 997         actions[i] = (struct mlx5_modification_cmd) {
 998                 .action_type = MLX5_MODIFICATION_TYPE_SET,
 999                 .field = reg_to_field[conf->id],
1000         };
1001         actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
1002         actions[i].data1 = rte_cpu_to_be_32(conf->data);
1003         ++i;
1004         resource->actions_num = i;
1005         return 0;
1006 }
1007
1008 /**
1009  * Convert SET_TAG action to DV specification.
1010  *
1011  * @param[in] dev
1012  *   Pointer to the rte_eth_dev structure.
1013  * @param[in,out] resource
1014  *   Pointer to the modify-header resource.
1015  * @param[in] conf
1016  *   Pointer to action specification.
1017  * @param[out] error
1018  *   Pointer to the error structure.
1019  *
1020  * @return
1021  *   0 on success, a negative errno value otherwise and rte_errno is set.
1022  */
1023 static int
1024 flow_dv_convert_action_set_tag
1025                         (struct rte_eth_dev *dev,
1026                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1027                          const struct rte_flow_action_set_tag *conf,
1028                          struct rte_flow_error *error)
1029 {
1030         rte_be32_t data = rte_cpu_to_be_32(conf->data);
1031         rte_be32_t mask = rte_cpu_to_be_32(conf->mask);
1032         struct rte_flow_item item = {
1033                 .spec = &data,
1034                 .mask = &mask,
1035         };
1036         struct field_modify_info reg_c_x[] = {
1037                 [1] = {0, 0, 0},
1038         };
1039         enum mlx5_modification_field reg_type;
1040         int ret;
1041
1042         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
1043         if (ret < 0)
1044                 return ret;
1045         MLX5_ASSERT(ret != REG_NONE);
1046         MLX5_ASSERT((unsigned int)ret < RTE_DIM(reg_to_field));
1047         reg_type = reg_to_field[ret];
1048         MLX5_ASSERT(reg_type > 0);
1049         reg_c_x[0] = (struct field_modify_info){4, 0, reg_type};
1050         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1051                                              MLX5_MODIFICATION_TYPE_SET, error);
1052 }
1053
1054 /**
1055  * Convert internal COPY_REG action to DV specification.
1056  *
1057  * @param[in] dev
1058  *   Pointer to the rte_eth_dev structure.
1059  * @param[in,out] res
1060  *   Pointer to the modify-header resource.
1061  * @param[in] action
1062  *   Pointer to action specification.
1063  * @param[out] error
1064  *   Pointer to the error structure.
1065  *
1066  * @return
1067  *   0 on success, a negative errno value otherwise and rte_errno is set.
1068  */
1069 static int
1070 flow_dv_convert_action_copy_mreg(struct rte_eth_dev *dev,
1071                                  struct mlx5_flow_dv_modify_hdr_resource *res,
1072                                  const struct rte_flow_action *action,
1073                                  struct rte_flow_error *error)
1074 {
1075         const struct mlx5_flow_action_copy_mreg *conf = action->conf;
1076         rte_be32_t mask = RTE_BE32(UINT32_MAX);
1077         struct rte_flow_item item = {
1078                 .spec = NULL,
1079                 .mask = &mask,
1080         };
1081         struct field_modify_info reg_src[] = {
1082                 {4, 0, reg_to_field[conf->src]},
1083                 {0, 0, 0},
1084         };
1085         struct field_modify_info reg_dst = {
1086                 .offset = 0,
1087                 .id = reg_to_field[conf->dst],
1088         };
1089         /* Adjust reg_c[0] usage according to reported mask. */
1090         if (conf->dst == REG_C_0 || conf->src == REG_C_0) {
1091                 struct mlx5_priv *priv = dev->data->dev_private;
1092                 uint32_t reg_c0 = priv->sh->dv_regc0_mask;
1093
1094                 MLX5_ASSERT(reg_c0);
1095                 MLX5_ASSERT(priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY);
1096                 if (conf->dst == REG_C_0) {
1097                         /* Copy to reg_c[0], within mask only. */
1098                         reg_dst.offset = rte_bsf32(reg_c0);
1099                         /*
1100                          * Mask is ignoring the enianness, because
1101                          * there is no conversion in datapath.
1102                          */
1103 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1104                         /* Copy from destination lower bits to reg_c[0]. */
1105                         mask = reg_c0 >> reg_dst.offset;
1106 #else
1107                         /* Copy from destination upper bits to reg_c[0]. */
1108                         mask = reg_c0 << (sizeof(reg_c0) * CHAR_BIT -
1109                                           rte_fls_u32(reg_c0));
1110 #endif
1111                 } else {
1112                         mask = rte_cpu_to_be_32(reg_c0);
1113 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1114                         /* Copy from reg_c[0] to destination lower bits. */
1115                         reg_dst.offset = 0;
1116 #else
1117                         /* Copy from reg_c[0] to destination upper bits. */
1118                         reg_dst.offset = sizeof(reg_c0) * CHAR_BIT -
1119                                          (rte_fls_u32(reg_c0) -
1120                                           rte_bsf32(reg_c0));
1121 #endif
1122                 }
1123         }
1124         return flow_dv_convert_modify_action(&item,
1125                                              reg_src, &reg_dst, res,
1126                                              MLX5_MODIFICATION_TYPE_COPY,
1127                                              error);
1128 }
1129
1130 /**
1131  * Convert MARK action to DV specification. This routine is used
1132  * in extensive metadata only and requires metadata register to be
1133  * handled. In legacy mode hardware tag resource is engaged.
1134  *
1135  * @param[in] dev
1136  *   Pointer to the rte_eth_dev structure.
1137  * @param[in] conf
1138  *   Pointer to MARK action specification.
1139  * @param[in,out] resource
1140  *   Pointer to the modify-header resource.
1141  * @param[out] error
1142  *   Pointer to the error structure.
1143  *
1144  * @return
1145  *   0 on success, a negative errno value otherwise and rte_errno is set.
1146  */
1147 static int
1148 flow_dv_convert_action_mark(struct rte_eth_dev *dev,
1149                             const struct rte_flow_action_mark *conf,
1150                             struct mlx5_flow_dv_modify_hdr_resource *resource,
1151                             struct rte_flow_error *error)
1152 {
1153         struct mlx5_priv *priv = dev->data->dev_private;
1154         rte_be32_t mask = rte_cpu_to_be_32(MLX5_FLOW_MARK_MASK &
1155                                            priv->sh->dv_mark_mask);
1156         rte_be32_t data = rte_cpu_to_be_32(conf->id) & mask;
1157         struct rte_flow_item item = {
1158                 .spec = &data,
1159                 .mask = &mask,
1160         };
1161         struct field_modify_info reg_c_x[] = {
1162                 {4, 0, 0}, /* dynamic instead of MLX5_MODI_META_REG_C_1. */
1163                 {0, 0, 0},
1164         };
1165         int reg;
1166
1167         if (!mask)
1168                 return rte_flow_error_set(error, EINVAL,
1169                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1170                                           NULL, "zero mark action mask");
1171         reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1172         if (reg < 0)
1173                 return reg;
1174         MLX5_ASSERT(reg > 0);
1175         if (reg == REG_C_0) {
1176                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1177                 uint32_t shl_c0 = rte_bsf32(msk_c0);
1178
1179                 data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
1180                 mask = rte_cpu_to_be_32(mask) & msk_c0;
1181                 mask = rte_cpu_to_be_32(mask << shl_c0);
1182         }
1183         reg_c_x[0].id = reg_to_field[reg];
1184         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1185                                              MLX5_MODIFICATION_TYPE_SET, error);
1186 }
1187
1188 /**
1189  * Get metadata register index for specified steering domain.
1190  *
1191  * @param[in] dev
1192  *   Pointer to the rte_eth_dev structure.
1193  * @param[in] attr
1194  *   Attributes of flow to determine steering domain.
1195  * @param[out] error
1196  *   Pointer to the error structure.
1197  *
1198  * @return
1199  *   positive index on success, a negative errno value otherwise
1200  *   and rte_errno is set.
1201  */
1202 static enum modify_reg
1203 flow_dv_get_metadata_reg(struct rte_eth_dev *dev,
1204                          const struct rte_flow_attr *attr,
1205                          struct rte_flow_error *error)
1206 {
1207         int reg =
1208                 mlx5_flow_get_reg_id(dev, attr->transfer ?
1209                                           MLX5_METADATA_FDB :
1210                                             attr->egress ?
1211                                             MLX5_METADATA_TX :
1212                                             MLX5_METADATA_RX, 0, error);
1213         if (reg < 0)
1214                 return rte_flow_error_set(error,
1215                                           ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1216                                           NULL, "unavailable "
1217                                           "metadata register");
1218         return reg;
1219 }
1220
1221 /**
1222  * Convert SET_META action to DV specification.
1223  *
1224  * @param[in] dev
1225  *   Pointer to the rte_eth_dev structure.
1226  * @param[in,out] resource
1227  *   Pointer to the modify-header resource.
1228  * @param[in] attr
1229  *   Attributes of flow that includes this item.
1230  * @param[in] conf
1231  *   Pointer to action specification.
1232  * @param[out] error
1233  *   Pointer to the error structure.
1234  *
1235  * @return
1236  *   0 on success, a negative errno value otherwise and rte_errno is set.
1237  */
1238 static int
1239 flow_dv_convert_action_set_meta
1240                         (struct rte_eth_dev *dev,
1241                          struct mlx5_flow_dv_modify_hdr_resource *resource,
1242                          const struct rte_flow_attr *attr,
1243                          const struct rte_flow_action_set_meta *conf,
1244                          struct rte_flow_error *error)
1245 {
1246         uint32_t data = conf->data;
1247         uint32_t mask = conf->mask;
1248         struct rte_flow_item item = {
1249                 .spec = &data,
1250                 .mask = &mask,
1251         };
1252         struct field_modify_info reg_c_x[] = {
1253                 [1] = {0, 0, 0},
1254         };
1255         int reg = flow_dv_get_metadata_reg(dev, attr, error);
1256
1257         if (reg < 0)
1258                 return reg;
1259         /*
1260          * In datapath code there is no endianness
1261          * coversions for perfromance reasons, all
1262          * pattern conversions are done in rte_flow.
1263          */
1264         if (reg == REG_C_0) {
1265                 struct mlx5_priv *priv = dev->data->dev_private;
1266                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
1267                 uint32_t shl_c0;
1268
1269                 MLX5_ASSERT(msk_c0);
1270 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
1271                 shl_c0 = rte_bsf32(msk_c0);
1272 #else
1273                 shl_c0 = sizeof(msk_c0) * CHAR_BIT - rte_fls_u32(msk_c0);
1274 #endif
1275                 mask <<= shl_c0;
1276                 data <<= shl_c0;
1277                 MLX5_ASSERT(!(~msk_c0 & rte_cpu_to_be_32(mask)));
1278         }
1279         reg_c_x[0] = (struct field_modify_info){4, 0, reg_to_field[reg]};
1280         /* The routine expects parameters in memory as big-endian ones. */
1281         return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
1282                                              MLX5_MODIFICATION_TYPE_SET, error);
1283 }
1284
1285 /**
1286  * Convert modify-header set IPv4 DSCP action to DV specification.
1287  *
1288  * @param[in,out] resource
1289  *   Pointer to the modify-header resource.
1290  * @param[in] action
1291  *   Pointer to action specification.
1292  * @param[out] error
1293  *   Pointer to the error structure.
1294  *
1295  * @return
1296  *   0 on success, a negative errno value otherwise and rte_errno is set.
1297  */
1298 static int
1299 flow_dv_convert_action_modify_ipv4_dscp
1300                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1301                          const struct rte_flow_action *action,
1302                          struct rte_flow_error *error)
1303 {
1304         const struct rte_flow_action_set_dscp *conf =
1305                 (const struct rte_flow_action_set_dscp *)(action->conf);
1306         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
1307         struct rte_flow_item_ipv4 ipv4;
1308         struct rte_flow_item_ipv4 ipv4_mask;
1309
1310         memset(&ipv4, 0, sizeof(ipv4));
1311         memset(&ipv4_mask, 0, sizeof(ipv4_mask));
1312         ipv4.hdr.type_of_service = conf->dscp;
1313         ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
1314         item.spec = &ipv4;
1315         item.mask = &ipv4_mask;
1316         return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
1317                                              MLX5_MODIFICATION_TYPE_SET, error);
1318 }
1319
1320 /**
1321  * Convert modify-header set IPv6 DSCP action to DV specification.
1322  *
1323  * @param[in,out] resource
1324  *   Pointer to the modify-header resource.
1325  * @param[in] action
1326  *   Pointer to action specification.
1327  * @param[out] error
1328  *   Pointer to the error structure.
1329  *
1330  * @return
1331  *   0 on success, a negative errno value otherwise and rte_errno is set.
1332  */
1333 static int
1334 flow_dv_convert_action_modify_ipv6_dscp
1335                         (struct mlx5_flow_dv_modify_hdr_resource *resource,
1336                          const struct rte_flow_action *action,
1337                          struct rte_flow_error *error)
1338 {
1339         const struct rte_flow_action_set_dscp *conf =
1340                 (const struct rte_flow_action_set_dscp *)(action->conf);
1341         struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
1342         struct rte_flow_item_ipv6 ipv6;
1343         struct rte_flow_item_ipv6 ipv6_mask;
1344
1345         memset(&ipv6, 0, sizeof(ipv6));
1346         memset(&ipv6_mask, 0, sizeof(ipv6_mask));
1347         /*
1348          * Even though the DSCP bits offset of IPv6 is not byte aligned,
1349          * rdma-core only accept the DSCP bits byte aligned start from
1350          * bit 0 to 5 as to be compatible with IPv4. No need to shift the
1351          * bits in IPv6 case as rdma-core requires byte aligned value.
1352          */
1353         ipv6.hdr.vtc_flow = conf->dscp;
1354         ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
1355         item.spec = &ipv6;
1356         item.mask = &ipv6_mask;
1357         return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
1358                                              MLX5_MODIFICATION_TYPE_SET, error);
1359 }
1360
1361 /**
1362  * Validate MARK item.
1363  *
1364  * @param[in] dev
1365  *   Pointer to the rte_eth_dev structure.
1366  * @param[in] item
1367  *   Item specification.
1368  * @param[in] attr
1369  *   Attributes of flow that includes this item.
1370  * @param[out] error
1371  *   Pointer to error structure.
1372  *
1373  * @return
1374  *   0 on success, a negative errno value otherwise and rte_errno is set.
1375  */
1376 static int
1377 flow_dv_validate_item_mark(struct rte_eth_dev *dev,
1378                            const struct rte_flow_item *item,
1379                            const struct rte_flow_attr *attr __rte_unused,
1380                            struct rte_flow_error *error)
1381 {
1382         struct mlx5_priv *priv = dev->data->dev_private;
1383         struct mlx5_dev_config *config = &priv->config;
1384         const struct rte_flow_item_mark *spec = item->spec;
1385         const struct rte_flow_item_mark *mask = item->mask;
1386         const struct rte_flow_item_mark nic_mask = {
1387                 .id = priv->sh->dv_mark_mask,
1388         };
1389         int ret;
1390
1391         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
1392                 return rte_flow_error_set(error, ENOTSUP,
1393                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1394                                           "extended metadata feature"
1395                                           " isn't enabled");
1396         if (!mlx5_flow_ext_mreg_supported(dev))
1397                 return rte_flow_error_set(error, ENOTSUP,
1398                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1399                                           "extended metadata register"
1400                                           " isn't supported");
1401         if (!nic_mask.id)
1402                 return rte_flow_error_set(error, ENOTSUP,
1403                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1404                                           "extended metadata register"
1405                                           " isn't available");
1406         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
1407         if (ret < 0)
1408                 return ret;
1409         if (!spec)
1410                 return rte_flow_error_set(error, EINVAL,
1411                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1412                                           item->spec,
1413                                           "data cannot be empty");
1414         if (spec->id >= (MLX5_FLOW_MARK_MAX & nic_mask.id))
1415                 return rte_flow_error_set(error, EINVAL,
1416                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
1417                                           &spec->id,
1418                                           "mark id exceeds the limit");
1419         if (!mask)
1420                 mask = &nic_mask;
1421         if (!mask->id)
1422                 return rte_flow_error_set(error, EINVAL,
1423                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1424                                         "mask cannot be zero");
1425
1426         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1427                                         (const uint8_t *)&nic_mask,
1428                                         sizeof(struct rte_flow_item_mark),
1429                                         error);
1430         if (ret < 0)
1431                 return ret;
1432         return 0;
1433 }
1434
1435 /**
1436  * Validate META item.
1437  *
1438  * @param[in] dev
1439  *   Pointer to the rte_eth_dev structure.
1440  * @param[in] item
1441  *   Item specification.
1442  * @param[in] attr
1443  *   Attributes of flow that includes this item.
1444  * @param[out] error
1445  *   Pointer to error structure.
1446  *
1447  * @return
1448  *   0 on success, a negative errno value otherwise and rte_errno is set.
1449  */
1450 static int
1451 flow_dv_validate_item_meta(struct rte_eth_dev *dev __rte_unused,
1452                            const struct rte_flow_item *item,
1453                            const struct rte_flow_attr *attr,
1454                            struct rte_flow_error *error)
1455 {
1456         struct mlx5_priv *priv = dev->data->dev_private;
1457         struct mlx5_dev_config *config = &priv->config;
1458         const struct rte_flow_item_meta *spec = item->spec;
1459         const struct rte_flow_item_meta *mask = item->mask;
1460         struct rte_flow_item_meta nic_mask = {
1461                 .data = UINT32_MAX
1462         };
1463         int reg;
1464         int ret;
1465
1466         if (!spec)
1467                 return rte_flow_error_set(error, EINVAL,
1468                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1469                                           item->spec,
1470                                           "data cannot be empty");
1471         if (config->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
1472                 if (!mlx5_flow_ext_mreg_supported(dev))
1473                         return rte_flow_error_set(error, ENOTSUP,
1474                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1475                                           "extended metadata register"
1476                                           " isn't supported");
1477                 reg = flow_dv_get_metadata_reg(dev, attr, error);
1478                 if (reg < 0)
1479                         return reg;
1480                 if (reg == REG_B)
1481                         return rte_flow_error_set(error, ENOTSUP,
1482                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1483                                           "match on reg_b "
1484                                           "isn't supported");
1485                 if (reg != REG_A)
1486                         nic_mask.data = priv->sh->dv_meta_mask;
1487         }
1488         if (!mask)
1489                 mask = &rte_flow_item_meta_mask;
1490         if (!mask->data)
1491                 return rte_flow_error_set(error, EINVAL,
1492                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1493                                         "mask cannot be zero");
1494
1495         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1496                                         (const uint8_t *)&nic_mask,
1497                                         sizeof(struct rte_flow_item_meta),
1498                                         error);
1499         return ret;
1500 }
1501
1502 /**
1503  * Validate TAG item.
1504  *
1505  * @param[in] dev
1506  *   Pointer to the rte_eth_dev structure.
1507  * @param[in] item
1508  *   Item specification.
1509  * @param[in] attr
1510  *   Attributes of flow that includes this item.
1511  * @param[out] error
1512  *   Pointer to error structure.
1513  *
1514  * @return
1515  *   0 on success, a negative errno value otherwise and rte_errno is set.
1516  */
1517 static int
1518 flow_dv_validate_item_tag(struct rte_eth_dev *dev,
1519                           const struct rte_flow_item *item,
1520                           const struct rte_flow_attr *attr __rte_unused,
1521                           struct rte_flow_error *error)
1522 {
1523         const struct rte_flow_item_tag *spec = item->spec;
1524         const struct rte_flow_item_tag *mask = item->mask;
1525         const struct rte_flow_item_tag nic_mask = {
1526                 .data = RTE_BE32(UINT32_MAX),
1527                 .index = 0xff,
1528         };
1529         int ret;
1530
1531         if (!mlx5_flow_ext_mreg_supported(dev))
1532                 return rte_flow_error_set(error, ENOTSUP,
1533                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1534                                           "extensive metadata register"
1535                                           " isn't supported");
1536         if (!spec)
1537                 return rte_flow_error_set(error, EINVAL,
1538                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
1539                                           item->spec,
1540                                           "data cannot be empty");
1541         if (!mask)
1542                 mask = &rte_flow_item_tag_mask;
1543         if (!mask->data)
1544                 return rte_flow_error_set(error, EINVAL,
1545                                         RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1546                                         "mask cannot be zero");
1547
1548         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1549                                         (const uint8_t *)&nic_mask,
1550                                         sizeof(struct rte_flow_item_tag),
1551                                         error);
1552         if (ret < 0)
1553                 return ret;
1554         if (mask->index != 0xff)
1555                 return rte_flow_error_set(error, EINVAL,
1556                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, NULL,
1557                                           "partial mask for tag index"
1558                                           " is not supported");
1559         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, spec->index, error);
1560         if (ret < 0)
1561                 return ret;
1562         MLX5_ASSERT(ret != REG_NONE);
1563         return 0;
1564 }
1565
1566 /**
1567  * Validate vport item.
1568  *
1569  * @param[in] dev
1570  *   Pointer to the rte_eth_dev structure.
1571  * @param[in] item
1572  *   Item specification.
1573  * @param[in] attr
1574  *   Attributes of flow that includes this item.
1575  * @param[in] item_flags
1576  *   Bit-fields that holds the items detected until now.
1577  * @param[out] error
1578  *   Pointer to error structure.
1579  *
1580  * @return
1581  *   0 on success, a negative errno value otherwise and rte_errno is set.
1582  */
1583 static int
1584 flow_dv_validate_item_port_id(struct rte_eth_dev *dev,
1585                               const struct rte_flow_item *item,
1586                               const struct rte_flow_attr *attr,
1587                               uint64_t item_flags,
1588                               struct rte_flow_error *error)
1589 {
1590         const struct rte_flow_item_port_id *spec = item->spec;
1591         const struct rte_flow_item_port_id *mask = item->mask;
1592         const struct rte_flow_item_port_id switch_mask = {
1593                         .id = 0xffffffff,
1594         };
1595         struct mlx5_priv *esw_priv;
1596         struct mlx5_priv *dev_priv;
1597         int ret;
1598
1599         if (!attr->transfer)
1600                 return rte_flow_error_set(error, EINVAL,
1601                                           RTE_FLOW_ERROR_TYPE_ITEM,
1602                                           NULL,
1603                                           "match on port id is valid only"
1604                                           " when transfer flag is enabled");
1605         if (item_flags & MLX5_FLOW_ITEM_PORT_ID)
1606                 return rte_flow_error_set(error, ENOTSUP,
1607                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1608                                           "multiple source ports are not"
1609                                           " supported");
1610         if (!mask)
1611                 mask = &switch_mask;
1612         if (mask->id != 0xffffffff)
1613                 return rte_flow_error_set(error, ENOTSUP,
1614                                            RTE_FLOW_ERROR_TYPE_ITEM_MASK,
1615                                            mask,
1616                                            "no support for partial mask on"
1617                                            " \"id\" field");
1618         ret = mlx5_flow_item_acceptable
1619                                 (item, (const uint8_t *)mask,
1620                                  (const uint8_t *)&rte_flow_item_port_id_mask,
1621                                  sizeof(struct rte_flow_item_port_id),
1622                                  error);
1623         if (ret)
1624                 return ret;
1625         if (!spec)
1626                 return 0;
1627         esw_priv = mlx5_port_to_eswitch_info(spec->id, false);
1628         if (!esw_priv)
1629                 return rte_flow_error_set(error, rte_errno,
1630                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1631                                           "failed to obtain E-Switch info for"
1632                                           " port");
1633         dev_priv = mlx5_dev_to_eswitch_info(dev);
1634         if (!dev_priv)
1635                 return rte_flow_error_set(error, rte_errno,
1636                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1637                                           NULL,
1638                                           "failed to obtain E-Switch info");
1639         if (esw_priv->domain_id != dev_priv->domain_id)
1640                 return rte_flow_error_set(error, EINVAL,
1641                                           RTE_FLOW_ERROR_TYPE_ITEM_SPEC, spec,
1642                                           "cannot match on a port from a"
1643                                           " different E-Switch");
1644         return 0;
1645 }
1646
1647 /**
1648  * Validate VLAN item.
1649  *
1650  * @param[in] item
1651  *   Item specification.
1652  * @param[in] item_flags
1653  *   Bit-fields that holds the items detected until now.
1654  * @param[in] dev
1655  *   Ethernet device flow is being created on.
1656  * @param[out] error
1657  *   Pointer to error structure.
1658  *
1659  * @return
1660  *   0 on success, a negative errno value otherwise and rte_errno is set.
1661  */
1662 static int
1663 flow_dv_validate_item_vlan(const struct rte_flow_item *item,
1664                            uint64_t item_flags,
1665                            struct rte_eth_dev *dev,
1666                            struct rte_flow_error *error)
1667 {
1668         const struct rte_flow_item_vlan *mask = item->mask;
1669         const struct rte_flow_item_vlan nic_mask = {
1670                 .tci = RTE_BE16(UINT16_MAX),
1671                 .inner_type = RTE_BE16(UINT16_MAX),
1672         };
1673         const int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
1674         int ret;
1675         const uint64_t l34m = tunnel ? (MLX5_FLOW_LAYER_INNER_L3 |
1676                                         MLX5_FLOW_LAYER_INNER_L4) :
1677                                        (MLX5_FLOW_LAYER_OUTER_L3 |
1678                                         MLX5_FLOW_LAYER_OUTER_L4);
1679         const uint64_t vlanm = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
1680                                         MLX5_FLOW_LAYER_OUTER_VLAN;
1681
1682         if (item_flags & vlanm)
1683                 return rte_flow_error_set(error, EINVAL,
1684                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1685                                           "multiple VLAN layers not supported");
1686         else if ((item_flags & l34m) != 0)
1687                 return rte_flow_error_set(error, EINVAL,
1688                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1689                                           "VLAN cannot follow L3/L4 layer");
1690         if (!mask)
1691                 mask = &rte_flow_item_vlan_mask;
1692         ret = mlx5_flow_item_acceptable(item, (const uint8_t *)mask,
1693                                         (const uint8_t *)&nic_mask,
1694                                         sizeof(struct rte_flow_item_vlan),
1695                                         error);
1696         if (ret)
1697                 return ret;
1698         if (!tunnel && mask->tci != RTE_BE16(0x0fff)) {
1699                 struct mlx5_priv *priv = dev->data->dev_private;
1700
1701                 if (priv->vmwa_context) {
1702                         /*
1703                          * Non-NULL context means we have a virtual machine
1704                          * and SR-IOV enabled, we have to create VLAN interface
1705                          * to make hypervisor to setup E-Switch vport
1706                          * context correctly. We avoid creating the multiple
1707                          * VLAN interfaces, so we cannot support VLAN tag mask.
1708                          */
1709                         return rte_flow_error_set(error, EINVAL,
1710                                                   RTE_FLOW_ERROR_TYPE_ITEM,
1711                                                   item,
1712                                                   "VLAN tag mask is not"
1713                                                   " supported in virtual"
1714                                                   " environment");
1715                 }
1716         }
1717         return 0;
1718 }
1719
1720 /*
1721  * GTP flags are contained in 1 byte of the format:
1722  * -------------------------------------------
1723  * | bit   | 0 - 2   | 3  | 4   | 5 | 6 | 7  |
1724  * |-----------------------------------------|
1725  * | value | Version | PT | Res | E | S | PN |
1726  * -------------------------------------------
1727  *
1728  * Matching is supported only for GTP flags E, S, PN.
1729  */
1730 #define MLX5_GTP_FLAGS_MASK     0x07
1731
1732 /**
1733  * Validate GTP item.
1734  *
1735  * @param[in] dev
1736  *   Pointer to the rte_eth_dev structure.
1737  * @param[in] item
1738  *   Item specification.
1739  * @param[in] item_flags
1740  *   Bit-fields that holds the items detected until now.
1741  * @param[out] error
1742  *   Pointer to error structure.
1743  *
1744  * @return
1745  *   0 on success, a negative errno value otherwise and rte_errno is set.
1746  */
1747 static int
1748 flow_dv_validate_item_gtp(struct rte_eth_dev *dev,
1749                           const struct rte_flow_item *item,
1750                           uint64_t item_flags,
1751                           struct rte_flow_error *error)
1752 {
1753         struct mlx5_priv *priv = dev->data->dev_private;
1754         const struct rte_flow_item_gtp *spec = item->spec;
1755         const struct rte_flow_item_gtp *mask = item->mask;
1756         const struct rte_flow_item_gtp nic_mask = {
1757                 .v_pt_rsv_flags = MLX5_GTP_FLAGS_MASK,
1758                 .msg_type = 0xff,
1759                 .teid = RTE_BE32(0xffffffff),
1760         };
1761
1762         if (!priv->config.hca_attr.tunnel_stateless_gtp)
1763                 return rte_flow_error_set(error, ENOTSUP,
1764                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1765                                           "GTP support is not enabled");
1766         if (item_flags & MLX5_FLOW_LAYER_TUNNEL)
1767                 return rte_flow_error_set(error, ENOTSUP,
1768                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1769                                           "multiple tunnel layers not"
1770                                           " supported");
1771         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_L4_UDP))
1772                 return rte_flow_error_set(error, EINVAL,
1773                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1774                                           "no outer UDP layer found");
1775         if (!mask)
1776                 mask = &rte_flow_item_gtp_mask;
1777         if (spec && spec->v_pt_rsv_flags & ~MLX5_GTP_FLAGS_MASK)
1778                 return rte_flow_error_set(error, ENOTSUP,
1779                                           RTE_FLOW_ERROR_TYPE_ITEM, item,
1780                                           "Match is supported for GTP"
1781                                           " flags only");
1782         return mlx5_flow_item_acceptable
1783                 (item, (const uint8_t *)mask,
1784                  (const uint8_t *)&nic_mask,
1785                  sizeof(struct rte_flow_item_gtp),
1786                  error);
1787 }
1788
1789 /**
1790  * Validate the pop VLAN action.
1791  *
1792  * @param[in] dev
1793  *   Pointer to the rte_eth_dev structure.
1794  * @param[in] action_flags
1795  *   Holds the actions detected until now.
1796  * @param[in] action
1797  *   Pointer to the pop vlan action.
1798  * @param[in] item_flags
1799  *   The items found in this flow rule.
1800  * @param[in] attr
1801  *   Pointer to flow attributes.
1802  * @param[out] error
1803  *   Pointer to error structure.
1804  *
1805  * @return
1806  *   0 on success, a negative errno value otherwise and rte_errno is set.
1807  */
1808 static int
1809 flow_dv_validate_action_pop_vlan(struct rte_eth_dev *dev,
1810                                  uint64_t action_flags,
1811                                  const struct rte_flow_action *action,
1812                                  uint64_t item_flags,
1813                                  const struct rte_flow_attr *attr,
1814                                  struct rte_flow_error *error)
1815 {
1816         const struct mlx5_priv *priv = dev->data->dev_private;
1817
1818         (void)action;
1819         (void)attr;
1820         if (!priv->sh->pop_vlan_action)
1821                 return rte_flow_error_set(error, ENOTSUP,
1822                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1823                                           NULL,
1824                                           "pop vlan action is not supported");
1825         if (attr->egress)
1826                 return rte_flow_error_set(error, ENOTSUP,
1827                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
1828                                           NULL,
1829                                           "pop vlan action not supported for "
1830                                           "egress");
1831         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
1832                 return rte_flow_error_set(error, ENOTSUP,
1833                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1834                                           "no support for multiple VLAN "
1835                                           "actions");
1836         if (!(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
1837                 return rte_flow_error_set(error, ENOTSUP,
1838                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1839                                           NULL,
1840                                           "cannot pop vlan without a "
1841                                           "match on (outer) vlan in the flow");
1842         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1843                 return rte_flow_error_set(error, EINVAL,
1844                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1845                                           "wrong action order, port_id should "
1846                                           "be after pop VLAN action");
1847         if (!attr->transfer && priv->representor)
1848                 return rte_flow_error_set(error, ENOTSUP,
1849                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1850                                           "pop vlan action for VF representor "
1851                                           "not supported on NIC table");
1852         return 0;
1853 }
1854
1855 /**
1856  * Get VLAN default info from vlan match info.
1857  *
1858  * @param[in] items
1859  *   the list of item specifications.
1860  * @param[out] vlan
1861  *   pointer VLAN info to fill to.
1862  *
1863  * @return
1864  *   0 on success, a negative errno value otherwise and rte_errno is set.
1865  */
1866 static void
1867 flow_dev_get_vlan_info_from_items(const struct rte_flow_item *items,
1868                                   struct rte_vlan_hdr *vlan)
1869 {
1870         const struct rte_flow_item_vlan nic_mask = {
1871                 .tci = RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK |
1872                                 MLX5DV_FLOW_VLAN_VID_MASK),
1873                 .inner_type = RTE_BE16(0xffff),
1874         };
1875
1876         if (items == NULL)
1877                 return;
1878         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
1879                 int type = items->type;
1880
1881                 if (type == RTE_FLOW_ITEM_TYPE_VLAN ||
1882                     type == MLX5_RTE_FLOW_ITEM_TYPE_VLAN)
1883                         break;
1884         }
1885         if (items->type != RTE_FLOW_ITEM_TYPE_END) {
1886                 const struct rte_flow_item_vlan *vlan_m = items->mask;
1887                 const struct rte_flow_item_vlan *vlan_v = items->spec;
1888
1889                 /* If VLAN item in pattern doesn't contain data, return here. */
1890                 if (!vlan_v)
1891                         return;
1892                 if (!vlan_m)
1893                         vlan_m = &nic_mask;
1894                 /* Only full match values are accepted */
1895                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) ==
1896                      MLX5DV_FLOW_VLAN_PCP_MASK_BE) {
1897                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_PCP_MASK;
1898                         vlan->vlan_tci |=
1899                                 rte_be_to_cpu_16(vlan_v->tci &
1900                                                  MLX5DV_FLOW_VLAN_PCP_MASK_BE);
1901                 }
1902                 if ((vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) ==
1903                      MLX5DV_FLOW_VLAN_VID_MASK_BE) {
1904                         vlan->vlan_tci &= ~MLX5DV_FLOW_VLAN_VID_MASK;
1905                         vlan->vlan_tci |=
1906                                 rte_be_to_cpu_16(vlan_v->tci &
1907                                                  MLX5DV_FLOW_VLAN_VID_MASK_BE);
1908                 }
1909                 if (vlan_m->inner_type == nic_mask.inner_type)
1910                         vlan->eth_proto = rte_be_to_cpu_16(vlan_v->inner_type &
1911                                                            vlan_m->inner_type);
1912         }
1913 }
1914
1915 /**
1916  * Validate the push VLAN action.
1917  *
1918  * @param[in] dev
1919  *   Pointer to the rte_eth_dev structure.
1920  * @param[in] action_flags
1921  *   Holds the actions detected until now.
1922  * @param[in] item_flags
1923  *   The items found in this flow rule.
1924  * @param[in] action
1925  *   Pointer to the action structure.
1926  * @param[in] attr
1927  *   Pointer to flow attributes
1928  * @param[out] error
1929  *   Pointer to error structure.
1930  *
1931  * @return
1932  *   0 on success, a negative errno value otherwise and rte_errno is set.
1933  */
1934 static int
1935 flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
1936                                   uint64_t action_flags,
1937                                   const struct rte_flow_item_vlan *vlan_m,
1938                                   const struct rte_flow_action *action,
1939                                   const struct rte_flow_attr *attr,
1940                                   struct rte_flow_error *error)
1941 {
1942         const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
1943         const struct mlx5_priv *priv = dev->data->dev_private;
1944
1945         if (!attr->transfer && attr->ingress)
1946                 return rte_flow_error_set(error, ENOTSUP,
1947                                           RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
1948                                           NULL,
1949                                           "push VLAN action not supported for "
1950                                           "ingress");
1951         if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
1952             push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
1953                 return rte_flow_error_set(error, EINVAL,
1954                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1955                                           "invalid vlan ethertype");
1956         if (action_flags & MLX5_FLOW_VLAN_ACTIONS)
1957                 return rte_flow_error_set(error, ENOTSUP,
1958                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1959                                           "no support for multiple VLAN "
1960                                           "actions");
1961         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
1962                 return rte_flow_error_set(error, EINVAL,
1963                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1964                                           "wrong action order, port_id should "
1965                                           "be after push VLAN");
1966         if (!attr->transfer && priv->representor)
1967                 return rte_flow_error_set(error, ENOTSUP,
1968                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1969                                           "push vlan action for VF representor "
1970                                           "not supported on NIC table");
1971         if (vlan_m &&
1972             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) &&
1973             (vlan_m->tci & MLX5DV_FLOW_VLAN_PCP_MASK_BE) !=
1974                 MLX5DV_FLOW_VLAN_PCP_MASK_BE &&
1975             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP) &&
1976             !(mlx5_flow_find_action
1977                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)))
1978                 return rte_flow_error_set(error, EINVAL,
1979                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1980                                           "not full match mask on VLAN PCP and "
1981                                           "there is no of_set_vlan_pcp action, "
1982                                           "push VLAN action cannot figure out "
1983                                           "PCP value");
1984         if (vlan_m &&
1985             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) &&
1986             (vlan_m->tci & MLX5DV_FLOW_VLAN_VID_MASK_BE) !=
1987                 MLX5DV_FLOW_VLAN_VID_MASK_BE &&
1988             !(action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID) &&
1989             !(mlx5_flow_find_action
1990                 (action + 1, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)))
1991                 return rte_flow_error_set(error, EINVAL,
1992                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
1993                                           "not full match mask on VLAN VID and "
1994                                           "there is no of_set_vlan_vid action, "
1995                                           "push VLAN action cannot figure out "
1996                                           "VID value");
1997         (void)attr;
1998         return 0;
1999 }
2000
2001 /**
2002  * Validate the set VLAN PCP.
2003  *
2004  * @param[in] action_flags
2005  *   Holds the actions detected until now.
2006  * @param[in] actions
2007  *   Pointer to the list of actions remaining in the flow rule.
2008  * @param[out] error
2009  *   Pointer to error structure.
2010  *
2011  * @return
2012  *   0 on success, a negative errno value otherwise and rte_errno is set.
2013  */
2014 static int
2015 flow_dv_validate_action_set_vlan_pcp(uint64_t action_flags,
2016                                      const struct rte_flow_action actions[],
2017                                      struct rte_flow_error *error)
2018 {
2019         const struct rte_flow_action *action = actions;
2020         const struct rte_flow_action_of_set_vlan_pcp *conf = action->conf;
2021
2022         if (conf->vlan_pcp > 7)
2023                 return rte_flow_error_set(error, EINVAL,
2024                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2025                                           "VLAN PCP value is too big");
2026         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN))
2027                 return rte_flow_error_set(error, ENOTSUP,
2028                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2029                                           "set VLAN PCP action must follow "
2030                                           "the push VLAN action");
2031         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_PCP)
2032                 return rte_flow_error_set(error, ENOTSUP,
2033                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2034                                           "Multiple VLAN PCP modification are "
2035                                           "not supported");
2036         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2037                 return rte_flow_error_set(error, EINVAL,
2038                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2039                                           "wrong action order, port_id should "
2040                                           "be after set VLAN PCP");
2041         return 0;
2042 }
2043
2044 /**
2045  * Validate the set VLAN VID.
2046  *
2047  * @param[in] item_flags
2048  *   Holds the items detected in this rule.
2049  * @param[in] action_flags
2050  *   Holds the actions detected until now.
2051  * @param[in] actions
2052  *   Pointer to the list of actions remaining in the flow rule.
2053  * @param[out] error
2054  *   Pointer to error structure.
2055  *
2056  * @return
2057  *   0 on success, a negative errno value otherwise and rte_errno is set.
2058  */
2059 static int
2060 flow_dv_validate_action_set_vlan_vid(uint64_t item_flags,
2061                                      uint64_t action_flags,
2062                                      const struct rte_flow_action actions[],
2063                                      struct rte_flow_error *error)
2064 {
2065         const struct rte_flow_action *action = actions;
2066         const struct rte_flow_action_of_set_vlan_vid *conf = action->conf;
2067
2068         if (rte_be_to_cpu_16(conf->vlan_vid) > 0xFFE)
2069                 return rte_flow_error_set(error, EINVAL,
2070                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2071                                           "VLAN VID value is too big");
2072         if (!(action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) &&
2073             !(item_flags & MLX5_FLOW_LAYER_OUTER_VLAN))
2074                 return rte_flow_error_set(error, ENOTSUP,
2075                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2076                                           "set VLAN VID action must follow push"
2077                                           " VLAN action or match on VLAN item");
2078         if (action_flags & MLX5_FLOW_ACTION_OF_SET_VLAN_VID)
2079                 return rte_flow_error_set(error, ENOTSUP,
2080                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2081                                           "Multiple VLAN VID modifications are "
2082                                           "not supported");
2083         if (action_flags & MLX5_FLOW_ACTION_PORT_ID)
2084                 return rte_flow_error_set(error, EINVAL,
2085                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2086                                           "wrong action order, port_id should "
2087                                           "be after set VLAN VID");
2088         return 0;
2089 }
2090
2091 /*
2092  * Validate the FLAG action.
2093  *
2094  * @param[in] dev
2095  *   Pointer to the rte_eth_dev structure.
2096  * @param[in] action_flags
2097  *   Holds the actions detected until now.
2098  * @param[in] attr
2099  *   Pointer to flow attributes
2100  * @param[out] error
2101  *   Pointer to error structure.
2102  *
2103  * @return
2104  *   0 on success, a negative errno value otherwise and rte_errno is set.
2105  */
2106 static int
2107 flow_dv_validate_action_flag(struct rte_eth_dev *dev,
2108                              uint64_t action_flags,
2109                              const struct rte_flow_attr *attr,
2110                              struct rte_flow_error *error)
2111 {
2112         struct mlx5_priv *priv = dev->data->dev_private;
2113         struct mlx5_dev_config *config = &priv->config;
2114         int ret;
2115
2116         /* Fall back if no extended metadata register support. */
2117         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2118                 return mlx5_flow_validate_action_flag(action_flags, attr,
2119                                                       error);
2120         /* Extensive metadata mode requires registers. */
2121         if (!mlx5_flow_ext_mreg_supported(dev))
2122                 return rte_flow_error_set(error, ENOTSUP,
2123                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2124                                           "no metadata registers "
2125                                           "to support flag action");
2126         if (!(priv->sh->dv_mark_mask & MLX5_FLOW_MARK_DEFAULT))
2127                 return rte_flow_error_set(error, ENOTSUP,
2128                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2129                                           "extended metadata register"
2130                                           " isn't available");
2131         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2132         if (ret < 0)
2133                 return ret;
2134         MLX5_ASSERT(ret > 0);
2135         if (action_flags & MLX5_FLOW_ACTION_MARK)
2136                 return rte_flow_error_set(error, EINVAL,
2137                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2138                                           "can't mark and flag in same flow");
2139         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2140                 return rte_flow_error_set(error, EINVAL,
2141                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2142                                           "can't have 2 flag"
2143                                           " actions in same flow");
2144         return 0;
2145 }
2146
2147 /**
2148  * Validate MARK action.
2149  *
2150  * @param[in] dev
2151  *   Pointer to the rte_eth_dev structure.
2152  * @param[in] action
2153  *   Pointer to action.
2154  * @param[in] action_flags
2155  *   Holds the actions detected until now.
2156  * @param[in] attr
2157  *   Pointer to flow attributes
2158  * @param[out] error
2159  *   Pointer to error structure.
2160  *
2161  * @return
2162  *   0 on success, a negative errno value otherwise and rte_errno is set.
2163  */
2164 static int
2165 flow_dv_validate_action_mark(struct rte_eth_dev *dev,
2166                              const struct rte_flow_action *action,
2167                              uint64_t action_flags,
2168                              const struct rte_flow_attr *attr,
2169                              struct rte_flow_error *error)
2170 {
2171         struct mlx5_priv *priv = dev->data->dev_private;
2172         struct mlx5_dev_config *config = &priv->config;
2173         const struct rte_flow_action_mark *mark = action->conf;
2174         int ret;
2175
2176         /* Fall back if no extended metadata register support. */
2177         if (config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY)
2178                 return mlx5_flow_validate_action_mark(action, action_flags,
2179                                                       attr, error);
2180         /* Extensive metadata mode requires registers. */
2181         if (!mlx5_flow_ext_mreg_supported(dev))
2182                 return rte_flow_error_set(error, ENOTSUP,
2183                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2184                                           "no metadata registers "
2185                                           "to support mark action");
2186         if (!priv->sh->dv_mark_mask)
2187                 return rte_flow_error_set(error, ENOTSUP,
2188                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2189                                           "extended metadata register"
2190                                           " isn't available");
2191         ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
2192         if (ret < 0)
2193                 return ret;
2194         MLX5_ASSERT(ret > 0);
2195         if (!mark)
2196                 return rte_flow_error_set(error, EINVAL,
2197                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2198                                           "configuration cannot be null");
2199         if (mark->id >= (MLX5_FLOW_MARK_MAX & priv->sh->dv_mark_mask))
2200                 return rte_flow_error_set(error, EINVAL,
2201                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
2202                                           &mark->id,
2203                                           "mark id exceeds the limit");
2204         if (action_flags & MLX5_FLOW_ACTION_FLAG)
2205                 return rte_flow_error_set(error, EINVAL,
2206                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2207                                           "can't flag and mark in same flow");
2208         if (action_flags & MLX5_FLOW_ACTION_MARK)
2209                 return rte_flow_error_set(error, EINVAL,
2210                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2211                                           "can't have 2 mark actions in same"
2212                                           " flow");
2213         return 0;
2214 }
2215
2216 /**
2217  * Validate SET_META action.
2218  *
2219  * @param[in] dev
2220  *   Pointer to the rte_eth_dev structure.
2221  * @param[in] action
2222  *   Pointer to the action structure.
2223  * @param[in] action_flags
2224  *   Holds the actions detected until now.
2225  * @param[in] attr
2226  *   Pointer to flow attributes
2227  * @param[out] error
2228  *   Pointer to error structure.
2229  *
2230  * @return
2231  *   0 on success, a negative errno value otherwise and rte_errno is set.
2232  */
2233 static int
2234 flow_dv_validate_action_set_meta(struct rte_eth_dev *dev,
2235                                  const struct rte_flow_action *action,
2236                                  uint64_t action_flags __rte_unused,
2237                                  const struct rte_flow_attr *attr,
2238                                  struct rte_flow_error *error)
2239 {
2240         const struct rte_flow_action_set_meta *conf;
2241         uint32_t nic_mask = UINT32_MAX;
2242         int reg;
2243
2244         if (!mlx5_flow_ext_mreg_supported(dev))
2245                 return rte_flow_error_set(error, ENOTSUP,
2246                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2247                                           "extended metadata register"
2248                                           " isn't supported");
2249         reg = flow_dv_get_metadata_reg(dev, attr, error);
2250         if (reg < 0)
2251                 return reg;
2252         if (reg != REG_A && reg != REG_B) {
2253                 struct mlx5_priv *priv = dev->data->dev_private;
2254
2255                 nic_mask = priv->sh->dv_meta_mask;
2256         }
2257         if (!(action->conf))
2258                 return rte_flow_error_set(error, EINVAL,
2259                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2260                                           "configuration cannot be null");
2261         conf = (const struct rte_flow_action_set_meta *)action->conf;
2262         if (!conf->mask)
2263                 return rte_flow_error_set(error, EINVAL,
2264                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2265                                           "zero mask doesn't have any effect");
2266         if (conf->mask & ~nic_mask)
2267                 return rte_flow_error_set(error, EINVAL,
2268                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2269                                           "meta data must be within reg C0");
2270         return 0;
2271 }
2272
2273 /**
2274  * Validate SET_TAG action.
2275  *
2276  * @param[in] dev
2277  *   Pointer to the rte_eth_dev structure.
2278  * @param[in] action
2279  *   Pointer to the action structure.
2280  * @param[in] action_flags
2281  *   Holds the actions detected until now.
2282  * @param[in] attr
2283  *   Pointer to flow attributes
2284  * @param[out] error
2285  *   Pointer to error structure.
2286  *
2287  * @return
2288  *   0 on success, a negative errno value otherwise and rte_errno is set.
2289  */
2290 static int
2291 flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
2292                                 const struct rte_flow_action *action,
2293                                 uint64_t action_flags,
2294                                 const struct rte_flow_attr *attr,
2295                                 struct rte_flow_error *error)
2296 {
2297         const struct rte_flow_action_set_tag *conf;
2298         const uint64_t terminal_action_flags =
2299                 MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_QUEUE |
2300                 MLX5_FLOW_ACTION_RSS;
2301         int ret;
2302
2303         if (!mlx5_flow_ext_mreg_supported(dev))
2304                 return rte_flow_error_set(error, ENOTSUP,
2305                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2306                                           "extensive metadata register"
2307                                           " isn't supported");
2308         if (!(action->conf))
2309                 return rte_flow_error_set(error, EINVAL,
2310                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2311                                           "configuration cannot be null");
2312         conf = (const struct rte_flow_action_set_tag *)action->conf;
2313         if (!conf->mask)
2314                 return rte_flow_error_set(error, EINVAL,
2315                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2316                                           "zero mask doesn't have any effect");
2317         ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, conf->index, error);
2318         if (ret < 0)
2319                 return ret;
2320         if (!attr->transfer && attr->ingress &&
2321             (action_flags & terminal_action_flags))
2322                 return rte_flow_error_set(error, EINVAL,
2323                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2324                                           "set_tag has no effect"
2325                                           " with terminal actions");
2326         return 0;
2327 }
2328
2329 /**
2330  * Validate count action.
2331  *
2332  * @param[in] dev
2333  *   Pointer to rte_eth_dev structure.
2334  * @param[out] error
2335  *   Pointer to error structure.
2336  *
2337  * @return
2338  *   0 on success, a negative errno value otherwise and rte_errno is set.
2339  */
2340 static int
2341 flow_dv_validate_action_count(struct rte_eth_dev *dev,
2342                               struct rte_flow_error *error)
2343 {
2344         struct mlx5_priv *priv = dev->data->dev_private;
2345
2346         if (!priv->config.devx)
2347                 goto notsup_err;
2348 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
2349         return 0;
2350 #endif
2351 notsup_err:
2352         return rte_flow_error_set
2353                       (error, ENOTSUP,
2354                        RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2355                        NULL,
2356                        "count action not supported");
2357 }
2358
2359 /**
2360  * Validate the L2 encap action.
2361  *
2362  * @param[in] dev
2363  *   Pointer to the rte_eth_dev structure.
2364  * @param[in] action_flags
2365  *   Holds the actions detected until now.
2366  * @param[in] action
2367  *   Pointer to the action structure.
2368  * @param[in] attr
2369  *   Pointer to flow attributes.
2370  * @param[out] error
2371  *   Pointer to error structure.
2372  *
2373  * @return
2374  *   0 on success, a negative errno value otherwise and rte_errno is set.
2375  */
2376 static int
2377 flow_dv_validate_action_l2_encap(struct rte_eth_dev *dev,
2378                                  uint64_t action_flags,
2379                                  const struct rte_flow_action *action,
2380                                  const struct rte_flow_attr *attr,
2381                                  struct rte_flow_error *error)
2382 {
2383         const struct mlx5_priv *priv = dev->data->dev_private;
2384
2385         if (!(action->conf))
2386                 return rte_flow_error_set(error, EINVAL,
2387                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
2388                                           "configuration cannot be null");
2389         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
2390                 return rte_flow_error_set(error, EINVAL,
2391                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2392                                           "can only have a single encap action "
2393                                           "in a flow");
2394         if (!attr->transfer && priv->representor)
2395                 return rte_flow_error_set(error, ENOTSUP,
2396                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2397                                           "encap action for VF representor "
2398                                           "not supported on NIC table");
2399         return 0;
2400 }
2401
2402 /**
2403  * Validate a decap action.
2404  *
2405  * @param[in] dev
2406  *   Pointer to the rte_eth_dev structure.
2407  * @param[in] action_flags
2408  *   Holds the actions detected until now.
2409  * @param[in] attr
2410  *   Pointer to flow attributes
2411  * @param[out] error
2412  *   Pointer to error structure.
2413  *
2414  * @return
2415  *   0 on success, a negative errno value otherwise and rte_errno is set.
2416  */
2417 static int
2418 flow_dv_validate_action_decap(struct rte_eth_dev *dev,
2419                               uint64_t action_flags,
2420                               const struct rte_flow_attr *attr,
2421                               struct rte_flow_error *error)
2422 {
2423         const struct mlx5_priv *priv = dev->data->dev_private;
2424
2425         if (action_flags & MLX5_FLOW_XCAP_ACTIONS)
2426                 return rte_flow_error_set(error, ENOTSUP,
2427                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2428                                           action_flags &
2429                                           MLX5_FLOW_ACTION_DECAP ? "can only "
2430                                           "have a single decap action" : "decap "
2431                                           "after encap is not supported");
2432         if (action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS)
2433                 return rte_flow_error_set(error, EINVAL,
2434                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2435                                           "can't have decap action after"
2436                                           " modify action");
2437         if (attr->egress)
2438                 return rte_flow_error_set(error, ENOTSUP,
2439                                           RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
2440                                           NULL,
2441                                           "decap action not supported for "
2442                                           "egress");
2443         if (!attr->transfer && priv->representor)
2444                 return rte_flow_error_set(error, ENOTSUP,
2445                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2446                                           "decap action for VF representor "
2447                                           "not supported on NIC table");
2448         return 0;
2449 }
2450
2451 const struct rte_flow_action_raw_decap empty_decap = {.data = NULL, .size = 0,};
2452
2453 /**
2454  * Validate the raw encap and decap actions.
2455  *
2456  * @param[in] dev
2457  *   Pointer to the rte_eth_dev structure.
2458  * @param[in] decap
2459  *   Pointer to the decap action.
2460  * @param[in] encap
2461  *   Pointer to the encap action.
2462  * @param[in] attr
2463  *   Pointer to flow attributes
2464  * @param[in/out] action_flags
2465  *   Holds the actions detected until now.
2466  * @param[out] actions_n
2467  *   pointer to the number of actions counter.
2468  * @param[out] error
2469  *   Pointer to error structure.
2470  *
2471  * @return
2472  *   0 on success, a negative errno value otherwise and rte_errno is set.
2473  */
2474 static int
2475 flow_dv_validate_action_raw_encap_decap
2476         (struct rte_eth_dev *dev,
2477          const struct rte_flow_action_raw_decap *decap,
2478          const struct rte_flow_action_raw_encap *encap,
2479          const struct rte_flow_attr *attr, uint64_t *action_flags,
2480          int *actions_n, struct rte_flow_error *error)
2481 {
2482         const struct mlx5_priv *priv = dev->data->dev_private;
2483         int ret;
2484
2485         if (encap && (!encap->size || !encap->data))
2486                 return rte_flow_error_set(error, EINVAL,
2487                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
2488                                           "raw encap data cannot be empty");
2489         if (decap && encap) {
2490                 if (decap->size <= MLX5_ENCAPSULATION_DECISION_SIZE &&
2491                     encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
2492                         /* L3 encap. */
2493                         decap = NULL;
2494                 else if (encap->size <=
2495                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2496                            decap->size >
2497                            MLX5_ENCAPSULATION_DECISION_SIZE)
2498                         /* L3 decap. */
2499                         encap = NULL;
2500                 else if (encap->size >
2501                            MLX5_ENCAPSULATION_DECISION_SIZE &&
2502                            decap->size >
2503                            MLX5_ENCAPSULATION_DECISION_SIZE)
2504                         /* 2 L2 actions: encap and decap. */
2505                         ;
2506                 else
2507                         return rte_flow_error_set(error,
2508                                 ENOTSUP,
2509                                 RTE_FLOW_ERROR_TYPE_ACTION,
2510                                 NULL, "unsupported too small "
2511                                 "raw decap and too small raw "
2512                                 "encap combination");
2513         }
2514         if (decap) {
2515                 ret = flow_dv_validate_action_decap(dev, *action_flags, attr,
2516                                                     error);
2517                 if (ret < 0)
2518                         return ret;
2519                 *action_flags |= MLX5_FLOW_ACTION_DECAP;
2520                 ++(*actions_n);
2521         }
2522         if (encap) {
2523                 if (encap->size <= MLX5_ENCAPSULATION_DECISION_SIZE)
2524                         return rte_flow_error_set(error, ENOTSUP,
2525                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2526                                                   NULL,
2527                                                   "small raw encap size");
2528                 if (*action_flags & MLX5_FLOW_ACTION_ENCAP)
2529                         return rte_flow_error_set(error, EINVAL,
2530                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2531                                                   NULL,
2532                                                   "more than one encap action");
2533                 if (!attr->transfer && priv->representor)
2534                         return rte_flow_error_set
2535                                         (error, ENOTSUP,
2536                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2537                                          "encap action for VF representor "
2538                                          "not supported on NIC table");
2539                 *action_flags |= MLX5_FLOW_ACTION_ENCAP;
2540                 ++(*actions_n);
2541         }
2542         return 0;
2543 }
2544
2545 /**
2546  * Find existing encap/decap resource or create and register a new one.
2547  *
2548  * @param[in, out] dev
2549  *   Pointer to rte_eth_dev structure.
2550  * @param[in, out] resource
2551  *   Pointer to encap/decap resource.
2552  * @parm[in, out] dev_flow
2553  *   Pointer to the dev_flow.
2554  * @param[out] error
2555  *   pointer to error structure.
2556  *
2557  * @return
2558  *   0 on success otherwise -errno and errno is set.
2559  */
2560 static int
2561 flow_dv_encap_decap_resource_register
2562                         (struct rte_eth_dev *dev,
2563                          struct mlx5_flow_dv_encap_decap_resource *resource,
2564                          struct mlx5_flow *dev_flow,
2565                          struct rte_flow_error *error)
2566 {
2567         struct mlx5_priv *priv = dev->data->dev_private;
2568         struct mlx5_dev_ctx_shared *sh = priv->sh;
2569         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
2570         struct mlx5dv_dr_domain *domain;
2571         uint32_t idx = 0;
2572
2573         resource->flags = dev_flow->dv.group ? 0 : 1;
2574         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2575                 domain = sh->fdb_domain;
2576         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2577                 domain = sh->rx_domain;
2578         else
2579                 domain = sh->tx_domain;
2580         /* Lookup a matching resource from cache. */
2581         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], sh->encaps_decaps, idx,
2582                       cache_resource, next) {
2583                 if (resource->reformat_type == cache_resource->reformat_type &&
2584                     resource->ft_type == cache_resource->ft_type &&
2585                     resource->flags == cache_resource->flags &&
2586                     resource->size == cache_resource->size &&
2587                     !memcmp((const void *)resource->buf,
2588                             (const void *)cache_resource->buf,
2589                             resource->size)) {
2590                         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
2591                                 (void *)cache_resource,
2592                                 rte_atomic32_read(&cache_resource->refcnt));
2593                         rte_atomic32_inc(&cache_resource->refcnt);
2594                         dev_flow->handle->dvh.rix_encap_decap = idx;
2595                         dev_flow->dv.encap_decap = cache_resource;
2596                         return 0;
2597                 }
2598         }
2599         /* Register new encap/decap resource. */
2600         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
2601                                        &dev_flow->handle->dvh.rix_encap_decap);
2602         if (!cache_resource)
2603                 return rte_flow_error_set(error, ENOMEM,
2604                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2605                                           "cannot allocate resource memory");
2606         *cache_resource = *resource;
2607         cache_resource->action =
2608                 mlx5_glue->dv_create_flow_action_packet_reformat
2609                         (sh->ctx, cache_resource->reformat_type,
2610                          cache_resource->ft_type, domain, cache_resource->flags,
2611                          cache_resource->size,
2612                          (cache_resource->size ? cache_resource->buf : NULL));
2613         if (!cache_resource->action) {
2614                 rte_free(cache_resource);
2615                 return rte_flow_error_set(error, ENOMEM,
2616                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2617                                           NULL, "cannot create action");
2618         }
2619         rte_atomic32_init(&cache_resource->refcnt);
2620         rte_atomic32_inc(&cache_resource->refcnt);
2621         ILIST_INSERT(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &sh->encaps_decaps,
2622                      dev_flow->handle->dvh.rix_encap_decap, cache_resource,
2623                      next);
2624         dev_flow->dv.encap_decap = cache_resource;
2625         DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
2626                 (void *)cache_resource,
2627                 rte_atomic32_read(&cache_resource->refcnt));
2628         return 0;
2629 }
2630
2631 /**
2632  * Find existing table jump resource or create and register a new one.
2633  *
2634  * @param[in, out] dev
2635  *   Pointer to rte_eth_dev structure.
2636  * @param[in, out] tbl
2637  *   Pointer to flow table resource.
2638  * @parm[in, out] dev_flow
2639  *   Pointer to the dev_flow.
2640  * @param[out] error
2641  *   pointer to error structure.
2642  *
2643  * @return
2644  *   0 on success otherwise -errno and errno is set.
2645  */
2646 static int
2647 flow_dv_jump_tbl_resource_register
2648                         (struct rte_eth_dev *dev __rte_unused,
2649                          struct mlx5_flow_tbl_resource *tbl,
2650                          struct mlx5_flow *dev_flow,
2651                          struct rte_flow_error *error)
2652 {
2653         struct mlx5_flow_tbl_data_entry *tbl_data =
2654                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
2655         int cnt;
2656
2657         MLX5_ASSERT(tbl);
2658         cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
2659         if (!cnt) {
2660                 tbl_data->jump.action =
2661                         mlx5_glue->dr_create_flow_action_dest_flow_tbl
2662                         (tbl->obj);
2663                 if (!tbl_data->jump.action)
2664                         return rte_flow_error_set(error, ENOMEM,
2665                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2666                                         NULL, "cannot create jump action");
2667                 DRV_LOG(DEBUG, "new jump table resource %p: refcnt %d++",
2668                         (void *)&tbl_data->jump, cnt);
2669         } else {
2670                 /* old jump should not make the table ref++. */
2671                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
2672                 MLX5_ASSERT(tbl_data->jump.action);
2673                 DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
2674                         (void *)&tbl_data->jump, cnt);
2675         }
2676         rte_atomic32_inc(&tbl_data->jump.refcnt);
2677         dev_flow->handle->rix_jump = tbl_data->idx;
2678         dev_flow->dv.jump = &tbl_data->jump;
2679         return 0;
2680 }
2681
2682 /**
2683  * Find existing default miss resource or create and register a new one.
2684  *
2685  * @param[in, out] dev
2686  *   Pointer to rte_eth_dev structure.
2687  * @param[out] error
2688  *   pointer to error structure.
2689  *
2690  * @return
2691  *   0 on success otherwise -errno and errno is set.
2692  */
2693 static int
2694 flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
2695                 struct rte_flow_error *error)
2696 {
2697         struct mlx5_priv *priv = dev->data->dev_private;
2698         struct mlx5_dev_ctx_shared *sh = priv->sh;
2699         struct mlx5_flow_default_miss_resource *cache_resource =
2700                         &sh->default_miss;
2701         int cnt = rte_atomic32_read(&cache_resource->refcnt);
2702
2703         if (!cnt) {
2704                 MLX5_ASSERT(cache_resource->action);
2705                 cache_resource->action =
2706                 mlx5_glue->dr_create_flow_action_default_miss();
2707                 if (!cache_resource->action)
2708                         return rte_flow_error_set(error, ENOMEM,
2709                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2710                                         "cannot create default miss action");
2711                 DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++",
2712                                 (void *)cache_resource->action, cnt);
2713         }
2714         rte_atomic32_inc(&cache_resource->refcnt);
2715         return 0;
2716 }
2717
2718 /**
2719  * Find existing table port ID resource or create and register a new one.
2720  *
2721  * @param[in, out] dev
2722  *   Pointer to rte_eth_dev structure.
2723  * @param[in, out] resource
2724  *   Pointer to port ID action resource.
2725  * @parm[in, out] dev_flow
2726  *   Pointer to the dev_flow.
2727  * @param[out] error
2728  *   pointer to error structure.
2729  *
2730  * @return
2731  *   0 on success otherwise -errno and errno is set.
2732  */
2733 static int
2734 flow_dv_port_id_action_resource_register
2735                         (struct rte_eth_dev *dev,
2736                          struct mlx5_flow_dv_port_id_action_resource *resource,
2737                          struct mlx5_flow *dev_flow,
2738                          struct rte_flow_error *error)
2739 {
2740         struct mlx5_priv *priv = dev->data->dev_private;
2741         struct mlx5_dev_ctx_shared *sh = priv->sh;
2742         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
2743         uint32_t idx = 0;
2744
2745         /* Lookup a matching resource from cache. */
2746         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PORT_ID], sh->port_id_action_list,
2747                       idx, cache_resource, next) {
2748                 if (resource->port_id == cache_resource->port_id) {
2749                         DRV_LOG(DEBUG, "port id action resource resource %p: "
2750                                 "refcnt %d++",
2751                                 (void *)cache_resource,
2752                                 rte_atomic32_read(&cache_resource->refcnt));
2753                         rte_atomic32_inc(&cache_resource->refcnt);
2754                         dev_flow->handle->rix_port_id_action = idx;
2755                         dev_flow->dv.port_id_action = cache_resource;
2756                         return 0;
2757                 }
2758         }
2759         /* Register new port id action resource. */
2760         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID],
2761                                        &dev_flow->handle->rix_port_id_action);
2762         if (!cache_resource)
2763                 return rte_flow_error_set(error, ENOMEM,
2764                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2765                                           "cannot allocate resource memory");
2766         *cache_resource = *resource;
2767         /*
2768          * Depending on rdma_core version the glue routine calls
2769          * either mlx5dv_dr_action_create_dest_ib_port(domain, dev_port)
2770          * or mlx5dv_dr_action_create_dest_vport(domain, vport_id).
2771          */
2772         cache_resource->action =
2773                 mlx5_glue->dr_create_flow_action_dest_port
2774                         (priv->sh->fdb_domain, resource->port_id);
2775         if (!cache_resource->action) {
2776                 rte_free(cache_resource);
2777                 return rte_flow_error_set(error, ENOMEM,
2778                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2779                                           NULL, "cannot create action");
2780         }
2781         rte_atomic32_init(&cache_resource->refcnt);
2782         rte_atomic32_inc(&cache_resource->refcnt);
2783         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
2784                      dev_flow->handle->rix_port_id_action, cache_resource,
2785                      next);
2786         dev_flow->dv.port_id_action = cache_resource;
2787         DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
2788                 (void *)cache_resource,
2789                 rte_atomic32_read(&cache_resource->refcnt));
2790         return 0;
2791 }
2792
2793 /**
2794  * Find existing push vlan resource or create and register a new one.
2795  *
2796  * @param [in, out] dev
2797  *   Pointer to rte_eth_dev structure.
2798  * @param[in, out] resource
2799  *   Pointer to port ID action resource.
2800  * @parm[in, out] dev_flow
2801  *   Pointer to the dev_flow.
2802  * @param[out] error
2803  *   pointer to error structure.
2804  *
2805  * @return
2806  *   0 on success otherwise -errno and errno is set.
2807  */
2808 static int
2809 flow_dv_push_vlan_action_resource_register
2810                        (struct rte_eth_dev *dev,
2811                         struct mlx5_flow_dv_push_vlan_action_resource *resource,
2812                         struct mlx5_flow *dev_flow,
2813                         struct rte_flow_error *error)
2814 {
2815         struct mlx5_priv *priv = dev->data->dev_private;
2816         struct mlx5_dev_ctx_shared *sh = priv->sh;
2817         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
2818         struct mlx5dv_dr_domain *domain;
2819         uint32_t idx = 0;
2820
2821         /* Lookup a matching resource from cache. */
2822         ILIST_FOREACH(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2823                       sh->push_vlan_action_list, idx, cache_resource, next) {
2824                 if (resource->vlan_tag == cache_resource->vlan_tag &&
2825                     resource->ft_type == cache_resource->ft_type) {
2826                         DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
2827                                 "refcnt %d++",
2828                                 (void *)cache_resource,
2829                                 rte_atomic32_read(&cache_resource->refcnt));
2830                         rte_atomic32_inc(&cache_resource->refcnt);
2831                         dev_flow->handle->dvh.rix_push_vlan = idx;
2832                         dev_flow->dv.push_vlan_res = cache_resource;
2833                         return 0;
2834                 }
2835         }
2836         /* Register new push_vlan action resource. */
2837         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2838                                        &dev_flow->handle->dvh.rix_push_vlan);
2839         if (!cache_resource)
2840                 return rte_flow_error_set(error, ENOMEM,
2841                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2842                                           "cannot allocate resource memory");
2843         *cache_resource = *resource;
2844         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
2845                 domain = sh->fdb_domain;
2846         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
2847                 domain = sh->rx_domain;
2848         else
2849                 domain = sh->tx_domain;
2850         cache_resource->action =
2851                 mlx5_glue->dr_create_flow_action_push_vlan(domain,
2852                                                            resource->vlan_tag);
2853         if (!cache_resource->action) {
2854                 rte_free(cache_resource);
2855                 return rte_flow_error_set(error, ENOMEM,
2856                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
2857                                           NULL, "cannot create action");
2858         }
2859         rte_atomic32_init(&cache_resource->refcnt);
2860         rte_atomic32_inc(&cache_resource->refcnt);
2861         ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
2862                      &sh->push_vlan_action_list,
2863                      dev_flow->handle->dvh.rix_push_vlan,
2864                      cache_resource, next);
2865         dev_flow->dv.push_vlan_res = cache_resource;
2866         DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
2867                 (void *)cache_resource,
2868                 rte_atomic32_read(&cache_resource->refcnt));
2869         return 0;
2870 }
2871 /**
2872  * Get the size of specific rte_flow_item_type
2873  *
2874  * @param[in] item_type
2875  *   Tested rte_flow_item_type.
2876  *
2877  * @return
2878  *   sizeof struct item_type, 0 if void or irrelevant.
2879  */
2880 static size_t
2881 flow_dv_get_item_len(const enum rte_flow_item_type item_type)
2882 {
2883         size_t retval;
2884
2885         switch (item_type) {
2886         case RTE_FLOW_ITEM_TYPE_ETH:
2887                 retval = sizeof(struct rte_flow_item_eth);
2888                 break;
2889         case RTE_FLOW_ITEM_TYPE_VLAN:
2890                 retval = sizeof(struct rte_flow_item_vlan);
2891                 break;
2892         case RTE_FLOW_ITEM_TYPE_IPV4:
2893                 retval = sizeof(struct rte_flow_item_ipv4);
2894                 break;
2895         case RTE_FLOW_ITEM_TYPE_IPV6:
2896                 retval = sizeof(struct rte_flow_item_ipv6);
2897                 break;
2898         case RTE_FLOW_ITEM_TYPE_UDP:
2899                 retval = sizeof(struct rte_flow_item_udp);
2900                 break;
2901         case RTE_FLOW_ITEM_TYPE_TCP:
2902                 retval = sizeof(struct rte_flow_item_tcp);
2903                 break;
2904         case RTE_FLOW_ITEM_TYPE_VXLAN:
2905                 retval = sizeof(struct rte_flow_item_vxlan);
2906                 break;
2907         case RTE_FLOW_ITEM_TYPE_GRE:
2908                 retval = sizeof(struct rte_flow_item_gre);
2909                 break;
2910         case RTE_FLOW_ITEM_TYPE_NVGRE:
2911                 retval = sizeof(struct rte_flow_item_nvgre);
2912                 break;
2913         case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
2914                 retval = sizeof(struct rte_flow_item_vxlan_gpe);
2915                 break;
2916         case RTE_FLOW_ITEM_TYPE_MPLS:
2917                 retval = sizeof(struct rte_flow_item_mpls);
2918                 break;
2919         case RTE_FLOW_ITEM_TYPE_VOID: /* Fall through. */
2920         default:
2921                 retval = 0;
2922                 break;
2923         }
2924         return retval;
2925 }
2926
2927 #define MLX5_ENCAP_IPV4_VERSION         0x40
2928 #define MLX5_ENCAP_IPV4_IHL_MIN         0x05
2929 #define MLX5_ENCAP_IPV4_TTL_DEF         0x40
2930 #define MLX5_ENCAP_IPV6_VTC_FLOW        0x60000000
2931 #define MLX5_ENCAP_IPV6_HOP_LIMIT       0xff
2932 #define MLX5_ENCAP_VXLAN_FLAGS          0x08000000
2933 #define MLX5_ENCAP_VXLAN_GPE_FLAGS      0x04
2934
2935 /**
2936  * Convert the encap action data from list of rte_flow_item to raw buffer
2937  *
2938  * @param[in] items
2939  *   Pointer to rte_flow_item objects list.
2940  * @param[out] buf
2941  *   Pointer to the output buffer.
2942  * @param[out] size
2943  *   Pointer to the output buffer size.
2944  * @param[out] error
2945  *   Pointer to the error structure.
2946  *
2947  * @return
2948  *   0 on success, a negative errno value otherwise and rte_errno is set.
2949  */
2950 static int
2951 flow_dv_convert_encap_data(const struct rte_flow_item *items, uint8_t *buf,
2952                            size_t *size, struct rte_flow_error *error)
2953 {
2954         struct rte_ether_hdr *eth = NULL;
2955         struct rte_vlan_hdr *vlan = NULL;
2956         struct rte_ipv4_hdr *ipv4 = NULL;
2957         struct rte_ipv6_hdr *ipv6 = NULL;
2958         struct rte_udp_hdr *udp = NULL;
2959         struct rte_vxlan_hdr *vxlan = NULL;
2960         struct rte_vxlan_gpe_hdr *vxlan_gpe = NULL;
2961         struct rte_gre_hdr *gre = NULL;
2962         size_t len;
2963         size_t temp_size = 0;
2964
2965         if (!items)
2966                 return rte_flow_error_set(error, EINVAL,
2967                                           RTE_FLOW_ERROR_TYPE_ACTION,
2968                                           NULL, "invalid empty data");
2969         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
2970                 len = flow_dv_get_item_len(items->type);
2971                 if (len + temp_size > MLX5_ENCAP_MAX_LEN)
2972                         return rte_flow_error_set(error, EINVAL,
2973                                                   RTE_FLOW_ERROR_TYPE_ACTION,
2974                                                   (void *)items->type,
2975                                                   "items total size is too big"
2976                                                   " for encap action");
2977                 rte_memcpy((void *)&buf[temp_size], items->spec, len);
2978                 switch (items->type) {
2979                 case RTE_FLOW_ITEM_TYPE_ETH:
2980                         eth = (struct rte_ether_hdr *)&buf[temp_size];
2981                         break;
2982                 case RTE_FLOW_ITEM_TYPE_VLAN:
2983                         vlan = (struct rte_vlan_hdr *)&buf[temp_size];
2984                         if (!eth)
2985                                 return rte_flow_error_set(error, EINVAL,
2986                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2987                                                 (void *)items->type,
2988                                                 "eth header not found");
2989                         if (!eth->ether_type)
2990                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_VLAN);
2991                         break;
2992                 case RTE_FLOW_ITEM_TYPE_IPV4:
2993                         ipv4 = (struct rte_ipv4_hdr *)&buf[temp_size];
2994                         if (!vlan && !eth)
2995                                 return rte_flow_error_set(error, EINVAL,
2996                                                 RTE_FLOW_ERROR_TYPE_ACTION,
2997                                                 (void *)items->type,
2998                                                 "neither eth nor vlan"
2999                                                 " header found");
3000                         if (vlan && !vlan->eth_proto)
3001                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3002                         else if (eth && !eth->ether_type)
3003                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV4);
3004                         if (!ipv4->version_ihl)
3005                                 ipv4->version_ihl = MLX5_ENCAP_IPV4_VERSION |
3006                                                     MLX5_ENCAP_IPV4_IHL_MIN;
3007                         if (!ipv4->time_to_live)
3008                                 ipv4->time_to_live = MLX5_ENCAP_IPV4_TTL_DEF;
3009                         break;
3010                 case RTE_FLOW_ITEM_TYPE_IPV6:
3011                         ipv6 = (struct rte_ipv6_hdr *)&buf[temp_size];
3012                         if (!vlan && !eth)
3013                                 return rte_flow_error_set(error, EINVAL,
3014                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3015                                                 (void *)items->type,
3016                                                 "neither eth nor vlan"
3017                                                 " header found");
3018                         if (vlan && !vlan->eth_proto)
3019                                 vlan->eth_proto = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3020                         else if (eth && !eth->ether_type)
3021                                 eth->ether_type = RTE_BE16(RTE_ETHER_TYPE_IPV6);
3022                         if (!ipv6->vtc_flow)
3023                                 ipv6->vtc_flow =
3024                                         RTE_BE32(MLX5_ENCAP_IPV6_VTC_FLOW);
3025                         if (!ipv6->hop_limits)
3026                                 ipv6->hop_limits = MLX5_ENCAP_IPV6_HOP_LIMIT;
3027                         break;
3028                 case RTE_FLOW_ITEM_TYPE_UDP:
3029                         udp = (struct rte_udp_hdr *)&buf[temp_size];
3030                         if (!ipv4 && !ipv6)
3031                                 return rte_flow_error_set(error, EINVAL,
3032                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3033                                                 (void *)items->type,
3034                                                 "ip header not found");
3035                         if (ipv4 && !ipv4->next_proto_id)
3036                                 ipv4->next_proto_id = IPPROTO_UDP;
3037                         else if (ipv6 && !ipv6->proto)
3038                                 ipv6->proto = IPPROTO_UDP;
3039                         break;
3040                 case RTE_FLOW_ITEM_TYPE_VXLAN:
3041                         vxlan = (struct rte_vxlan_hdr *)&buf[temp_size];
3042                         if (!udp)
3043                                 return rte_flow_error_set(error, EINVAL,
3044                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3045                                                 (void *)items->type,
3046                                                 "udp header not found");
3047                         if (!udp->dst_port)
3048                                 udp->dst_port = RTE_BE16(MLX5_UDP_PORT_VXLAN);
3049                         if (!vxlan->vx_flags)
3050                                 vxlan->vx_flags =
3051                                         RTE_BE32(MLX5_ENCAP_VXLAN_FLAGS);
3052                         break;
3053                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
3054                         vxlan_gpe = (struct rte_vxlan_gpe_hdr *)&buf[temp_size];
3055                         if (!udp)
3056                                 return rte_flow_error_set(error, EINVAL,
3057                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3058                                                 (void *)items->type,
3059                                                 "udp header not found");
3060                         if (!vxlan_gpe->proto)
3061                                 return rte_flow_error_set(error, EINVAL,
3062                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3063                                                 (void *)items->type,
3064                                                 "next protocol not found");
3065                         if (!udp->dst_port)
3066                                 udp->dst_port =
3067                                         RTE_BE16(MLX5_UDP_PORT_VXLAN_GPE);
3068                         if (!vxlan_gpe->vx_flags)
3069                                 vxlan_gpe->vx_flags =
3070                                                 MLX5_ENCAP_VXLAN_GPE_FLAGS;
3071                         break;
3072                 case RTE_FLOW_ITEM_TYPE_GRE:
3073                 case RTE_FLOW_ITEM_TYPE_NVGRE:
3074                         gre = (struct rte_gre_hdr *)&buf[temp_size];
3075                         if (!gre->proto)
3076                                 return rte_flow_error_set(error, EINVAL,
3077                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3078                                                 (void *)items->type,
3079                                                 "next protocol not found");
3080                         if (!ipv4 && !ipv6)
3081                                 return rte_flow_error_set(error, EINVAL,
3082                                                 RTE_FLOW_ERROR_TYPE_ACTION,
3083                                                 (void *)items->type,
3084                                                 "ip header not found");
3085                         if (ipv4 && !ipv4->next_proto_id)
3086                                 ipv4->next_proto_id = IPPROTO_GRE;
3087                         else if (ipv6 && !ipv6->proto)
3088                                 ipv6->proto = IPPROTO_GRE;
3089                         break;
3090                 case RTE_FLOW_ITEM_TYPE_VOID:
3091                         break;
3092                 default:
3093                         return rte_flow_error_set(error, EINVAL,
3094                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3095                                                   (void *)items->type,
3096                                                   "unsupported item type");
3097                         break;
3098                 }
3099                 temp_size += len;
3100         }
3101         *size = temp_size;
3102         return 0;
3103 }
3104
3105 static int
3106 flow_dv_zero_encap_udp_csum(void *data, struct rte_flow_error *error)
3107 {
3108         struct rte_ether_hdr *eth = NULL;
3109         struct rte_vlan_hdr *vlan = NULL;
3110         struct rte_ipv6_hdr *ipv6 = NULL;
3111         struct rte_udp_hdr *udp = NULL;
3112         char *next_hdr;
3113         uint16_t proto;
3114
3115         eth = (struct rte_ether_hdr *)data;
3116         next_hdr = (char *)(eth + 1);
3117         proto = RTE_BE16(eth->ether_type);
3118
3119         /* VLAN skipping */
3120         while (proto == RTE_ETHER_TYPE_VLAN || proto == RTE_ETHER_TYPE_QINQ) {
3121                 vlan = (struct rte_vlan_hdr *)next_hdr;
3122                 proto = RTE_BE16(vlan->eth_proto);
3123                 next_hdr += sizeof(struct rte_vlan_hdr);
3124         }
3125
3126         /* HW calculates IPv4 csum. no need to proceed */
3127         if (proto == RTE_ETHER_TYPE_IPV4)
3128                 return 0;
3129
3130         /* non IPv4/IPv6 header. not supported */
3131         if (proto != RTE_ETHER_TYPE_IPV6) {
3132                 return rte_flow_error_set(error, ENOTSUP,
3133                                           RTE_FLOW_ERROR_TYPE_ACTION,
3134                                           NULL, "Cannot offload non IPv4/IPv6");
3135         }
3136
3137         ipv6 = (struct rte_ipv6_hdr *)next_hdr;
3138
3139         /* ignore non UDP */
3140         if (ipv6->proto != IPPROTO_UDP)
3141                 return 0;
3142
3143         udp = (struct rte_udp_hdr *)(ipv6 + 1);
3144         udp->dgram_cksum = 0;
3145
3146         return 0;
3147 }
3148
3149 /**
3150  * Convert L2 encap action to DV specification.
3151  *
3152  * @param[in] dev
3153  *   Pointer to rte_eth_dev structure.
3154  * @param[in] action
3155  *   Pointer to action structure.
3156  * @param[in, out] dev_flow
3157  *   Pointer to the mlx5_flow.
3158  * @param[in] transfer
3159  *   Mark if the flow is E-Switch flow.
3160  * @param[out] error
3161  *   Pointer to the error structure.
3162  *
3163  * @return
3164  *   0 on success, a negative errno value otherwise and rte_errno is set.
3165  */
3166 static int
3167 flow_dv_create_action_l2_encap(struct rte_eth_dev *dev,
3168                                const struct rte_flow_action *action,
3169                                struct mlx5_flow *dev_flow,
3170                                uint8_t transfer,
3171                                struct rte_flow_error *error)
3172 {
3173         const struct rte_flow_item *encap_data;
3174         const struct rte_flow_action_raw_encap *raw_encap_data;
3175         struct mlx5_flow_dv_encap_decap_resource res = {
3176                 .reformat_type =
3177                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL,
3178                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3179                                       MLX5DV_FLOW_TABLE_TYPE_NIC_TX,
3180         };
3181
3182         if (action->type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
3183                 raw_encap_data =
3184                         (const struct rte_flow_action_raw_encap *)action->conf;
3185                 res.size = raw_encap_data->size;
3186                 memcpy(res.buf, raw_encap_data->data, res.size);
3187         } else {
3188                 if (action->type == RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP)
3189                         encap_data =
3190                                 ((const struct rte_flow_action_vxlan_encap *)
3191                                                 action->conf)->definition;
3192                 else
3193                         encap_data =
3194                                 ((const struct rte_flow_action_nvgre_encap *)
3195                                                 action->conf)->definition;
3196                 if (flow_dv_convert_encap_data(encap_data, res.buf,
3197                                                &res.size, error))
3198                         return -rte_errno;
3199         }
3200         if (flow_dv_zero_encap_udp_csum(res.buf, error))
3201                 return -rte_errno;
3202         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3203                 return rte_flow_error_set(error, EINVAL,
3204                                           RTE_FLOW_ERROR_TYPE_ACTION,
3205                                           NULL, "can't create L2 encap action");
3206         return 0;
3207 }
3208
3209 /**
3210  * Convert L2 decap action to DV specification.
3211  *
3212  * @param[in] dev
3213  *   Pointer to rte_eth_dev structure.
3214  * @param[in, out] dev_flow
3215  *   Pointer to the mlx5_flow.
3216  * @param[in] transfer
3217  *   Mark if the flow is E-Switch flow.
3218  * @param[out] error
3219  *   Pointer to the error structure.
3220  *
3221  * @return
3222  *   0 on success, a negative errno value otherwise and rte_errno is set.
3223  */
3224 static int
3225 flow_dv_create_action_l2_decap(struct rte_eth_dev *dev,
3226                                struct mlx5_flow *dev_flow,
3227                                uint8_t transfer,
3228                                struct rte_flow_error *error)
3229 {
3230         struct mlx5_flow_dv_encap_decap_resource res = {
3231                 .size = 0,
3232                 .reformat_type =
3233                         MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2,
3234                 .ft_type = transfer ? MLX5DV_FLOW_TABLE_TYPE_FDB :
3235                                       MLX5DV_FLOW_TABLE_TYPE_NIC_RX,
3236         };
3237
3238         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3239                 return rte_flow_error_set(error, EINVAL,
3240                                           RTE_FLOW_ERROR_TYPE_ACTION,
3241                                           NULL, "can't create L2 decap action");
3242         return 0;
3243 }
3244
3245 /**
3246  * Convert raw decap/encap (L3 tunnel) action to DV specification.
3247  *
3248  * @param[in] dev
3249  *   Pointer to rte_eth_dev structure.
3250  * @param[in] action
3251  *   Pointer to action structure.
3252  * @param[in, out] dev_flow
3253  *   Pointer to the mlx5_flow.
3254  * @param[in] attr
3255  *   Pointer to the flow attributes.
3256  * @param[out] error
3257  *   Pointer to the error structure.
3258  *
3259  * @return
3260  *   0 on success, a negative errno value otherwise and rte_errno is set.
3261  */
3262 static int
3263 flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
3264                                 const struct rte_flow_action *action,
3265                                 struct mlx5_flow *dev_flow,
3266                                 const struct rte_flow_attr *attr,
3267                                 struct rte_flow_error *error)
3268 {
3269         const struct rte_flow_action_raw_encap *encap_data;
3270         struct mlx5_flow_dv_encap_decap_resource res;
3271
3272         memset(&res, 0, sizeof(res));
3273         encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
3274         res.size = encap_data->size;
3275         memcpy(res.buf, encap_data->data, res.size);
3276         res.reformat_type = res.size < MLX5_ENCAPSULATION_DECISION_SIZE ?
3277                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 :
3278                 MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3279         if (attr->transfer)
3280                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3281         else
3282                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3283                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3284         if (flow_dv_encap_decap_resource_register(dev, &res, dev_flow, error))
3285                 return rte_flow_error_set(error, EINVAL,
3286                                           RTE_FLOW_ERROR_TYPE_ACTION,
3287                                           NULL, "can't create encap action");
3288         return 0;
3289 }
3290
3291 /**
3292  * Create action push VLAN.
3293  *
3294  * @param[in] dev
3295  *   Pointer to rte_eth_dev structure.
3296  * @param[in] attr
3297  *   Pointer to the flow attributes.
3298  * @param[in] vlan
3299  *   Pointer to the vlan to push to the Ethernet header.
3300  * @param[in, out] dev_flow
3301  *   Pointer to the mlx5_flow.
3302  * @param[out] error
3303  *   Pointer to the error structure.
3304  *
3305  * @return
3306  *   0 on success, a negative errno value otherwise and rte_errno is set.
3307  */
3308 static int
3309 flow_dv_create_action_push_vlan(struct rte_eth_dev *dev,
3310                                 const struct rte_flow_attr *attr,
3311                                 const struct rte_vlan_hdr *vlan,
3312                                 struct mlx5_flow *dev_flow,
3313                                 struct rte_flow_error *error)
3314 {
3315         struct mlx5_flow_dv_push_vlan_action_resource res;
3316
3317         memset(&res, 0, sizeof(res));
3318         res.vlan_tag =
3319                 rte_cpu_to_be_32(((uint32_t)vlan->eth_proto) << 16 |
3320                                  vlan->vlan_tci);
3321         if (attr->transfer)
3322                 res.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
3323         else
3324                 res.ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
3325                                              MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
3326         return flow_dv_push_vlan_action_resource_register
3327                                             (dev, &res, dev_flow, error);
3328 }
3329
3330 /**
3331  * Validate the modify-header actions.
3332  *
3333  * @param[in] action_flags
3334  *   Holds the actions detected until now.
3335  * @param[in] action
3336  *   Pointer to the modify action.
3337  * @param[out] error
3338  *   Pointer to error structure.
3339  *
3340  * @return
3341  *   0 on success, a negative errno value otherwise and rte_errno is set.
3342  */
3343 static int
3344 flow_dv_validate_action_modify_hdr(const uint64_t action_flags,
3345                                    const struct rte_flow_action *action,
3346                                    struct rte_flow_error *error)
3347 {
3348         if (action->type != RTE_FLOW_ACTION_TYPE_DEC_TTL && !action->conf)
3349                 return rte_flow_error_set(error, EINVAL,
3350                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3351                                           NULL, "action configuration not set");
3352         if (action_flags & MLX5_FLOW_ACTION_ENCAP)
3353                 return rte_flow_error_set(error, EINVAL,
3354                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3355                                           "can't have encap action before"
3356                                           " modify action");
3357         return 0;
3358 }
3359
3360 /**
3361  * Validate the modify-header MAC address actions.
3362  *
3363  * @param[in] action_flags
3364  *   Holds the actions detected until now.
3365  * @param[in] action
3366  *   Pointer to the modify action.
3367  * @param[in] item_flags
3368  *   Holds the items detected.
3369  * @param[out] error
3370  *   Pointer to error structure.
3371  *
3372  * @return
3373  *   0 on success, a negative errno value otherwise and rte_errno is set.
3374  */
3375 static int
3376 flow_dv_validate_action_modify_mac(const uint64_t action_flags,
3377                                    const struct rte_flow_action *action,
3378                                    const uint64_t item_flags,
3379                                    struct rte_flow_error *error)
3380 {
3381         int ret = 0;
3382
3383         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3384         if (!ret) {
3385                 if (!(item_flags & MLX5_FLOW_LAYER_L2))
3386                         return rte_flow_error_set(error, EINVAL,
3387                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3388                                                   NULL,
3389                                                   "no L2 item in pattern");
3390         }
3391         return ret;
3392 }
3393
3394 /**
3395  * Validate the modify-header IPv4 address actions.
3396  *
3397  * @param[in] action_flags
3398  *   Holds the actions detected until now.
3399  * @param[in] action
3400  *   Pointer to the modify action.
3401  * @param[in] item_flags
3402  *   Holds the items detected.
3403  * @param[out] error
3404  *   Pointer to error structure.
3405  *
3406  * @return
3407  *   0 on success, a negative errno value otherwise and rte_errno is set.
3408  */
3409 static int
3410 flow_dv_validate_action_modify_ipv4(const uint64_t action_flags,
3411                                     const struct rte_flow_action *action,
3412                                     const uint64_t item_flags,
3413                                     struct rte_flow_error *error)
3414 {
3415         int ret = 0;
3416         uint64_t layer;
3417
3418         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3419         if (!ret) {
3420                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3421                                  MLX5_FLOW_LAYER_INNER_L3_IPV4 :
3422                                  MLX5_FLOW_LAYER_OUTER_L3_IPV4;
3423                 if (!(item_flags & layer))
3424                         return rte_flow_error_set(error, EINVAL,
3425                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3426                                                   NULL,
3427                                                   "no ipv4 item in pattern");
3428         }
3429         return ret;
3430 }
3431
3432 /**
3433  * Validate the modify-header IPv6 address actions.
3434  *
3435  * @param[in] action_flags
3436  *   Holds the actions detected until now.
3437  * @param[in] action
3438  *   Pointer to the modify action.
3439  * @param[in] item_flags
3440  *   Holds the items detected.
3441  * @param[out] error
3442  *   Pointer to error structure.
3443  *
3444  * @return
3445  *   0 on success, a negative errno value otherwise and rte_errno is set.
3446  */
3447 static int
3448 flow_dv_validate_action_modify_ipv6(const uint64_t action_flags,
3449                                     const struct rte_flow_action *action,
3450                                     const uint64_t item_flags,
3451                                     struct rte_flow_error *error)
3452 {
3453         int ret = 0;
3454         uint64_t layer;
3455
3456         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3457         if (!ret) {
3458                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3459                                  MLX5_FLOW_LAYER_INNER_L3_IPV6 :
3460                                  MLX5_FLOW_LAYER_OUTER_L3_IPV6;
3461                 if (!(item_flags & layer))
3462                         return rte_flow_error_set(error, EINVAL,
3463                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3464                                                   NULL,
3465                                                   "no ipv6 item in pattern");
3466         }
3467         return ret;
3468 }
3469
3470 /**
3471  * Validate the modify-header TP actions.
3472  *
3473  * @param[in] action_flags
3474  *   Holds the actions detected until now.
3475  * @param[in] action
3476  *   Pointer to the modify action.
3477  * @param[in] item_flags
3478  *   Holds the items detected.
3479  * @param[out] error
3480  *   Pointer to error structure.
3481  *
3482  * @return
3483  *   0 on success, a negative errno value otherwise and rte_errno is set.
3484  */
3485 static int
3486 flow_dv_validate_action_modify_tp(const uint64_t action_flags,
3487                                   const struct rte_flow_action *action,
3488                                   const uint64_t item_flags,
3489                                   struct rte_flow_error *error)
3490 {
3491         int ret = 0;
3492         uint64_t layer;
3493
3494         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3495         if (!ret) {
3496                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3497                                  MLX5_FLOW_LAYER_INNER_L4 :
3498                                  MLX5_FLOW_LAYER_OUTER_L4;
3499                 if (!(item_flags & layer))
3500                         return rte_flow_error_set(error, EINVAL,
3501                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3502                                                   NULL, "no transport layer "
3503                                                   "in pattern");
3504         }
3505         return ret;
3506 }
3507
3508 /**
3509  * Validate the modify-header actions of increment/decrement
3510  * TCP Sequence-number.
3511  *
3512  * @param[in] action_flags
3513  *   Holds the actions detected until now.
3514  * @param[in] action
3515  *   Pointer to the modify action.
3516  * @param[in] item_flags
3517  *   Holds the items detected.
3518  * @param[out] error
3519  *   Pointer to error structure.
3520  *
3521  * @return
3522  *   0 on success, a negative errno value otherwise and rte_errno is set.
3523  */
3524 static int
3525 flow_dv_validate_action_modify_tcp_seq(const uint64_t action_flags,
3526                                        const struct rte_flow_action *action,
3527                                        const uint64_t item_flags,
3528                                        struct rte_flow_error *error)
3529 {
3530         int ret = 0;
3531         uint64_t layer;
3532
3533         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3534         if (!ret) {
3535                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3536                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3537                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3538                 if (!(item_flags & layer))
3539                         return rte_flow_error_set(error, EINVAL,
3540                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3541                                                   NULL, "no TCP item in"
3542                                                   " pattern");
3543                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ &&
3544                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_SEQ)) ||
3545                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ &&
3546                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_SEQ)))
3547                         return rte_flow_error_set(error, EINVAL,
3548                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3549                                                   NULL,
3550                                                   "cannot decrease and increase"
3551                                                   " TCP sequence number"
3552                                                   " at the same time");
3553         }
3554         return ret;
3555 }
3556
3557 /**
3558  * Validate the modify-header actions of increment/decrement
3559  * TCP Acknowledgment number.
3560  *
3561  * @param[in] action_flags
3562  *   Holds the actions detected until now.
3563  * @param[in] action
3564  *   Pointer to the modify action.
3565  * @param[in] item_flags
3566  *   Holds the items detected.
3567  * @param[out] error
3568  *   Pointer to error structure.
3569  *
3570  * @return
3571  *   0 on success, a negative errno value otherwise and rte_errno is set.
3572  */
3573 static int
3574 flow_dv_validate_action_modify_tcp_ack(const uint64_t action_flags,
3575                                        const struct rte_flow_action *action,
3576                                        const uint64_t item_flags,
3577                                        struct rte_flow_error *error)
3578 {
3579         int ret = 0;
3580         uint64_t layer;
3581
3582         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3583         if (!ret) {
3584                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3585                                  MLX5_FLOW_LAYER_INNER_L4_TCP :
3586                                  MLX5_FLOW_LAYER_OUTER_L4_TCP;
3587                 if (!(item_flags & layer))
3588                         return rte_flow_error_set(error, EINVAL,
3589                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3590                                                   NULL, "no TCP item in"
3591                                                   " pattern");
3592                 if ((action->type == RTE_FLOW_ACTION_TYPE_INC_TCP_ACK &&
3593                         (action_flags & MLX5_FLOW_ACTION_DEC_TCP_ACK)) ||
3594                     (action->type == RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK &&
3595                         (action_flags & MLX5_FLOW_ACTION_INC_TCP_ACK)))
3596                         return rte_flow_error_set(error, EINVAL,
3597                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3598                                                   NULL,
3599                                                   "cannot decrease and increase"
3600                                                   " TCP acknowledgment number"
3601                                                   " at the same time");
3602         }
3603         return ret;
3604 }
3605
3606 /**
3607  * Validate the modify-header TTL actions.
3608  *
3609  * @param[in] action_flags
3610  *   Holds the actions detected until now.
3611  * @param[in] action
3612  *   Pointer to the modify action.
3613  * @param[in] item_flags
3614  *   Holds the items detected.
3615  * @param[out] error
3616  *   Pointer to error structure.
3617  *
3618  * @return
3619  *   0 on success, a negative errno value otherwise and rte_errno is set.
3620  */
3621 static int
3622 flow_dv_validate_action_modify_ttl(const uint64_t action_flags,
3623                                    const struct rte_flow_action *action,
3624                                    const uint64_t item_flags,
3625                                    struct rte_flow_error *error)
3626 {
3627         int ret = 0;
3628         uint64_t layer;
3629
3630         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3631         if (!ret) {
3632                 layer = (action_flags & MLX5_FLOW_ACTION_DECAP) ?
3633                                  MLX5_FLOW_LAYER_INNER_L3 :
3634                                  MLX5_FLOW_LAYER_OUTER_L3;
3635                 if (!(item_flags & layer))
3636                         return rte_flow_error_set(error, EINVAL,
3637                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3638                                                   NULL,
3639                                                   "no IP protocol in pattern");
3640         }
3641         return ret;
3642 }
3643
3644 /**
3645  * Validate jump action.
3646  *
3647  * @param[in] action
3648  *   Pointer to the jump action.
3649  * @param[in] action_flags
3650  *   Holds the actions detected until now.
3651  * @param[in] attributes
3652  *   Pointer to flow attributes
3653  * @param[in] external
3654  *   Action belongs to flow rule created by request external to PMD.
3655  * @param[out] error
3656  *   Pointer to error structure.
3657  *
3658  * @return
3659  *   0 on success, a negative errno value otherwise and rte_errno is set.
3660  */
3661 static int
3662 flow_dv_validate_action_jump(const struct rte_flow_action *action,
3663                              uint64_t action_flags,
3664                              const struct rte_flow_attr *attributes,
3665                              bool external, struct rte_flow_error *error)
3666 {
3667         uint32_t target_group, table;
3668         int ret = 0;
3669
3670         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3671                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3672                 return rte_flow_error_set(error, EINVAL,
3673                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3674                                           "can't have 2 fate actions in"
3675                                           " same flow");
3676         if (action_flags & MLX5_FLOW_ACTION_METER)
3677                 return rte_flow_error_set(error, ENOTSUP,
3678                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3679                                           "jump with meter not support");
3680         if (!action->conf)
3681                 return rte_flow_error_set(error, EINVAL,
3682                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3683                                           NULL, "action configuration not set");
3684         target_group =
3685                 ((const struct rte_flow_action_jump *)action->conf)->group;
3686         ret = mlx5_flow_group_to_table(attributes, external, target_group,
3687                                        true, &table, error);
3688         if (ret)
3689                 return ret;
3690         if (attributes->group == target_group)
3691                 return rte_flow_error_set(error, EINVAL,
3692                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3693                                           "target group must be other than"
3694                                           " the current flow group");
3695         return 0;
3696 }
3697
3698 /*
3699  * Validate the port_id action.
3700  *
3701  * @param[in] dev
3702  *   Pointer to rte_eth_dev structure.
3703  * @param[in] action_flags
3704  *   Bit-fields that holds the actions detected until now.
3705  * @param[in] action
3706  *   Port_id RTE action structure.
3707  * @param[in] attr
3708  *   Attributes of flow that includes this action.
3709  * @param[out] error
3710  *   Pointer to error structure.
3711  *
3712  * @return
3713  *   0 on success, a negative errno value otherwise and rte_errno is set.
3714  */
3715 static int
3716 flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
3717                                 uint64_t action_flags,
3718                                 const struct rte_flow_action *action,
3719                                 const struct rte_flow_attr *attr,
3720                                 struct rte_flow_error *error)
3721 {
3722         const struct rte_flow_action_port_id *port_id;
3723         struct mlx5_priv *act_priv;
3724         struct mlx5_priv *dev_priv;
3725         uint16_t port;
3726
3727         if (!attr->transfer)
3728                 return rte_flow_error_set(error, ENOTSUP,
3729                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3730                                           NULL,
3731                                           "port id action is valid in transfer"
3732                                           " mode only");
3733         if (!action || !action->conf)
3734                 return rte_flow_error_set(error, ENOTSUP,
3735                                           RTE_FLOW_ERROR_TYPE_ACTION_CONF,
3736                                           NULL,
3737                                           "port id action parameters must be"
3738                                           " specified");
3739         if (action_flags & (MLX5_FLOW_FATE_ACTIONS |
3740                             MLX5_FLOW_FATE_ESWITCH_ACTIONS))
3741                 return rte_flow_error_set(error, EINVAL,
3742                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3743                                           "can have only one fate actions in"
3744                                           " a flow");
3745         dev_priv = mlx5_dev_to_eswitch_info(dev);
3746         if (!dev_priv)
3747                 return rte_flow_error_set(error, rte_errno,
3748                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3749                                           NULL,
3750                                           "failed to obtain E-Switch info");
3751         port_id = action->conf;
3752         port = port_id->original ? dev->data->port_id : port_id->id;
3753         act_priv = mlx5_port_to_eswitch_info(port, false);
3754         if (!act_priv)
3755                 return rte_flow_error_set
3756                                 (error, rte_errno,
3757                                  RTE_FLOW_ERROR_TYPE_ACTION_CONF, port_id,
3758                                  "failed to obtain E-Switch port id for port");
3759         if (act_priv->domain_id != dev_priv->domain_id)
3760                 return rte_flow_error_set
3761                                 (error, EINVAL,
3762                                  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3763                                  "port does not belong to"
3764                                  " E-Switch being configured");
3765         return 0;
3766 }
3767
3768 /**
3769  * Get the maximum number of modify header actions.
3770  *
3771  * @param dev
3772  *   Pointer to rte_eth_dev structure.
3773  * @param flags
3774  *   Flags bits to check if root level.
3775  *
3776  * @return
3777  *   Max number of modify header actions device can support.
3778  */
3779 static inline unsigned int
3780 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
3781                               uint64_t flags)
3782 {
3783         /*
3784          * There's no way to directly query the max capacity from FW.
3785          * The maximal value on root table should be assumed to be supported.
3786          */
3787         if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
3788                 return MLX5_MAX_MODIFY_NUM;
3789         else
3790                 return MLX5_ROOT_TBL_MODIFY_NUM;
3791 }
3792
3793 /**
3794  * Validate the meter action.
3795  *
3796  * @param[in] dev
3797  *   Pointer to rte_eth_dev structure.
3798  * @param[in] action_flags
3799  *   Bit-fields that holds the actions detected until now.
3800  * @param[in] action
3801  *   Pointer to the meter action.
3802  * @param[in] attr
3803  *   Attributes of flow that includes this action.
3804  * @param[out] error
3805  *   Pointer to error structure.
3806  *
3807  * @return
3808  *   0 on success, a negative errno value otherwise and rte_ernno is set.
3809  */
3810 static int
3811 mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
3812                                 uint64_t action_flags,
3813                                 const struct rte_flow_action *action,
3814                                 const struct rte_flow_attr *attr,
3815                                 struct rte_flow_error *error)
3816 {
3817         struct mlx5_priv *priv = dev->data->dev_private;
3818         const struct rte_flow_action_meter *am = action->conf;
3819         struct mlx5_flow_meter *fm;
3820
3821         if (!am)
3822                 return rte_flow_error_set(error, EINVAL,
3823                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3824                                           "meter action conf is NULL");
3825
3826         if (action_flags & MLX5_FLOW_ACTION_METER)
3827                 return rte_flow_error_set(error, ENOTSUP,
3828                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3829                                           "meter chaining not support");
3830         if (action_flags & MLX5_FLOW_ACTION_JUMP)
3831                 return rte_flow_error_set(error, ENOTSUP,
3832                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3833                                           "meter with jump not support");
3834         if (!priv->mtr_en)
3835                 return rte_flow_error_set(error, ENOTSUP,
3836                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3837                                           NULL,
3838                                           "meter action not supported");
3839         fm = mlx5_flow_meter_find(priv, am->mtr_id);
3840         if (!fm)
3841                 return rte_flow_error_set(error, EINVAL,
3842                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3843                                           "Meter not found");
3844         if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
3845               (!fm->ingress && !attr->ingress && attr->egress) ||
3846               (!fm->egress && !attr->egress && attr->ingress))))
3847                 return rte_flow_error_set(error, EINVAL,
3848                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3849                                           "Flow attributes are either invalid "
3850                                           "or have a conflict with current "
3851                                           "meter attributes");
3852         return 0;
3853 }
3854
3855 /**
3856  * Validate the age action.
3857  *
3858  * @param[in] action_flags
3859  *   Holds the actions detected until now.
3860  * @param[in] action
3861  *   Pointer to the age action.
3862  * @param[in] dev
3863  *   Pointer to the Ethernet device structure.
3864  * @param[out] error
3865  *   Pointer to error structure.
3866  *
3867  * @return
3868  *   0 on success, a negative errno value otherwise and rte_errno is set.
3869  */
3870 static int
3871 flow_dv_validate_action_age(uint64_t action_flags,
3872                             const struct rte_flow_action *action,
3873                             struct rte_eth_dev *dev,
3874                             struct rte_flow_error *error)
3875 {
3876         struct mlx5_priv *priv = dev->data->dev_private;
3877         const struct rte_flow_action_age *age = action->conf;
3878
3879         if (!priv->config.devx || priv->counter_fallback)
3880                 return rte_flow_error_set(error, ENOTSUP,
3881                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
3882                                           NULL,
3883                                           "age action not supported");
3884         if (!(action->conf))
3885                 return rte_flow_error_set(error, EINVAL,
3886                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3887                                           "configuration cannot be null");
3888         if (age->timeout >= UINT16_MAX / 2 / 10)
3889                 return rte_flow_error_set(error, ENOTSUP,
3890                                           RTE_FLOW_ERROR_TYPE_ACTION, action,
3891                                           "Max age time: 3275 seconds");
3892         if (action_flags & MLX5_FLOW_ACTION_AGE)
3893                 return rte_flow_error_set(error, EINVAL,
3894                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
3895                                           "Duplicate age ctions set");
3896         return 0;
3897 }
3898
3899 /**
3900  * Validate the modify-header IPv4 DSCP actions.
3901  *
3902  * @param[in] action_flags
3903  *   Holds the actions detected until now.
3904  * @param[in] action
3905  *   Pointer to the modify action.
3906  * @param[in] item_flags
3907  *   Holds the items detected.
3908  * @param[out] error
3909  *   Pointer to error structure.
3910  *
3911  * @return
3912  *   0 on success, a negative errno value otherwise and rte_errno is set.
3913  */
3914 static int
3915 flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
3916                                          const struct rte_flow_action *action,
3917                                          const uint64_t item_flags,
3918                                          struct rte_flow_error *error)
3919 {
3920         int ret = 0;
3921
3922         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3923         if (!ret) {
3924                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
3925                         return rte_flow_error_set(error, EINVAL,
3926                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3927                                                   NULL,
3928                                                   "no ipv4 item in pattern");
3929         }
3930         return ret;
3931 }
3932
3933 /**
3934  * Validate the modify-header IPv6 DSCP actions.
3935  *
3936  * @param[in] action_flags
3937  *   Holds the actions detected until now.
3938  * @param[in] action
3939  *   Pointer to the modify action.
3940  * @param[in] item_flags
3941  *   Holds the items detected.
3942  * @param[out] error
3943  *   Pointer to error structure.
3944  *
3945  * @return
3946  *   0 on success, a negative errno value otherwise and rte_errno is set.
3947  */
3948 static int
3949 flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
3950                                          const struct rte_flow_action *action,
3951                                          const uint64_t item_flags,
3952                                          struct rte_flow_error *error)
3953 {
3954         int ret = 0;
3955
3956         ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
3957         if (!ret) {
3958                 if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
3959                         return rte_flow_error_set(error, EINVAL,
3960                                                   RTE_FLOW_ERROR_TYPE_ACTION,
3961                                                   NULL,
3962                                                   "no ipv6 item in pattern");
3963         }
3964         return ret;
3965 }
3966
3967 /**
3968  * Find existing modify-header resource or create and register a new one.
3969  *
3970  * @param dev[in, out]
3971  *   Pointer to rte_eth_dev structure.
3972  * @param[in, out] resource
3973  *   Pointer to modify-header resource.
3974  * @parm[in, out] dev_flow
3975  *   Pointer to the dev_flow.
3976  * @param[out] error
3977  *   pointer to error structure.
3978  *
3979  * @return
3980  *   0 on success otherwise -errno and errno is set.
3981  */
3982 static int
3983 flow_dv_modify_hdr_resource_register
3984                         (struct rte_eth_dev *dev,
3985                          struct mlx5_flow_dv_modify_hdr_resource *resource,
3986                          struct mlx5_flow *dev_flow,
3987                          struct rte_flow_error *error)
3988 {
3989         struct mlx5_priv *priv = dev->data->dev_private;
3990         struct mlx5_dev_ctx_shared *sh = priv->sh;
3991         struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
3992         struct mlx5dv_dr_domain *ns;
3993         uint32_t actions_len;
3994
3995         resource->flags = dev_flow->dv.group ? 0 :
3996                           MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
3997         if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
3998                                     resource->flags))
3999                 return rte_flow_error_set(error, EOVERFLOW,
4000                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
4001                                           "too many modify header items");
4002         if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
4003                 ns = sh->fdb_domain;
4004         else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
4005                 ns = sh->tx_domain;
4006         else
4007                 ns = sh->rx_domain;
4008         /* Lookup a matching resource from cache. */
4009         actions_len = resource->actions_num * sizeof(resource->actions[0]);
4010         LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
4011                 if (resource->ft_type == cache_resource->ft_type &&
4012                     resource->actions_num == cache_resource->actions_num &&
4013                     resource->flags == cache_resource->flags &&
4014                     !memcmp((const void *)resource->actions,
4015                             (const void *)cache_resource->actions,
4016                             actions_len)) {
4017                         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
4018                                 (void *)cache_resource,
4019                                 rte_atomic32_read(&cache_resource->refcnt));
4020                         rte_atomic32_inc(&cache_resource->refcnt);
4021                         dev_flow->handle->dvh.modify_hdr = cache_resource;
4022                         return 0;
4023                 }
4024         }
4025         /* Register new modify-header resource. */
4026         cache_resource = rte_calloc(__func__, 1,
4027                                     sizeof(*cache_resource) + actions_len, 0);
4028         if (!cache_resource)
4029                 return rte_flow_error_set(error, ENOMEM,
4030                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4031                                           "cannot allocate resource memory");
4032         *cache_resource = *resource;
4033         rte_memcpy(cache_resource->actions, resource->actions, actions_len);
4034         cache_resource->action =
4035                 mlx5_glue->dv_create_flow_action_modify_header
4036                                         (sh->ctx, cache_resource->ft_type, ns,
4037                                          cache_resource->flags, actions_len,
4038                                          (uint64_t *)cache_resource->actions);
4039         if (!cache_resource->action) {
4040                 rte_free(cache_resource);
4041                 return rte_flow_error_set(error, ENOMEM,
4042                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4043                                           NULL, "cannot create action");
4044         }
4045         rte_atomic32_init(&cache_resource->refcnt);
4046         rte_atomic32_inc(&cache_resource->refcnt);
4047         LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
4048         dev_flow->handle->dvh.modify_hdr = cache_resource;
4049         DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
4050                 (void *)cache_resource,
4051                 rte_atomic32_read(&cache_resource->refcnt));
4052         return 0;
4053 }
4054
4055 /**
4056  * Get DV flow counter by index.
4057  *
4058  * @param[in] dev
4059  *   Pointer to the Ethernet device structure.
4060  * @param[in] idx
4061  *   mlx5 flow counter index in the container.
4062  * @param[out] ppool
4063  *   mlx5 flow counter pool in the container,
4064  *
4065  * @return
4066  *   Pointer to the counter, NULL otherwise.
4067  */
4068 static struct mlx5_flow_counter *
4069 flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
4070                            uint32_t idx,
4071                            struct mlx5_flow_counter_pool **ppool)
4072 {
4073         struct mlx5_priv *priv = dev->data->dev_private;
4074         struct mlx5_pools_container *cont;
4075         struct mlx5_flow_counter_pool *pool;
4076         uint32_t batch = 0, age = 0;
4077
4078         idx--;
4079         age = MLX_CNT_IS_AGE(idx);
4080         idx = age ? idx - MLX5_CNT_AGE_OFFSET : idx;
4081         if (idx >= MLX5_CNT_BATCH_OFFSET) {
4082                 idx -= MLX5_CNT_BATCH_OFFSET;
4083                 batch = 1;
4084         }
4085         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4086         MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n);
4087         pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
4088         MLX5_ASSERT(pool);
4089         if (ppool)
4090                 *ppool = pool;
4091         return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL);
4092 }
4093
4094 /**
4095  * Check the devx counter belongs to the pool.
4096  *
4097  * @param[in] pool
4098  *   Pointer to the counter pool.
4099  * @param[in] id
4100  *   The counter devx ID.
4101  *
4102  * @return
4103  *   True if counter belongs to the pool, false otherwise.
4104  */
4105 static bool
4106 flow_dv_is_counter_in_pool(struct mlx5_flow_counter_pool *pool, int id)
4107 {
4108         int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
4109                    MLX5_COUNTERS_PER_POOL;
4110
4111         if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
4112                 return true;
4113         return false;
4114 }
4115
4116 /**
4117  * Get a pool by devx counter ID.
4118  *
4119  * @param[in] cont
4120  *   Pointer to the counter container.
4121  * @param[in] id
4122  *   The counter devx ID.
4123  *
4124  * @return
4125  *   The counter pool pointer if exists, NULL otherwise,
4126  */
4127 static struct mlx5_flow_counter_pool *
4128 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
4129 {
4130         uint32_t i;
4131
4132         /* Check last used pool. */
4133         if (cont->last_pool_idx != POOL_IDX_INVALID &&
4134             flow_dv_is_counter_in_pool(cont->pools[cont->last_pool_idx], id))
4135                 return cont->pools[cont->last_pool_idx];
4136         /* ID out of range means no suitable pool in the container. */
4137         if (id > cont->max_id || id < cont->min_id)
4138                 return NULL;
4139         /*
4140          * Find the pool from the end of the container, since mostly counter
4141          * ID is sequence increasing, and the last pool should be the needed
4142          * one.
4143          */
4144         i = rte_atomic16_read(&cont->n_valid);
4145         while (i--) {
4146                 struct mlx5_flow_counter_pool *pool = cont->pools[i];
4147
4148                 if (flow_dv_is_counter_in_pool(pool, id))
4149                         return pool;
4150         }
4151         return NULL;
4152 }
4153
4154 /**
4155  * Allocate a new memory for the counter values wrapped by all the needed
4156  * management.
4157  *
4158  * @param[in] dev
4159  *   Pointer to the Ethernet device structure.
4160  * @param[in] raws_n
4161  *   The raw memory areas - each one for MLX5_COUNTERS_PER_POOL counters.
4162  *
4163  * @return
4164  *   The new memory management pointer on success, otherwise NULL and rte_errno
4165  *   is set.
4166  */
4167 static struct mlx5_counter_stats_mem_mng *
4168 flow_dv_create_counter_stat_mem_mng(struct rte_eth_dev *dev, int raws_n)
4169 {
4170         struct mlx5_priv *priv = dev->data->dev_private;
4171         struct mlx5_dev_ctx_shared *sh = priv->sh;
4172         struct mlx5_devx_mkey_attr mkey_attr;
4173         struct mlx5_counter_stats_mem_mng *mem_mng;
4174         volatile struct flow_counter_stats *raw_data;
4175         int size = (sizeof(struct flow_counter_stats) *
4176                         MLX5_COUNTERS_PER_POOL +
4177                         sizeof(struct mlx5_counter_stats_raw)) * raws_n +
4178                         sizeof(struct mlx5_counter_stats_mem_mng);
4179         uint8_t *mem = rte_calloc(__func__, 1, size, sysconf(_SC_PAGESIZE));
4180         int i;
4181
4182         if (!mem) {
4183                 rte_errno = ENOMEM;
4184                 return NULL;
4185         }
4186         mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
4187         size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
4188         mem_mng->umem = mlx5_glue->devx_umem_reg(sh->ctx, mem, size,
4189                                                  IBV_ACCESS_LOCAL_WRITE);
4190         if (!mem_mng->umem) {
4191                 rte_errno = errno;
4192                 rte_free(mem);
4193                 return NULL;
4194         }
4195         mkey_attr.addr = (uintptr_t)mem;
4196         mkey_attr.size = size;
4197         mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
4198         mkey_attr.pd = sh->pdn;
4199         mkey_attr.log_entity_size = 0;
4200         mkey_attr.pg_access = 0;
4201         mkey_attr.klm_array = NULL;
4202         mkey_attr.klm_num = 0;
4203         if (priv->config.hca_attr.relaxed_ordering_write &&
4204                 priv->config.hca_attr.relaxed_ordering_read  &&
4205                 !haswell_broadwell_cpu)
4206                 mkey_attr.relaxed_ordering = 1;
4207         mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
4208         if (!mem_mng->dm) {
4209                 mlx5_glue->devx_umem_dereg(mem_mng->umem);
4210                 rte_errno = errno;
4211                 rte_free(mem);
4212                 return NULL;
4213         }
4214         mem_mng->raws = (struct mlx5_counter_stats_raw *)(mem + size);
4215         raw_data = (volatile struct flow_counter_stats *)mem;
4216         for (i = 0; i < raws_n; ++i) {
4217                 mem_mng->raws[i].mem_mng = mem_mng;
4218                 mem_mng->raws[i].data = raw_data + i * MLX5_COUNTERS_PER_POOL;
4219         }
4220         LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
4221         return mem_mng;
4222 }
4223
4224 /**
4225  * Resize a counter container.
4226  *
4227  * @param[in] dev
4228  *   Pointer to the Ethernet device structure.
4229  * @param[in] batch
4230  *   Whether the pool is for counter that was allocated by batch command.
4231  * @param[in] age
4232  *   Whether the pool is for Aging counter.
4233  *
4234  * @return
4235  *   0 on success, otherwise negative errno value and rte_errno is set.
4236  */
4237 static int
4238 flow_dv_container_resize(struct rte_eth_dev *dev,
4239                                 uint32_t batch, uint32_t age)
4240 {
4241         struct mlx5_priv *priv = dev->data->dev_private;
4242         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4243                                                                age);
4244         struct mlx5_counter_stats_mem_mng *mem_mng = NULL;
4245         void *old_pools = cont->pools;
4246         uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
4247         uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
4248         void *pools = rte_calloc(__func__, 1, mem_size, 0);
4249
4250         if (!pools) {
4251                 rte_errno = ENOMEM;
4252                 return -ENOMEM;
4253         }
4254         if (old_pools)
4255                 memcpy(pools, old_pools, cont->n *
4256                                        sizeof(struct mlx5_flow_counter_pool *));
4257         /*
4258          * Fallback mode query the counter directly, no background query
4259          * resources are needed.
4260          */
4261         if (!priv->counter_fallback) {
4262                 int i;
4263
4264                 mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
4265                           MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
4266                 if (!mem_mng) {
4267                         rte_free(pools);
4268                         return -ENOMEM;
4269                 }
4270                 for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
4271                         LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
4272                                          mem_mng->raws +
4273                                          MLX5_CNT_CONTAINER_RESIZE +
4274                                          i, next);
4275         }
4276         rte_spinlock_lock(&cont->resize_sl);
4277         cont->n = resize;
4278         cont->mem_mng = mem_mng;
4279         cont->pools = pools;
4280         rte_spinlock_unlock(&cont->resize_sl);
4281         if (old_pools)
4282                 rte_free(old_pools);
4283         return 0;
4284 }
4285
4286 /**
4287  * Query a devx flow counter.
4288  *
4289  * @param[in] dev
4290  *   Pointer to the Ethernet device structure.
4291  * @param[in] cnt
4292  *   Index to the flow counter.
4293  * @param[out] pkts
4294  *   The statistics value of packets.
4295  * @param[out] bytes
4296  *   The statistics value of bytes.
4297  *
4298  * @return
4299  *   0 on success, otherwise a negative errno value and rte_errno is set.
4300  */
4301 static inline int
4302 _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
4303                      uint64_t *bytes)
4304 {
4305         struct mlx5_priv *priv = dev->data->dev_private;
4306         struct mlx5_flow_counter_pool *pool = NULL;
4307         struct mlx5_flow_counter *cnt;
4308         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4309         int offset;
4310
4311         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4312         MLX5_ASSERT(pool);
4313         if (counter < MLX5_CNT_BATCH_OFFSET) {
4314                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4315                 if (priv->counter_fallback)
4316                         return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
4317                                         0, pkts, bytes, 0, NULL, NULL, 0);
4318         }
4319
4320         rte_spinlock_lock(&pool->sl);
4321         /*
4322          * The single counters allocation may allocate smaller ID than the
4323          * current allocated in parallel to the host reading.
4324          * In this case the new counter values must be reported as 0.
4325          */
4326         if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) {
4327                 *pkts = 0;
4328                 *bytes = 0;
4329         } else {
4330                 offset = MLX5_CNT_ARRAY_IDX(pool, cnt);
4331                 *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
4332                 *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
4333         }
4334         rte_spinlock_unlock(&pool->sl);
4335         return 0;
4336 }
4337
4338 /**
4339  * Create and initialize a new counter pool.
4340  *
4341  * @param[in] dev
4342  *   Pointer to the Ethernet device structure.
4343  * @param[out] dcs
4344  *   The devX counter handle.
4345  * @param[in] batch
4346  *   Whether the pool is for counter that was allocated by batch command.
4347  * @param[in] age
4348  *   Whether the pool is for counter that was allocated for aging.
4349  * @param[in/out] cont_cur
4350  *   Pointer to the container pointer, it will be update in pool resize.
4351  *
4352  * @return
4353  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
4354  */
4355 static struct mlx5_flow_counter_pool *
4356 flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
4357                     uint32_t batch, uint32_t age)
4358 {
4359         struct mlx5_priv *priv = dev->data->dev_private;
4360         struct mlx5_flow_counter_pool *pool;
4361         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4362                                                                age);
4363         int16_t n_valid = rte_atomic16_read(&cont->n_valid);
4364         uint32_t size = sizeof(*pool);
4365
4366         if (cont->n == n_valid && flow_dv_container_resize(dev, batch, age))
4367                 return NULL;
4368         size += MLX5_COUNTERS_PER_POOL * CNT_SIZE;
4369         size += (batch ? 0 : MLX5_COUNTERS_PER_POOL * CNTEXT_SIZE);
4370         size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * AGE_SIZE);
4371         pool = rte_calloc(__func__, 1, size, 0);
4372         if (!pool) {
4373                 rte_errno = ENOMEM;
4374                 return NULL;
4375         }
4376         pool->min_dcs = dcs;
4377         if (!priv->counter_fallback)
4378                 pool->raw = cont->mem_mng->raws + n_valid %
4379                                                       MLX5_CNT_CONTAINER_RESIZE;
4380         pool->raw_hw = NULL;
4381         pool->type = 0;
4382         pool->type |= (batch ? 0 :  CNT_POOL_TYPE_EXT);
4383         pool->type |= (!age ? 0 :  CNT_POOL_TYPE_AGE);
4384         pool->query_gen = 0;
4385         rte_spinlock_init(&pool->sl);
4386         TAILQ_INIT(&pool->counters[0]);
4387         TAILQ_INIT(&pool->counters[1]);
4388         TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
4389         pool->index = n_valid;
4390         cont->pools[n_valid] = pool;
4391         if (!batch) {
4392                 int base = RTE_ALIGN_FLOOR(dcs->id, MLX5_COUNTERS_PER_POOL);
4393
4394                 if (base < cont->min_id)
4395                         cont->min_id = base;
4396                 if (base > cont->max_id)
4397                         cont->max_id = base + MLX5_COUNTERS_PER_POOL - 1;
4398                 cont->last_pool_idx = pool->index;
4399         }
4400         /* Pool initialization must be updated before host thread access. */
4401         rte_cio_wmb();
4402         rte_atomic16_add(&cont->n_valid, 1);
4403         return pool;
4404 }
4405
4406 /**
4407  * Update the minimum dcs-id for aged or no-aged counter pool.
4408  *
4409  * @param[in] dev
4410  *   Pointer to the Ethernet device structure.
4411  * @param[in] pool
4412  *   Current counter pool.
4413  * @param[in] batch
4414  *   Whether the pool is for counter that was allocated by batch command.
4415  * @param[in] age
4416  *   Whether the counter is for aging.
4417  */
4418 static void
4419 flow_dv_counter_update_min_dcs(struct rte_eth_dev *dev,
4420                         struct mlx5_flow_counter_pool *pool,
4421                         uint32_t batch, uint32_t age)
4422 {
4423         struct mlx5_priv *priv = dev->data->dev_private;
4424         struct mlx5_flow_counter_pool *other;
4425         struct mlx5_pools_container *cont;
4426
4427         cont = MLX5_CNT_CONTAINER(priv->sh, batch, (age ^ 0x1));
4428         other = flow_dv_find_pool_by_id(cont, pool->min_dcs->id);
4429         if (!other)
4430                 return;
4431         if (pool->min_dcs->id < other->min_dcs->id) {
4432                 rte_atomic64_set(&other->a64_dcs,
4433                         rte_atomic64_read(&pool->a64_dcs));
4434         } else {
4435                 rte_atomic64_set(&pool->a64_dcs,
4436                         rte_atomic64_read(&other->a64_dcs));
4437         }
4438 }
4439 /**
4440  * Prepare a new counter and/or a new counter pool.
4441  *
4442  * @param[in] dev
4443  *   Pointer to the Ethernet device structure.
4444  * @param[out] cnt_free
4445  *   Where to put the pointer of a new counter.
4446  * @param[in] batch
4447  *   Whether the pool is for counter that was allocated by batch command.
4448  * @param[in] age
4449  *   Whether the pool is for counter that was allocated for aging.
4450  *
4451  * @return
4452  *   The counter pool pointer and @p cnt_free is set on success,
4453  *   NULL otherwise and rte_errno is set.
4454  */
4455 static struct mlx5_flow_counter_pool *
4456 flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
4457                              struct mlx5_flow_counter **cnt_free,
4458                              uint32_t batch, uint32_t age)
4459 {
4460         struct mlx5_priv *priv = dev->data->dev_private;
4461         struct mlx5_pools_container *cont;
4462         struct mlx5_flow_counter_pool *pool;
4463         struct mlx5_counters tmp_tq;
4464         struct mlx5_devx_obj *dcs = NULL;
4465         struct mlx5_flow_counter *cnt;
4466         uint32_t i;
4467
4468         cont = MLX5_CNT_CONTAINER(priv->sh, batch, age);
4469         if (!batch) {
4470                 /* bulk_bitmap must be 0 for single counter allocation. */
4471                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
4472                 if (!dcs)
4473                         return NULL;
4474                 pool = flow_dv_find_pool_by_id(cont, dcs->id);
4475                 if (!pool) {
4476                         pool = flow_dv_pool_create(dev, dcs, batch, age);
4477                         if (!pool) {
4478                                 mlx5_devx_cmd_destroy(dcs);
4479                                 return NULL;
4480                         }
4481                 } else if (dcs->id < pool->min_dcs->id) {
4482                         rte_atomic64_set(&pool->a64_dcs,
4483                                          (int64_t)(uintptr_t)dcs);
4484                 }
4485                 flow_dv_counter_update_min_dcs(dev,
4486                                                 pool, batch, age);
4487                 i = dcs->id % MLX5_COUNTERS_PER_POOL;
4488                 cnt = MLX5_POOL_GET_CNT(pool, i);
4489                 cnt->pool = pool;
4490                 MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs;
4491                 *cnt_free = cnt;
4492                 return pool;
4493         }
4494         /* bulk_bitmap is in 128 counters units. */
4495         if (priv->config.hca_attr.flow_counter_bulk_alloc_bitmap & 0x4)
4496                 dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
4497         if (!dcs) {
4498                 rte_errno = ENODATA;
4499                 return NULL;
4500         }
4501         pool = flow_dv_pool_create(dev, dcs, batch, age);
4502         if (!pool) {
4503                 mlx5_devx_cmd_destroy(dcs);
4504                 return NULL;
4505         }
4506         TAILQ_INIT(&tmp_tq);
4507         for (i = 1; i < MLX5_COUNTERS_PER_POOL; ++i) {
4508                 cnt = MLX5_POOL_GET_CNT(pool, i);
4509                 cnt->pool = pool;
4510                 TAILQ_INSERT_HEAD(&tmp_tq, cnt, next);
4511         }
4512         rte_spinlock_lock(&cont->csl);
4513         TAILQ_CONCAT(&cont->counters, &tmp_tq, next);
4514         rte_spinlock_unlock(&cont->csl);
4515         *cnt_free = MLX5_POOL_GET_CNT(pool, 0);
4516         (*cnt_free)->pool = pool;
4517         return pool;
4518 }
4519
4520 /**
4521  * Search for existed shared counter.
4522  *
4523  * @param[in] dev
4524  *   Pointer to the Ethernet device structure.
4525  * @param[in] id
4526  *   The shared counter ID to search.
4527  * @param[out] ppool
4528  *   mlx5 flow counter pool in the container,
4529  *
4530  * @return
4531  *   NULL if not existed, otherwise pointer to the shared extend counter.
4532  */
4533 static struct mlx5_flow_counter_ext *
4534 flow_dv_counter_shared_search(struct rte_eth_dev *dev, uint32_t id,
4535                               struct mlx5_flow_counter_pool **ppool)
4536 {
4537         struct mlx5_priv *priv = dev->data->dev_private;
4538         union mlx5_l3t_data data;
4539         uint32_t cnt_idx;
4540
4541         if (mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) || !data.dword)
4542                 return NULL;
4543         cnt_idx = data.dword;
4544         /*
4545          * Shared counters don't have age info. The counter extend is after
4546          * the counter datat structure.
4547          */
4548         return (struct mlx5_flow_counter_ext *)
4549                ((flow_dv_counter_get_by_idx(dev, cnt_idx, ppool)) + 1);
4550 }
4551
4552 /**
4553  * Allocate a flow counter.
4554  *
4555  * @param[in] dev
4556  *   Pointer to the Ethernet device structure.
4557  * @param[in] shared
4558  *   Indicate if this counter is shared with other flows.
4559  * @param[in] id
4560  *   Counter identifier.
4561  * @param[in] group
4562  *   Counter flow group.
4563  * @param[in] age
4564  *   Whether the counter was allocated for aging.
4565  *
4566  * @return
4567  *   Index to flow counter on success, 0 otherwise and rte_errno is set.
4568  */
4569 static uint32_t
4570 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
4571                       uint16_t group, uint32_t age)
4572 {
4573         struct mlx5_priv *priv = dev->data->dev_private;
4574         struct mlx5_flow_counter_pool *pool = NULL;
4575         struct mlx5_flow_counter *cnt_free = NULL;
4576         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4577         /*
4578          * Currently group 0 flow counter cannot be assigned to a flow if it is
4579          * not the first one in the batch counter allocation, so it is better
4580          * to allocate counters one by one for these flows in a separate
4581          * container.
4582          * A counter can be shared between different groups so need to take
4583          * shared counters from the single container.
4584          */
4585         uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0;
4586         struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
4587                                                                age);
4588         uint32_t cnt_idx;
4589
4590         if (!priv->config.devx) {
4591                 rte_errno = ENOTSUP;
4592                 return 0;
4593         }
4594         if (shared) {
4595                 cnt_ext = flow_dv_counter_shared_search(dev, id, &pool);
4596                 if (cnt_ext) {
4597                         if (cnt_ext->ref_cnt + 1 == 0) {
4598                                 rte_errno = E2BIG;
4599                                 return 0;
4600                         }
4601                         cnt_ext->ref_cnt++;
4602                         cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL +
4603                                   (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL)
4604                                   + 1;
4605                         return cnt_idx;
4606                 }
4607         }
4608         /* Get free counters from container. */
4609         rte_spinlock_lock(&cont->csl);
4610         cnt_free = TAILQ_FIRST(&cont->counters);
4611         if (cnt_free)
4612                 TAILQ_REMOVE(&cont->counters, cnt_free, next);
4613         rte_spinlock_unlock(&cont->csl);
4614         if (!cnt_free && !flow_dv_counter_pool_prepare(dev, &cnt_free,
4615                                                        batch, age))
4616                 goto err;
4617         pool = cnt_free->pool;
4618         if (!batch)
4619                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free);
4620         /* Create a DV counter action only in the first time usage. */
4621         if (!cnt_free->action) {
4622                 uint16_t offset;
4623                 struct mlx5_devx_obj *dcs;
4624
4625                 if (batch) {
4626                         offset = MLX5_CNT_ARRAY_IDX(pool, cnt_free);
4627                         dcs = pool->min_dcs;
4628                 } else {
4629                         offset = 0;
4630                         dcs = cnt_ext->dcs;
4631                 }
4632                 cnt_free->action = mlx5_glue->dv_create_flow_action_counter
4633                                         (dcs->obj, offset);
4634                 if (!cnt_free->action) {
4635                         rte_errno = errno;
4636                         goto err;
4637                 }
4638         }
4639         cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
4640                                 MLX5_CNT_ARRAY_IDX(pool, cnt_free));
4641         cnt_idx += batch * MLX5_CNT_BATCH_OFFSET;
4642         cnt_idx += age * MLX5_CNT_AGE_OFFSET;
4643         /* Update the counter reset values. */
4644         if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
4645                                  &cnt_free->bytes))
4646                 goto err;
4647         if (cnt_ext) {
4648                 cnt_ext->shared = shared;
4649                 cnt_ext->ref_cnt = 1;
4650                 cnt_ext->id = id;
4651                 if (shared) {
4652                         union mlx5_l3t_data data;
4653
4654                         data.dword = cnt_idx;
4655                         if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data))
4656                                 return 0;
4657                 }
4658         }
4659         if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on)
4660                 /* Start the asynchronous batch query by the host thread. */
4661                 mlx5_set_query_alarm(priv->sh);
4662         return cnt_idx;
4663 err:
4664         if (cnt_free) {
4665                 cnt_free->pool = pool;
4666                 rte_spinlock_lock(&cont->csl);
4667                 TAILQ_INSERT_TAIL(&cont->counters, cnt_free, next);
4668                 rte_spinlock_unlock(&cont->csl);
4669         }
4670         return 0;
4671 }
4672
4673 /**
4674  * Get age param from counter index.
4675  *
4676  * @param[in] dev
4677  *   Pointer to the Ethernet device structure.
4678  * @param[in] counter
4679  *   Index to the counter handler.
4680  *
4681  * @return
4682  *   The aging parameter specified for the counter index.
4683  */
4684 static struct mlx5_age_param*
4685 flow_dv_counter_idx_get_age(struct rte_eth_dev *dev,
4686                                 uint32_t counter)
4687 {
4688         struct mlx5_flow_counter *cnt;
4689         struct mlx5_flow_counter_pool *pool = NULL;
4690
4691         flow_dv_counter_get_by_idx(dev, counter, &pool);
4692         counter = (counter - 1) % MLX5_COUNTERS_PER_POOL;
4693         cnt = MLX5_POOL_GET_CNT(pool, counter);
4694         return MLX5_CNT_TO_AGE(cnt);
4695 }
4696
4697 /**
4698  * Remove a flow counter from aged counter list.
4699  *
4700  * @param[in] dev
4701  *   Pointer to the Ethernet device structure.
4702  * @param[in] counter
4703  *   Index to the counter handler.
4704  * @param[in] cnt
4705  *   Pointer to the counter handler.
4706  */
4707 static void
4708 flow_dv_counter_remove_from_age(struct rte_eth_dev *dev,
4709                                 uint32_t counter, struct mlx5_flow_counter *cnt)
4710 {
4711         struct mlx5_age_info *age_info;
4712         struct mlx5_age_param *age_param;
4713         struct mlx5_priv *priv = dev->data->dev_private;
4714
4715         age_info = GET_PORT_AGE_INFO(priv);
4716         age_param = flow_dv_counter_idx_get_age(dev, counter);
4717         if (rte_atomic16_cmpset((volatile uint16_t *)
4718                         &age_param->state,
4719                         AGE_CANDIDATE, AGE_FREE)
4720                         != AGE_CANDIDATE) {
4721                 /**
4722                  * We need the lock even it is age timeout,
4723                  * since counter may still in process.
4724                  */
4725                 rte_spinlock_lock(&age_info->aged_sl);
4726                 TAILQ_REMOVE(&age_info->aged_counters, cnt, next);
4727                 rte_spinlock_unlock(&age_info->aged_sl);
4728         }
4729         rte_atomic16_set(&age_param->state, AGE_FREE);
4730 }
4731 /**
4732  * Release a flow counter.
4733  *
4734  * @param[in] dev
4735  *   Pointer to the Ethernet device structure.
4736  * @param[in] counter
4737  *   Index to the counter handler.
4738  */
4739 static void
4740 flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
4741 {
4742         struct mlx5_priv *priv = dev->data->dev_private;
4743         struct mlx5_flow_counter_pool *pool = NULL;
4744         struct mlx5_flow_counter *cnt;
4745         struct mlx5_flow_counter_ext *cnt_ext = NULL;
4746
4747         if (!counter)
4748                 return;
4749         cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
4750         MLX5_ASSERT(pool);
4751         if (counter < MLX5_CNT_BATCH_OFFSET) {
4752                 cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
4753                 if (cnt_ext) {
4754                         if (--cnt_ext->ref_cnt)
4755                                 return;
4756                         if (cnt_ext->shared)
4757                                 mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl,
4758                                                      cnt_ext->id);
4759                 }
4760         }
4761         if (IS_AGE_POOL(pool))
4762                 flow_dv_counter_remove_from_age(dev, counter, cnt);
4763         cnt->pool = pool;
4764         /*
4765          * Put the counter back to list to be updated in none fallback mode.
4766          * Currently, we are using two list alternately, while one is in query,
4767          * add the freed counter to the other list based on the pool query_gen
4768          * value. After query finishes, add counter the list to the global
4769          * container counter list. The list changes while query starts. In
4770          * this case, lock will not be needed as query callback and release
4771          * function both operate with the different list.
4772          *
4773          */
4774         if (!priv->counter_fallback)
4775                 TAILQ_INSERT_TAIL(&pool->counters[pool->query_gen], cnt, next);
4776         else
4777                 TAILQ_INSERT_TAIL(&((MLX5_CNT_CONTAINER
4778                                   (priv->sh, 0, 0))->counters),
4779                                   cnt, next);
4780 }
4781
4782 /**
4783  * Verify the @p attributes will be correctly understood by the NIC and store
4784  * them in the @p flow if everything is correct.
4785  *
4786  * @param[in] dev
4787  *   Pointer to dev struct.
4788  * @param[in] attributes
4789  *   Pointer to flow attributes
4790  * @param[in] external
4791  *   This flow rule is created by request external to PMD.
4792  * @param[out] error
4793  *   Pointer to error structure.
4794  *
4795  * @return
4796  *   - 0 on success and non root table.
4797  *   - 1 on success and root table.
4798  *   - a negative errno value otherwise and rte_errno is set.
4799  */
4800 static int
4801 flow_dv_validate_attributes(struct rte_eth_dev *dev,
4802                             const struct rte_flow_attr *attributes,
4803                             bool external __rte_unused,
4804                             struct rte_flow_error *error)
4805 {
4806         struct mlx5_priv *priv = dev->data->dev_private;
4807         uint32_t priority_max = priv->config.flow_prio - 1;
4808         int ret = 0;
4809
4810 #ifndef HAVE_MLX5DV_DR
4811         if (attributes->group)
4812                 return rte_flow_error_set(error, ENOTSUP,
4813                                           RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
4814                                           NULL,
4815                                           "groups are not supported");
4816 #else
4817         uint32_t table = 0;
4818
4819         ret = mlx5_flow_group_to_table(attributes, external,
4820                                        attributes->group, !!priv->fdb_def_rule,
4821                                        &table, error);
4822         if (ret)
4823                 return ret;
4824         if (!table)
4825                 ret = MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
4826 #endif
4827         if (attributes->priority != MLX5_FLOW_PRIO_RSVD &&
4828             attributes->priority >= priority_max)
4829                 return rte_flow_error_set(error, ENOTSUP,
4830                                           RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
4831                                           NULL,
4832                                           "priority out of range");
4833         if (attributes->transfer) {
4834                 if (!priv->config.dv_esw_en)
4835                         return rte_flow_error_set
4836                                 (error, ENOTSUP,
4837                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
4838                                  "E-Switch dr is not supported");
4839                 if (!(priv->representor || priv->master))
4840                         return rte_flow_error_set
4841                                 (error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
4842                                  NULL, "E-Switch configuration can only be"
4843                                  " done by a master or a representor device");
4844                 if (attributes->egress)
4845                         return rte_flow_error_set
4846                                 (error, ENOTSUP,
4847                                  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, attributes,
4848                                  "egress is not supported");
4849         }
4850         if (!(attributes->egress ^ attributes->ingress))
4851                 return rte_flow_error_set(error, ENOTSUP,
4852                                           RTE_FLOW_ERROR_TYPE_ATTR, NULL,
4853                                           "must specify exactly one of "
4854                                           "ingress or egress");
4855         return ret;
4856 }
4857
4858 /**
4859  * Internal validation function. For validating both actions and items.
4860  *
4861  * @param[in] dev
4862  *   Pointer to the rte_eth_dev structure.
4863  * @param[in] attr
4864  *   Pointer to the flow attributes.
4865  * @param[in] items
4866  *   Pointer to the list of items.
4867  * @param[in] actions
4868  *   Pointer to the list of actions.
4869  * @param[in] external
4870  *   This flow rule is created by request external to PMD.
4871  * @param[in] hairpin
4872  *   Number of hairpin TX actions, 0 means classic flow.
4873  * @param[out] error
4874  *   Pointer to the error structure.
4875  *
4876  * @return
4877  *   0 on success, a negative errno value otherwise and rte_errno is set.
4878  */
4879 static int
4880 flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
4881                  const struct rte_flow_item items[],
4882                  const struct rte_flow_action actions[],
4883                  bool external, int hairpin, struct rte_flow_error *error)
4884 {
4885         int ret;
4886         uint64_t action_flags = 0;
4887         uint64_t item_flags = 0;
4888         uint64_t last_item = 0;
4889         uint8_t next_protocol = 0xff;
4890         uint16_t ether_type = 0;
4891         int actions_n = 0;
4892         uint8_t item_ipv6_proto = 0;
4893         const struct rte_flow_item *gre_item = NULL;
4894         const struct rte_flow_action_raw_decap *decap;
4895         const struct rte_flow_action_raw_encap *encap;
4896         const struct rte_flow_action_rss *rss;
4897         const struct rte_flow_item_tcp nic_tcp_mask = {
4898                 .hdr = {
4899                         .tcp_flags = 0xFF,
4900                         .src_port = RTE_BE16(UINT16_MAX),
4901                         .dst_port = RTE_BE16(UINT16_MAX),
4902                 }
4903         };
4904         const struct rte_flow_item_ipv4 nic_ipv4_mask = {
4905                 .hdr = {
4906                         .src_addr = RTE_BE32(0xffffffff),
4907                         .dst_addr = RTE_BE32(0xffffffff),
4908                         .type_of_service = 0xff,
4909                         .next_proto_id = 0xff,
4910                         .time_to_live = 0xff,
4911                 },
4912         };
4913         const struct rte_flow_item_ipv6 nic_ipv6_mask = {
4914                 .hdr = {
4915                         .src_addr =
4916                         "\xff\xff\xff\xff\xff\xff\xff\xff"
4917                         "\xff\xff\xff\xff\xff\xff\xff\xff",
4918                         .dst_addr =
4919                         "\xff\xff\xff\xff\xff\xff\xff\xff"
4920                         "\xff\xff\xff\xff\xff\xff\xff\xff",
4921                         .vtc_flow = RTE_BE32(0xffffffff),
4922                         .proto = 0xff,
4923                         .hop_limits = 0xff,
4924                 },
4925         };
4926         struct mlx5_priv *priv = dev->data->dev_private;
4927         struct mlx5_dev_config *dev_conf = &priv->config;
4928         uint16_t queue_index = 0xFFFF;
4929         const struct rte_flow_item_vlan *vlan_m = NULL;
4930         int16_t rw_act_num = 0;
4931         uint64_t is_root;
4932
4933         if (items == NULL)
4934                 return -1;
4935         ret = flow_dv_validate_attributes(dev, attr, external, error);
4936         if (ret < 0)
4937                 return ret;
4938         is_root = (uint64_t)ret;
4939         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
4940                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
4941                 int type = items->type;
4942
4943                 if (!mlx5_flow_os_item_supported(type))
4944                         return rte_flow_error_set(error, ENOTSUP,
4945                                                   RTE_FLOW_ERROR_TYPE_ITEM,
4946                                                   NULL, "item not supported");
4947                 switch (type) {
4948                 case RTE_FLOW_ITEM_TYPE_VOID:
4949                         break;
4950                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
4951                         ret = flow_dv_validate_item_port_id
4952                                         (dev, items, attr, item_flags, error);
4953                         if (ret < 0)
4954                                 return ret;
4955                         last_item = MLX5_FLOW_ITEM_PORT_ID;
4956                         break;
4957                 case RTE_FLOW_ITEM_TYPE_ETH:
4958                         ret = mlx5_flow_validate_item_eth(items, item_flags,
4959                                                           error);
4960                         if (ret < 0)
4961                                 return ret;
4962                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
4963                                              MLX5_FLOW_LAYER_OUTER_L2;
4964                         if (items->mask != NULL && items->spec != NULL) {
4965                                 ether_type =
4966                                         ((const struct rte_flow_item_eth *)
4967                                          items->spec)->type;
4968                                 ether_type &=
4969                                         ((const struct rte_flow_item_eth *)
4970                                          items->mask)->type;
4971                                 ether_type = rte_be_to_cpu_16(ether_type);
4972                         } else {
4973                                 ether_type = 0;
4974                         }
4975                         break;
4976                 case RTE_FLOW_ITEM_TYPE_VLAN:
4977                         ret = flow_dv_validate_item_vlan(items, item_flags,
4978                                                          dev, error);
4979                         if (ret < 0)
4980                                 return ret;
4981                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_VLAN :
4982                                              MLX5_FLOW_LAYER_OUTER_VLAN;
4983                         if (items->mask != NULL && items->spec != NULL) {
4984                                 ether_type =
4985                                         ((const struct rte_flow_item_vlan *)
4986                                          items->spec)->inner_type;
4987                                 ether_type &=
4988                                         ((const struct rte_flow_item_vlan *)
4989                                          items->mask)->inner_type;
4990                                 ether_type = rte_be_to_cpu_16(ether_type);
4991                         } else {
4992                                 ether_type = 0;
4993                         }
4994                         /* Store outer VLAN mask for of_push_vlan action. */
4995                         if (!tunnel)
4996                                 vlan_m = items->mask;
4997                         break;
4998                 case RTE_FLOW_ITEM_TYPE_IPV4:
4999                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5000                                                   &item_flags, &tunnel);
5001                         ret = mlx5_flow_validate_item_ipv4(items, item_flags,
5002                                                            last_item,
5003                                                            ether_type,
5004                                                            &nic_ipv4_mask,
5005                                                            error);
5006                         if (ret < 0)
5007                                 return ret;
5008                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
5009                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
5010                         if (items->mask != NULL &&
5011                             ((const struct rte_flow_item_ipv4 *)
5012                              items->mask)->hdr.next_proto_id) {
5013                                 next_protocol =
5014                                         ((const struct rte_flow_item_ipv4 *)
5015                                          (items->spec))->hdr.next_proto_id;
5016                                 next_protocol &=
5017                                         ((const struct rte_flow_item_ipv4 *)
5018                                          (items->mask))->hdr.next_proto_id;
5019                         } else {
5020                                 /* Reset for inner layer. */
5021                                 next_protocol = 0xff;
5022                         }
5023                         break;
5024                 case RTE_FLOW_ITEM_TYPE_IPV6:
5025                         mlx5_flow_tunnel_ip_check(items, next_protocol,
5026                                                   &item_flags, &tunnel);
5027                         ret = mlx5_flow_validate_item_ipv6(items, item_flags,
5028                                                            last_item,
5029                                                            ether_type,
5030                                                            &nic_ipv6_mask,
5031                                                            error);
5032                         if (ret < 0)
5033                                 return ret;
5034                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
5035                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
5036                         if (items->mask != NULL &&
5037                             ((const struct rte_flow_item_ipv6 *)
5038                              items->mask)->hdr.proto) {
5039                                 item_ipv6_proto =
5040                                         ((const struct rte_flow_item_ipv6 *)
5041                                          items->spec)->hdr.proto;
5042                                 next_protocol =
5043                                         ((const struct rte_flow_item_ipv6 *)
5044                                          items->spec)->hdr.proto;
5045                                 next_protocol &=
5046                                         ((const struct rte_flow_item_ipv6 *)
5047                                          items->mask)->hdr.proto;
5048                         } else {
5049                                 /* Reset for inner layer. */
5050                                 next_protocol = 0xff;
5051                         }
5052                         break;
5053                 case RTE_FLOW_ITEM_TYPE_TCP:
5054                         ret = mlx5_flow_validate_item_tcp
5055                                                 (items, item_flags,
5056                                                  next_protocol,
5057                                                  &nic_tcp_mask,
5058                                                  error);
5059                         if (ret < 0)
5060                                 return ret;
5061                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
5062                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
5063                         break;
5064                 case RTE_FLOW_ITEM_TYPE_UDP:
5065                         ret = mlx5_flow_validate_item_udp(items, item_flags,
5066                                                           next_protocol,
5067                                                           error);
5068                         if (ret < 0)
5069                                 return ret;
5070                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
5071                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
5072                         break;
5073                 case RTE_FLOW_ITEM_TYPE_GRE:
5074                         ret = mlx5_flow_validate_item_gre(items, item_flags,
5075                                                           next_protocol, error);
5076                         if (ret < 0)
5077                                 return ret;
5078                         gre_item = items;
5079                         last_item = MLX5_FLOW_LAYER_GRE;
5080                         break;
5081                 case RTE_FLOW_ITEM_TYPE_NVGRE:
5082                         ret = mlx5_flow_validate_item_nvgre(items, item_flags,
5083                                                             next_protocol,
5084                                                             error);
5085                         if (ret < 0)
5086                                 return ret;
5087                         last_item = MLX5_FLOW_LAYER_NVGRE;
5088                         break;
5089                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
5090                         ret = mlx5_flow_validate_item_gre_key
5091                                 (items, item_flags, gre_item, error);
5092                         if (ret < 0)
5093                                 return ret;
5094                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
5095                         break;
5096                 case RTE_FLOW_ITEM_TYPE_VXLAN:
5097                         ret = mlx5_flow_validate_item_vxlan(items, item_flags,
5098                                                             error);
5099                         if (ret < 0)
5100                                 return ret;
5101                         last_item = MLX5_FLOW_LAYER_VXLAN;
5102                         break;
5103                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
5104                         ret = mlx5_flow_validate_item_vxlan_gpe(items,
5105                                                                 item_flags, dev,
5106                                                                 error);
5107                         if (ret < 0)
5108                                 return ret;
5109                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
5110                         break;
5111                 case RTE_FLOW_ITEM_TYPE_GENEVE:
5112                         ret = mlx5_flow_validate_item_geneve(items,
5113                                                              item_flags, dev,
5114                                                              error);
5115                         if (ret < 0)
5116                                 return ret;
5117                         last_item = MLX5_FLOW_LAYER_GENEVE;
5118                         break;
5119                 case RTE_FLOW_ITEM_TYPE_MPLS:
5120                         ret = mlx5_flow_validate_item_mpls(dev, items,
5121                                                            item_flags,
5122                                                            last_item, error);
5123                         if (ret < 0)
5124                                 return ret;
5125                         last_item = MLX5_FLOW_LAYER_MPLS;
5126                         break;
5127
5128                 case RTE_FLOW_ITEM_TYPE_MARK:
5129                         ret = flow_dv_validate_item_mark(dev, items, attr,
5130                                                          error);
5131                         if (ret < 0)
5132                                 return ret;
5133                         last_item = MLX5_FLOW_ITEM_MARK;
5134                         break;
5135                 case RTE_FLOW_ITEM_TYPE_META:
5136                         ret = flow_dv_validate_item_meta(dev, items, attr,
5137                                                          error);
5138                         if (ret < 0)
5139                                 return ret;
5140                         last_item = MLX5_FLOW_ITEM_METADATA;
5141                         break;
5142                 case RTE_FLOW_ITEM_TYPE_ICMP:
5143                         ret = mlx5_flow_validate_item_icmp(items, item_flags,
5144                                                            next_protocol,
5145                                                            error);
5146                         if (ret < 0)
5147                                 return ret;
5148                         last_item = MLX5_FLOW_LAYER_ICMP;
5149                         break;
5150                 case RTE_FLOW_ITEM_TYPE_ICMP6:
5151                         ret = mlx5_flow_validate_item_icmp6(items, item_flags,
5152                                                             next_protocol,
5153                                                             error);
5154                         if (ret < 0)
5155                                 return ret;
5156                         item_ipv6_proto = IPPROTO_ICMPV6;
5157                         last_item = MLX5_FLOW_LAYER_ICMP6;
5158                         break;
5159                 case RTE_FLOW_ITEM_TYPE_TAG:
5160                         ret = flow_dv_validate_item_tag(dev, items,
5161                                                         attr, error);
5162                         if (ret < 0)
5163                                 return ret;
5164                         last_item = MLX5_FLOW_ITEM_TAG;
5165                         break;
5166                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
5167                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
5168                         break;
5169                 case RTE_FLOW_ITEM_TYPE_GTP:
5170                         ret = flow_dv_validate_item_gtp(dev, items, item_flags,
5171                                                         error);
5172                         if (ret < 0)
5173                                 return ret;
5174                         last_item = MLX5_FLOW_LAYER_GTP;
5175                         break;
5176                 default:
5177                         return rte_flow_error_set(error, ENOTSUP,
5178                                                   RTE_FLOW_ERROR_TYPE_ITEM,
5179                                                   NULL, "item not supported");
5180                 }
5181                 item_flags |= last_item;
5182         }
5183         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
5184                 int type = actions->type;
5185
5186                 if (!mlx5_flow_os_action_supported(type))
5187                         return rte_flow_error_set(error, ENOTSUP,
5188                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5189                                                   actions,
5190                                                   "action not supported");
5191                 if (actions_n == MLX5_DV_MAX_NUMBER_OF_ACTIONS)
5192                         return rte_flow_error_set(error, ENOTSUP,
5193                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5194                                                   actions, "too many actions");
5195                 switch (type) {
5196                 case RTE_FLOW_ACTION_TYPE_VOID:
5197                         break;
5198                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
5199                         ret = flow_dv_validate_action_port_id(dev,
5200                                                               action_flags,
5201                                                               actions,
5202                                                               attr,
5203                                                               error);
5204                         if (ret)
5205                                 return ret;
5206                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
5207                         ++actions_n;
5208                         break;
5209                 case RTE_FLOW_ACTION_TYPE_FLAG:
5210                         ret = flow_dv_validate_action_flag(dev, action_flags,
5211                                                            attr, error);
5212                         if (ret < 0)
5213                                 return ret;
5214                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5215                                 /* Count all modify-header actions as one. */
5216                                 if (!(action_flags &
5217                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5218                                         ++actions_n;
5219                                 action_flags |= MLX5_FLOW_ACTION_FLAG |
5220                                                 MLX5_FLOW_ACTION_MARK_EXT;
5221                         } else {
5222                                 action_flags |= MLX5_FLOW_ACTION_FLAG;
5223                                 ++actions_n;
5224                         }
5225                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5226                         break;
5227                 case RTE_FLOW_ACTION_TYPE_MARK:
5228                         ret = flow_dv_validate_action_mark(dev, actions,
5229                                                            action_flags,
5230                                                            attr, error);
5231                         if (ret < 0)
5232                                 return ret;
5233                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
5234                                 /* Count all modify-header actions as one. */
5235                                 if (!(action_flags &
5236                                       MLX5_FLOW_MODIFY_HDR_ACTIONS))
5237                                         ++actions_n;
5238                                 action_flags |= MLX5_FLOW_ACTION_MARK |
5239                                                 MLX5_FLOW_ACTION_MARK_EXT;
5240                         } else {
5241                                 action_flags |= MLX5_FLOW_ACTION_MARK;
5242                                 ++actions_n;
5243                         }
5244                         rw_act_num += MLX5_ACT_NUM_SET_MARK;
5245                         break;
5246                 case RTE_FLOW_ACTION_TYPE_SET_META:
5247                         ret = flow_dv_validate_action_set_meta(dev, actions,
5248                                                                action_flags,
5249                                                                attr, error);
5250                         if (ret < 0)
5251                                 return ret;
5252                         /* Count all modify-header actions as one action. */
5253                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5254                                 ++actions_n;
5255                         action_flags |= MLX5_FLOW_ACTION_SET_META;
5256                         rw_act_num += MLX5_ACT_NUM_SET_META;
5257                         break;
5258                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
5259                         ret = flow_dv_validate_action_set_tag(dev, actions,
5260                                                               action_flags,
5261                                                               attr, error);
5262                         if (ret < 0)
5263                                 return ret;
5264                         /* Count all modify-header actions as one action. */
5265                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5266                                 ++actions_n;
5267                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
5268                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5269                         break;
5270                 case RTE_FLOW_ACTION_TYPE_DROP:
5271                         ret = mlx5_flow_validate_action_drop(action_flags,
5272                                                              attr, error);
5273                         if (ret < 0)
5274                                 return ret;
5275                         action_flags |= MLX5_FLOW_ACTION_DROP;
5276                         ++actions_n;
5277                         break;
5278                 case RTE_FLOW_ACTION_TYPE_QUEUE:
5279                         ret = mlx5_flow_validate_action_queue(actions,
5280                                                               action_flags, dev,
5281                                                               attr, error);
5282                         if (ret < 0)
5283                                 return ret;
5284                         queue_index = ((const struct rte_flow_action_queue *)
5285                                                         (actions->conf))->index;
5286                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
5287                         ++actions_n;
5288                         break;
5289                 case RTE_FLOW_ACTION_TYPE_RSS:
5290                         rss = actions->conf;
5291                         ret = mlx5_flow_validate_action_rss(actions,
5292                                                             action_flags, dev,
5293                                                             attr, item_flags,
5294                                                             error);
5295                         if (ret < 0)
5296                                 return ret;
5297                         if (rss != NULL && rss->queue_num)
5298                                 queue_index = rss->queue[0];
5299                         action_flags |= MLX5_FLOW_ACTION_RSS;
5300                         ++actions_n;
5301                         break;
5302                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
5303                         ret =
5304                         mlx5_flow_validate_action_default_miss(action_flags,
5305                                         attr, error);
5306                         if (ret < 0)
5307                                 return ret;
5308                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
5309                         ++actions_n;
5310                         break;
5311                 case RTE_FLOW_ACTION_TYPE_COUNT:
5312                         ret = flow_dv_validate_action_count(dev, error);
5313                         if (ret < 0)
5314                                 return ret;
5315                         action_flags |= MLX5_FLOW_ACTION_COUNT;
5316                         ++actions_n;
5317                         break;
5318                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
5319                         if (flow_dv_validate_action_pop_vlan(dev,
5320                                                              action_flags,
5321                                                              actions,
5322                                                              item_flags, attr,
5323                                                              error))
5324                                 return -rte_errno;
5325                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
5326                         ++actions_n;
5327                         break;
5328                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
5329                         ret = flow_dv_validate_action_push_vlan(dev,
5330                                                                 action_flags,
5331                                                                 vlan_m,
5332                                                                 actions, attr,
5333                                                                 error);
5334                         if (ret < 0)
5335                                 return ret;
5336                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
5337                         ++actions_n;
5338                         break;
5339                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
5340                         ret = flow_dv_validate_action_set_vlan_pcp
5341                                                 (action_flags, actions, error);
5342                         if (ret < 0)
5343                                 return ret;
5344                         /* Count PCP with push_vlan command. */
5345                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_PCP;
5346                         break;
5347                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
5348                         ret = flow_dv_validate_action_set_vlan_vid
5349                                                 (item_flags, action_flags,
5350                                                  actions, error);
5351                         if (ret < 0)
5352                                 return ret;
5353                         /* Count VID with push_vlan command. */
5354                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
5355                         rw_act_num += MLX5_ACT_NUM_MDF_VID;
5356                         break;
5357                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
5358                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
5359                         ret = flow_dv_validate_action_l2_encap(dev,
5360                                                                action_flags,
5361                                                                actions, attr,
5362                                                                error);
5363                         if (ret < 0)
5364                                 return ret;
5365                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
5366                         ++actions_n;
5367                         break;
5368                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
5369                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
5370                         ret = flow_dv_validate_action_decap(dev, action_flags,
5371                                                             attr, error);
5372                         if (ret < 0)
5373                                 return ret;
5374                         action_flags |= MLX5_FLOW_ACTION_DECAP;
5375                         ++actions_n;
5376                         break;
5377                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
5378                         ret = flow_dv_validate_action_raw_encap_decap
5379                                 (dev, NULL, actions->conf, attr, &action_flags,
5380                                  &actions_n, error);
5381                         if (ret < 0)
5382                                 return ret;
5383                         break;
5384                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
5385                         decap = actions->conf;
5386                         while ((++actions)->type == RTE_FLOW_ACTION_TYPE_VOID)
5387                                 ;
5388                         if (actions->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
5389                                 encap = NULL;
5390                                 actions--;
5391                         } else {
5392                                 encap = actions->conf;
5393                         }
5394                         ret = flow_dv_validate_action_raw_encap_decap
5395                                            (dev,
5396                                             decap ? decap : &empty_decap, encap,
5397                                             attr, &action_flags, &actions_n,
5398                                             error);
5399                         if (ret < 0)
5400                                 return ret;
5401                         break;
5402                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
5403                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
5404                         ret = flow_dv_validate_action_modify_mac(action_flags,
5405                                                                  actions,
5406                                                                  item_flags,
5407                                                                  error);
5408                         if (ret < 0)
5409                                 return ret;
5410                         /* Count all modify-header actions as one action. */
5411                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5412                                 ++actions_n;
5413                         action_flags |= actions->type ==
5414                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
5415                                                 MLX5_FLOW_ACTION_SET_MAC_SRC :
5416                                                 MLX5_FLOW_ACTION_SET_MAC_DST;
5417                         /*
5418                          * Even if the source and destination MAC addresses have
5419                          * overlap in the header with 4B alignment, the convert
5420                          * function will handle them separately and 4 SW actions
5421                          * will be created. And 2 actions will be added each
5422                          * time no matter how many bytes of address will be set.
5423                          */
5424                         rw_act_num += MLX5_ACT_NUM_MDF_MAC;
5425                         break;
5426                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
5427                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
5428                         ret = flow_dv_validate_action_modify_ipv4(action_flags,
5429                                                                   actions,
5430                                                                   item_flags,
5431                                                                   error);
5432                         if (ret < 0)
5433                                 return ret;
5434                         /* Count all modify-header actions as one action. */
5435                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5436                                 ++actions_n;
5437                         action_flags |= actions->type ==
5438                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
5439                                                 MLX5_FLOW_ACTION_SET_IPV4_SRC :
5440                                                 MLX5_FLOW_ACTION_SET_IPV4_DST;
5441                         rw_act_num += MLX5_ACT_NUM_MDF_IPV4;
5442                         break;
5443                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
5444                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
5445                         ret = flow_dv_validate_action_modify_ipv6(action_flags,
5446                                                                   actions,
5447                                                                   item_flags,
5448                                                                   error);
5449                         if (ret < 0)
5450                                 return ret;
5451                         if (item_ipv6_proto == IPPROTO_ICMPV6)
5452                                 return rte_flow_error_set(error, ENOTSUP,
5453                                         RTE_FLOW_ERROR_TYPE_ACTION,
5454                                         actions,
5455                                         "Can't change header "
5456                                         "with ICMPv6 proto");
5457                         /* Count all modify-header actions as one action. */
5458                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5459                                 ++actions_n;
5460                         action_flags |= actions->type ==
5461                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
5462                                                 MLX5_FLOW_ACTION_SET_IPV6_SRC :
5463                                                 MLX5_FLOW_ACTION_SET_IPV6_DST;
5464                         rw_act_num += MLX5_ACT_NUM_MDF_IPV6;
5465                         break;
5466                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
5467                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
5468                         ret = flow_dv_validate_action_modify_tp(action_flags,
5469                                                                 actions,
5470                                                                 item_flags,
5471                                                                 error);
5472                         if (ret < 0)
5473                                 return ret;
5474                         /* Count all modify-header actions as one action. */
5475                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5476                                 ++actions_n;
5477                         action_flags |= actions->type ==
5478                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
5479                                                 MLX5_FLOW_ACTION_SET_TP_SRC :
5480                                                 MLX5_FLOW_ACTION_SET_TP_DST;
5481                         rw_act_num += MLX5_ACT_NUM_MDF_PORT;
5482                         break;
5483                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
5484                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
5485                         ret = flow_dv_validate_action_modify_ttl(action_flags,
5486                                                                  actions,
5487                                                                  item_flags,
5488                                                                  error);
5489                         if (ret < 0)
5490                                 return ret;
5491                         /* Count all modify-header actions as one action. */
5492                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5493                                 ++actions_n;
5494                         action_flags |= actions->type ==
5495                                         RTE_FLOW_ACTION_TYPE_SET_TTL ?
5496                                                 MLX5_FLOW_ACTION_SET_TTL :
5497                                                 MLX5_FLOW_ACTION_DEC_TTL;
5498                         rw_act_num += MLX5_ACT_NUM_MDF_TTL;
5499                         break;
5500                 case RTE_FLOW_ACTION_TYPE_JUMP:
5501                         ret = flow_dv_validate_action_jump(actions,
5502                                                            action_flags,
5503                                                            attr, external,
5504                                                            error);
5505                         if (ret)
5506                                 return ret;
5507                         ++actions_n;
5508                         action_flags |= MLX5_FLOW_ACTION_JUMP;
5509                         break;
5510                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
5511                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
5512                         ret = flow_dv_validate_action_modify_tcp_seq
5513                                                                 (action_flags,
5514                                                                  actions,
5515                                                                  item_flags,
5516                                                                  error);
5517                         if (ret < 0)
5518                                 return ret;
5519                         /* Count all modify-header actions as one action. */
5520                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5521                                 ++actions_n;
5522                         action_flags |= actions->type ==
5523                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
5524                                                 MLX5_FLOW_ACTION_INC_TCP_SEQ :
5525                                                 MLX5_FLOW_ACTION_DEC_TCP_SEQ;
5526                         rw_act_num += MLX5_ACT_NUM_MDF_TCPSEQ;
5527                         break;
5528                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
5529                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
5530                         ret = flow_dv_validate_action_modify_tcp_ack
5531                                                                 (action_flags,
5532                                                                  actions,
5533                                                                  item_flags,
5534                                                                  error);
5535                         if (ret < 0)
5536                                 return ret;
5537                         /* Count all modify-header actions as one action. */
5538                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5539                                 ++actions_n;
5540                         action_flags |= actions->type ==
5541                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
5542                                                 MLX5_FLOW_ACTION_INC_TCP_ACK :
5543                                                 MLX5_FLOW_ACTION_DEC_TCP_ACK;
5544                         rw_act_num += MLX5_ACT_NUM_MDF_TCPACK;
5545                         break;
5546                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
5547                         break;
5548                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
5549                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
5550                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5551                         break;
5552                 case RTE_FLOW_ACTION_TYPE_METER:
5553                         ret = mlx5_flow_validate_action_meter(dev,
5554                                                               action_flags,
5555                                                               actions, attr,
5556                                                               error);
5557                         if (ret < 0)
5558                                 return ret;
5559                         action_flags |= MLX5_FLOW_ACTION_METER;
5560                         ++actions_n;
5561                         /* Meter action will add one more TAG action. */
5562                         rw_act_num += MLX5_ACT_NUM_SET_TAG;
5563                         break;
5564                 case RTE_FLOW_ACTION_TYPE_AGE:
5565                         ret = flow_dv_validate_action_age(action_flags,
5566                                                           actions, dev,
5567                                                           error);
5568                         if (ret < 0)
5569                                 return ret;
5570                         action_flags |= MLX5_FLOW_ACTION_AGE;
5571                         ++actions_n;
5572                         break;
5573                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
5574                         ret = flow_dv_validate_action_modify_ipv4_dscp
5575                                                          (action_flags,
5576                                                           actions,
5577                                                           item_flags,
5578                                                           error);
5579                         if (ret < 0)
5580                                 return ret;
5581                         /* Count all modify-header actions as one action. */
5582                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5583                                 ++actions_n;
5584                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
5585                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5586                         break;
5587                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
5588                         ret = flow_dv_validate_action_modify_ipv6_dscp
5589                                                                 (action_flags,
5590                                                                  actions,
5591                                                                  item_flags,
5592                                                                  error);
5593                         if (ret < 0)
5594                                 return ret;
5595                         /* Count all modify-header actions as one action. */
5596                         if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
5597                                 ++actions_n;
5598                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
5599                         rw_act_num += MLX5_ACT_NUM_SET_DSCP;
5600                         break;
5601                 default:
5602                         return rte_flow_error_set(error, ENOTSUP,
5603                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5604                                                   actions,
5605                                                   "action not supported");
5606                 }
5607         }
5608         /*
5609          * Validate the drop action mutual exclusion with other actions.
5610          * Drop action is mutually-exclusive with any other action, except for
5611          * Count action.
5612          */
5613         if ((action_flags & MLX5_FLOW_ACTION_DROP) &&
5614             (action_flags & ~(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_COUNT)))
5615                 return rte_flow_error_set(error, EINVAL,
5616                                           RTE_FLOW_ERROR_TYPE_ACTION, NULL,
5617                                           "Drop action is mutually-exclusive "
5618                                           "with any other action, except for "
5619                                           "Count action");
5620         /* Eswitch has few restrictions on using items and actions */
5621         if (attr->transfer) {
5622                 if (!mlx5_flow_ext_mreg_supported(dev) &&
5623                     action_flags & MLX5_FLOW_ACTION_FLAG)
5624                         return rte_flow_error_set(error, ENOTSUP,
5625                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5626                                                   NULL,
5627                                                   "unsupported action FLAG");
5628                 if (!mlx5_flow_ext_mreg_supported(dev) &&
5629                     action_flags & MLX5_FLOW_ACTION_MARK)
5630                         return rte_flow_error_set(error, ENOTSUP,
5631                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5632                                                   NULL,
5633                                                   "unsupported action MARK");
5634                 if (action_flags & MLX5_FLOW_ACTION_QUEUE)
5635                         return rte_flow_error_set(error, ENOTSUP,
5636                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5637                                                   NULL,
5638                                                   "unsupported action QUEUE");
5639                 if (action_flags & MLX5_FLOW_ACTION_RSS)
5640                         return rte_flow_error_set(error, ENOTSUP,
5641                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5642                                                   NULL,
5643                                                   "unsupported action RSS");
5644                 if (!(action_flags & MLX5_FLOW_FATE_ESWITCH_ACTIONS))
5645                         return rte_flow_error_set(error, EINVAL,
5646                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5647                                                   actions,
5648                                                   "no fate action is found");
5649         } else {
5650                 if (!(action_flags & MLX5_FLOW_FATE_ACTIONS) && attr->ingress)
5651                         return rte_flow_error_set(error, EINVAL,
5652                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5653                                                   actions,
5654                                                   "no fate action is found");
5655         }
5656         /* Continue validation for Xcap actions.*/
5657         if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) && (queue_index == 0xFFFF ||
5658             mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
5659                 if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
5660                     MLX5_FLOW_XCAP_ACTIONS)
5661                         return rte_flow_error_set(error, ENOTSUP,
5662                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5663                                                   NULL, "encap and decap "
5664                                                   "combination aren't supported");
5665                 if (!attr->transfer && attr->ingress && (action_flags &
5666                                                         MLX5_FLOW_ACTION_ENCAP))
5667                         return rte_flow_error_set(error, ENOTSUP,
5668                                                   RTE_FLOW_ERROR_TYPE_ACTION,
5669                                                   NULL, "encap is not supported"
5670                                                   " for ingress traffic");
5671         }
5672         /* Hairpin flow will add one more TAG action. */
5673         if (hairpin > 0)
5674                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5675         /* extra metadata enabled: one more TAG action will be add. */
5676         if (dev_conf->dv_flow_en &&
5677             dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
5678             mlx5_flow_ext_mreg_supported(dev))
5679                 rw_act_num += MLX5_ACT_NUM_SET_TAG;
5680         if ((uint32_t)rw_act_num >
5681                         flow_dv_modify_hdr_action_max(dev, is_root)) {
5682                 return rte_flow_error_set(error, ENOTSUP,
5683                                           RTE_FLOW_ERROR_TYPE_ACTION,
5684                                           NULL, "too many header modify"
5685                                           " actions to support");
5686         }
5687         return 0;
5688 }
5689
5690 /**
5691  * Internal preparation function. Allocates the DV flow size,
5692  * this size is constant.
5693  *
5694  * @param[in] dev
5695  *   Pointer to the rte_eth_dev structure.
5696  * @param[in] attr
5697  *   Pointer to the flow attributes.
5698  * @param[in] items
5699  *   Pointer to the list of items.
5700  * @param[in] actions
5701  *   Pointer to the list of actions.
5702  * @param[out] error
5703  *   Pointer to the error structure.
5704  *
5705  * @return
5706  *   Pointer to mlx5_flow object on success,
5707  *   otherwise NULL and rte_errno is set.
5708  */
5709 static struct mlx5_flow *
5710 flow_dv_prepare(struct rte_eth_dev *dev,
5711                 const struct rte_flow_attr *attr __rte_unused,
5712                 const struct rte_flow_item items[] __rte_unused,
5713                 const struct rte_flow_action actions[] __rte_unused,
5714                 struct rte_flow_error *error)
5715 {
5716         uint32_t handle_idx = 0;
5717         struct mlx5_flow *dev_flow;
5718         struct mlx5_flow_handle *dev_handle;
5719         struct mlx5_priv *priv = dev->data->dev_private;
5720
5721         /* In case of corrupting the memory. */
5722         if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
5723                 rte_flow_error_set(error, ENOSPC,
5724                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5725                                    "not free temporary device flow");
5726                 return NULL;
5727         }
5728         dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
5729                                    &handle_idx);
5730         if (!dev_handle) {
5731                 rte_flow_error_set(error, ENOMEM,
5732                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
5733                                    "not enough memory to create flow handle");
5734                 return NULL;
5735         }
5736         /* No multi-thread supporting. */
5737         dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
5738         dev_flow->handle = dev_handle;
5739         dev_flow->handle_idx = handle_idx;
5740         dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
5741         /*
5742          * The matching value needs to be cleared to 0 before using. In the
5743          * past, it will be automatically cleared when using rte_*alloc
5744          * API. The time consumption will be almost the same as before.
5745          */
5746         memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
5747         dev_flow->ingress = attr->ingress;
5748         dev_flow->dv.transfer = attr->transfer;
5749         return dev_flow;
5750 }
5751
5752 #ifdef RTE_LIBRTE_MLX5_DEBUG
5753 /**
5754  * Sanity check for match mask and value. Similar to check_valid_spec() in
5755  * kernel driver. If unmasked bit is present in value, it returns failure.
5756  *
5757  * @param match_mask
5758  *   pointer to match mask buffer.
5759  * @param match_value
5760  *   pointer to match value buffer.
5761  *
5762  * @return
5763  *   0 if valid, -EINVAL otherwise.
5764  */
5765 static int
5766 flow_dv_check_valid_spec(void *match_mask, void *match_value)
5767 {
5768         uint8_t *m = match_mask;
5769         uint8_t *v = match_value;
5770         unsigned int i;
5771
5772         for (i = 0; i < MLX5_ST_SZ_BYTES(fte_match_param); ++i) {
5773                 if (v[i] & ~m[i]) {
5774                         DRV_LOG(ERR,
5775                                 "match_value differs from match_criteria"
5776                                 " %p[%u] != %p[%u]",
5777                                 match_value, i, match_mask, i);
5778                         return -EINVAL;
5779                 }
5780         }
5781         return 0;
5782 }
5783 #endif
5784
5785 /**
5786  * Add match of ip_version.
5787  *
5788  * @param[in] group
5789  *   Flow group.
5790  * @param[in] headers_v
5791  *   Values header pointer.
5792  * @param[in] headers_m
5793  *   Masks header pointer.
5794  * @param[in] ip_version
5795  *   The IP version to set.
5796  */
5797 static inline void
5798 flow_dv_set_match_ip_version(uint32_t group,
5799                              void *headers_v,
5800                              void *headers_m,
5801                              uint8_t ip_version)
5802 {
5803         if (group == 0)
5804                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
5805         else
5806                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
5807                          ip_version);
5808         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
5809         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
5810         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
5811 }
5812
5813 /**
5814  * Add Ethernet item to matcher and to the value.
5815  *
5816  * @param[in, out] matcher
5817  *   Flow matcher.
5818  * @param[in, out] key
5819  *   Flow matcher value.
5820  * @param[in] item
5821  *   Flow pattern to translate.
5822  * @param[in] inner
5823  *   Item is inner pattern.
5824  */
5825 static void
5826 flow_dv_translate_item_eth(void *matcher, void *key,
5827                            const struct rte_flow_item *item, int inner,
5828                            uint32_t group)
5829 {
5830         const struct rte_flow_item_eth *eth_m = item->mask;
5831         const struct rte_flow_item_eth *eth_v = item->spec;
5832         const struct rte_flow_item_eth nic_mask = {
5833                 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
5834                 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
5835                 .type = RTE_BE16(0xffff),
5836         };
5837         void *headers_m;
5838         void *headers_v;
5839         char *l24_v;
5840         unsigned int i;
5841
5842         if (!eth_v)
5843                 return;
5844         if (!eth_m)
5845                 eth_m = &nic_mask;
5846         if (inner) {
5847                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5848                                          inner_headers);
5849                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5850         } else {
5851                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5852                                          outer_headers);
5853                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5854         }
5855         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, dmac_47_16),
5856                &eth_m->dst, sizeof(eth_m->dst));
5857         /* The value must be in the range of the mask. */
5858         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, dmac_47_16);
5859         for (i = 0; i < sizeof(eth_m->dst); ++i)
5860                 l24_v[i] = eth_m->dst.addr_bytes[i] & eth_v->dst.addr_bytes[i];
5861         memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m, smac_47_16),
5862                &eth_m->src, sizeof(eth_m->src));
5863         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, smac_47_16);
5864         /* The value must be in the range of the mask. */
5865         for (i = 0; i < sizeof(eth_m->dst); ++i)
5866                 l24_v[i] = eth_m->src.addr_bytes[i] & eth_v->src.addr_bytes[i];
5867         if (eth_v->type) {
5868                 /* When ethertype is present set mask for tagged VLAN. */
5869                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
5870                 /* Set value for tagged VLAN if ethertype is 802.1Q. */
5871                 if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
5872                     eth_v->type == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
5873                         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag,
5874                                  1);
5875                         /* Return here to avoid setting match on ethertype. */
5876                         return;
5877                 }
5878         }
5879         /*
5880          * HW supports match on one Ethertype, the Ethertype following the last
5881          * VLAN tag of the packet (see PRM).
5882          * Set match on ethertype only if ETH header is not followed by VLAN.
5883          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
5884          * ethertype, and use ip_version field instead.
5885          */
5886         if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
5887             eth_m->type == 0xFFFF) {
5888                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
5889         } else if (eth_v->type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
5890                    eth_m->type == 0xFFFF) {
5891                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
5892         } else {
5893                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
5894                          rte_be_to_cpu_16(eth_m->type));
5895                 l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
5896                                      ethertype);
5897                 *(uint16_t *)(l24_v) = eth_m->type & eth_v->type;
5898         }
5899 }
5900
5901 /**
5902  * Add VLAN item to matcher and to the value.
5903  *
5904  * @param[in, out] dev_flow
5905  *   Flow descriptor.
5906  * @param[in, out] matcher
5907  *   Flow matcher.
5908  * @param[in, out] key
5909  *   Flow matcher value.
5910  * @param[in] item
5911  *   Flow pattern to translate.
5912  * @param[in] inner
5913  *   Item is inner pattern.
5914  */
5915 static void
5916 flow_dv_translate_item_vlan(struct mlx5_flow *dev_flow,
5917                             void *matcher, void *key,
5918                             const struct rte_flow_item *item,
5919                             int inner, uint32_t group)
5920 {
5921         const struct rte_flow_item_vlan *vlan_m = item->mask;
5922         const struct rte_flow_item_vlan *vlan_v = item->spec;
5923         void *headers_m;
5924         void *headers_v;
5925         uint16_t tci_m;
5926         uint16_t tci_v;
5927
5928         if (inner) {
5929                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5930                                          inner_headers);
5931                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
5932         } else {
5933                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
5934                                          outer_headers);
5935                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
5936                 /*
5937                  * This is workaround, masks are not supported,
5938                  * and pre-validated.
5939                  */
5940                 if (vlan_v)
5941                         dev_flow->handle->vf_vlan.tag =
5942                                         rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
5943         }
5944         /*
5945          * When VLAN item exists in flow, mark packet as tagged,
5946          * even if TCI is not specified.
5947          */
5948         MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
5949         MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1);
5950         if (!vlan_v)
5951                 return;
5952         if (!vlan_m)
5953                 vlan_m = &rte_flow_item_vlan_mask;
5954         tci_m = rte_be_to_cpu_16(vlan_m->tci);
5955         tci_v = rte_be_to_cpu_16(vlan_m->tci & vlan_v->tci);
5956         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_vid, tci_m);
5957         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, tci_v);
5958         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_cfi, tci_m >> 12);
5959         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_cfi, tci_v >> 12);
5960         MLX5_SET(fte_match_set_lyr_2_4, headers_m, first_prio, tci_m >> 13);
5961         MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, tci_v >> 13);
5962         /*
5963          * HW is optimized for IPv4/IPv6. In such cases, avoid setting
5964          * ethertype, and use ip_version field instead.
5965          */
5966         if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV4) &&
5967             vlan_m->inner_type == 0xFFFF) {
5968                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
5969         } else if (vlan_v->inner_type == RTE_BE16(RTE_ETHER_TYPE_IPV6) &&
5970                    vlan_m->inner_type == 0xFFFF) {
5971                 flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
5972         } else {
5973                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype,
5974                          rte_be_to_cpu_16(vlan_m->inner_type));
5975                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
5976                          rte_be_to_cpu_16(vlan_m->inner_type &
5977                                           vlan_v->inner_type));
5978         }
5979 }
5980
5981 /**
5982  * Add IPV4 item to matcher and to the value.
5983  *
5984  * @param[in, out] matcher
5985  *   Flow matcher.
5986  * @param[in, out] key
5987  *   Flow matcher value.
5988  * @param[in] item
5989  *   Flow pattern to translate.
5990  * @param[in] item_flags
5991  *   Bit-fields that holds the items detected until now.
5992  * @param[in] inner
5993  *   Item is inner pattern.
5994  * @param[in] group
5995  *   The group to insert the rule.
5996  */
5997 static void
5998 flow_dv_translate_item_ipv4(void *matcher, void *key,
5999                             const struct rte_flow_item *item,
6000                             const uint64_t item_flags,
6001                             int inner, uint32_t group)
6002 {
6003         const struct rte_flow_item_ipv4 *ipv4_m = item->mask;
6004         const struct rte_flow_item_ipv4 *ipv4_v = item->spec;
6005         const struct rte_flow_item_ipv4 nic_mask = {
6006                 .hdr = {
6007                         .src_addr = RTE_BE32(0xffffffff),
6008                         .dst_addr = RTE_BE32(0xffffffff),
6009                         .type_of_service = 0xff,
6010                         .next_proto_id = 0xff,
6011                         .time_to_live = 0xff,
6012                 },
6013         };
6014         void *headers_m;
6015         void *headers_v;
6016         char *l24_m;
6017         char *l24_v;
6018         uint8_t tos;
6019
6020         if (inner) {
6021                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6022                                          inner_headers);
6023                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6024         } else {
6025                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6026                                          outer_headers);
6027                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6028         }
6029         flow_dv_set_match_ip_version(group, headers_v, headers_m, 4);
6030         /*
6031          * On outer header (which must contains L2), or inner header with L2,
6032          * set cvlan_tag mask bit to mark this packet as untagged.
6033          * This should be done even if item->spec is empty.
6034          */
6035         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6036                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6037         if (!ipv4_v)
6038                 return;
6039         if (!ipv4_m)
6040                 ipv4_m = &nic_mask;
6041         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6042                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6043         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6044                              dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
6045         *(uint32_t *)l24_m = ipv4_m->hdr.dst_addr;
6046         *(uint32_t *)l24_v = ipv4_m->hdr.dst_addr & ipv4_v->hdr.dst_addr;
6047         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6048                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6049         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6050                           src_ipv4_src_ipv6.ipv4_layout.ipv4);
6051         *(uint32_t *)l24_m = ipv4_m->hdr.src_addr;
6052         *(uint32_t *)l24_v = ipv4_m->hdr.src_addr & ipv4_v->hdr.src_addr;
6053         tos = ipv4_m->hdr.type_of_service & ipv4_v->hdr.type_of_service;
6054         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn,
6055                  ipv4_m->hdr.type_of_service);
6056         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, tos);
6057         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp,
6058                  ipv4_m->hdr.type_of_service >> 2);
6059         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, tos >> 2);
6060         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6061                  ipv4_m->hdr.next_proto_id);
6062         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6063                  ipv4_v->hdr.next_proto_id & ipv4_m->hdr.next_proto_id);
6064         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6065                  ipv4_m->hdr.time_to_live);
6066         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6067                  ipv4_v->hdr.time_to_live & ipv4_m->hdr.time_to_live);
6068 }
6069
6070 /**
6071  * Add IPV6 item to matcher and to the value.
6072  *
6073  * @param[in, out] matcher
6074  *   Flow matcher.
6075  * @param[in, out] key
6076  *   Flow matcher value.
6077  * @param[in] item
6078  *   Flow pattern to translate.
6079  * @param[in] item_flags
6080  *   Bit-fields that holds the items detected until now.
6081  * @param[in] inner
6082  *   Item is inner pattern.
6083  * @param[in] group
6084  *   The group to insert the rule.
6085  */
6086 static void
6087 flow_dv_translate_item_ipv6(void *matcher, void *key,
6088                             const struct rte_flow_item *item,
6089                             const uint64_t item_flags,
6090                             int inner, uint32_t group)
6091 {
6092         const struct rte_flow_item_ipv6 *ipv6_m = item->mask;
6093         const struct rte_flow_item_ipv6 *ipv6_v = item->spec;
6094         const struct rte_flow_item_ipv6 nic_mask = {
6095                 .hdr = {
6096                         .src_addr =
6097                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6098                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6099                         .dst_addr =
6100                                 "\xff\xff\xff\xff\xff\xff\xff\xff"
6101                                 "\xff\xff\xff\xff\xff\xff\xff\xff",
6102                         .vtc_flow = RTE_BE32(0xffffffff),
6103                         .proto = 0xff,
6104                         .hop_limits = 0xff,
6105                 },
6106         };
6107         void *headers_m;
6108         void *headers_v;
6109         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6110         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6111         char *l24_m;
6112         char *l24_v;
6113         uint32_t vtc_m;
6114         uint32_t vtc_v;
6115         int i;
6116         int size;
6117
6118         if (inner) {
6119                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6120                                          inner_headers);
6121                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6122         } else {
6123                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6124                                          outer_headers);
6125                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6126         }
6127         flow_dv_set_match_ip_version(group, headers_v, headers_m, 6);
6128         /*
6129          * On outer header (which must contains L2), or inner header with L2,
6130          * set cvlan_tag mask bit to mark this packet as untagged.
6131          * This should be done even if item->spec is empty.
6132          */
6133         if (!inner || item_flags & MLX5_FLOW_LAYER_INNER_L2)
6134                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, cvlan_tag, 1);
6135         if (!ipv6_v)
6136                 return;
6137         if (!ipv6_m)
6138                 ipv6_m = &nic_mask;
6139         size = sizeof(ipv6_m->hdr.dst_addr);
6140         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6141                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6142         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6143                              dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
6144         memcpy(l24_m, ipv6_m->hdr.dst_addr, size);
6145         for (i = 0; i < size; ++i)
6146                 l24_v[i] = l24_m[i] & ipv6_v->hdr.dst_addr[i];
6147         l24_m = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_m,
6148                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6149         l24_v = MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
6150                              src_ipv4_src_ipv6.ipv6_layout.ipv6);
6151         memcpy(l24_m, ipv6_m->hdr.src_addr, size);
6152         for (i = 0; i < size; ++i)
6153                 l24_v[i] = l24_m[i] & ipv6_v->hdr.src_addr[i];
6154         /* TOS. */
6155         vtc_m = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow);
6156         vtc_v = rte_be_to_cpu_32(ipv6_m->hdr.vtc_flow & ipv6_v->hdr.vtc_flow);
6157         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ecn, vtc_m >> 20);
6158         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn, vtc_v >> 20);
6159         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_dscp, vtc_m >> 22);
6160         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp, vtc_v >> 22);
6161         /* Label. */
6162         if (inner) {
6163                 MLX5_SET(fte_match_set_misc, misc_m, inner_ipv6_flow_label,
6164                          vtc_m);
6165                 MLX5_SET(fte_match_set_misc, misc_v, inner_ipv6_flow_label,
6166                          vtc_v);
6167         } else {
6168                 MLX5_SET(fte_match_set_misc, misc_m, outer_ipv6_flow_label,
6169                          vtc_m);
6170                 MLX5_SET(fte_match_set_misc, misc_v, outer_ipv6_flow_label,
6171                          vtc_v);
6172         }
6173         /* Protocol. */
6174         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
6175                  ipv6_m->hdr.proto);
6176         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6177                  ipv6_v->hdr.proto & ipv6_m->hdr.proto);
6178         /* Hop limit. */
6179         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_ttl_hoplimit,
6180                  ipv6_m->hdr.hop_limits);
6181         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ttl_hoplimit,
6182                  ipv6_v->hdr.hop_limits & ipv6_m->hdr.hop_limits);
6183 }
6184
6185 /**
6186  * Add TCP item to matcher and to the value.
6187  *
6188  * @param[in, out] matcher
6189  *   Flow matcher.
6190  * @param[in, out] key
6191  *   Flow matcher value.
6192  * @param[in] item
6193  *   Flow pattern to translate.
6194  * @param[in] inner
6195  *   Item is inner pattern.
6196  */
6197 static void
6198 flow_dv_translate_item_tcp(void *matcher, void *key,
6199                            const struct rte_flow_item *item,
6200                            int inner)
6201 {
6202         const struct rte_flow_item_tcp *tcp_m = item->mask;
6203         const struct rte_flow_item_tcp *tcp_v = item->spec;
6204         void *headers_m;
6205         void *headers_v;
6206
6207         if (inner) {
6208                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6209                                          inner_headers);
6210                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6211         } else {
6212                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6213                                          outer_headers);
6214                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6215         }
6216         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6217         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_TCP);
6218         if (!tcp_v)
6219                 return;
6220         if (!tcp_m)
6221                 tcp_m = &rte_flow_item_tcp_mask;
6222         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_sport,
6223                  rte_be_to_cpu_16(tcp_m->hdr.src_port));
6224         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
6225                  rte_be_to_cpu_16(tcp_v->hdr.src_port & tcp_m->hdr.src_port));
6226         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_dport,
6227                  rte_be_to_cpu_16(tcp_m->hdr.dst_port));
6228         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
6229                  rte_be_to_cpu_16(tcp_v->hdr.dst_port & tcp_m->hdr.dst_port));
6230         MLX5_SET(fte_match_set_lyr_2_4, headers_m, tcp_flags,
6231                  tcp_m->hdr.tcp_flags);
6232         MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
6233                  (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
6234 }
6235
6236 /**
6237  * Add UDP item to matcher and to the value.
6238  *
6239  * @param[in, out] matcher
6240  *   Flow matcher.
6241  * @param[in, out] key
6242  *   Flow matcher value.
6243  * @param[in] item
6244  *   Flow pattern to translate.
6245  * @param[in] inner
6246  *   Item is inner pattern.
6247  */
6248 static void
6249 flow_dv_translate_item_udp(void *matcher, void *key,
6250                            const struct rte_flow_item *item,
6251                            int inner)
6252 {
6253         const struct rte_flow_item_udp *udp_m = item->mask;
6254         const struct rte_flow_item_udp *udp_v = item->spec;
6255         void *headers_m;
6256         void *headers_v;
6257
6258         if (inner) {
6259                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6260                                          inner_headers);
6261                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6262         } else {
6263                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6264                                          outer_headers);
6265                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6266         }
6267         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6268         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
6269         if (!udp_v)
6270                 return;
6271         if (!udp_m)
6272                 udp_m = &rte_flow_item_udp_mask;
6273         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_sport,
6274                  rte_be_to_cpu_16(udp_m->hdr.src_port));
6275         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
6276                  rte_be_to_cpu_16(udp_v->hdr.src_port & udp_m->hdr.src_port));
6277         MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport,
6278                  rte_be_to_cpu_16(udp_m->hdr.dst_port));
6279         MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6280                  rte_be_to_cpu_16(udp_v->hdr.dst_port & udp_m->hdr.dst_port));
6281 }
6282
6283 /**
6284  * Add GRE optional Key item to matcher and to the value.
6285  *
6286  * @param[in, out] matcher
6287  *   Flow matcher.
6288  * @param[in, out] key
6289  *   Flow matcher value.
6290  * @param[in] item
6291  *   Flow pattern to translate.
6292  * @param[in] inner
6293  *   Item is inner pattern.
6294  */
6295 static void
6296 flow_dv_translate_item_gre_key(void *matcher, void *key,
6297                                    const struct rte_flow_item *item)
6298 {
6299         const rte_be32_t *key_m = item->mask;
6300         const rte_be32_t *key_v = item->spec;
6301         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6302         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6303         rte_be32_t gre_key_default_mask = RTE_BE32(UINT32_MAX);
6304
6305         /* GRE K bit must be on and should already be validated */
6306         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present, 1);
6307         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present, 1);
6308         if (!key_v)
6309                 return;
6310         if (!key_m)
6311                 key_m = &gre_key_default_mask;
6312         MLX5_SET(fte_match_set_misc, misc_m, gre_key_h,
6313                  rte_be_to_cpu_32(*key_m) >> 8);
6314         MLX5_SET(fte_match_set_misc, misc_v, gre_key_h,
6315                  rte_be_to_cpu_32((*key_v) & (*key_m)) >> 8);
6316         MLX5_SET(fte_match_set_misc, misc_m, gre_key_l,
6317                  rte_be_to_cpu_32(*key_m) & 0xFF);
6318         MLX5_SET(fte_match_set_misc, misc_v, gre_key_l,
6319                  rte_be_to_cpu_32((*key_v) & (*key_m)) & 0xFF);
6320 }
6321
6322 /**
6323  * Add GRE item to matcher and to the value.
6324  *
6325  * @param[in, out] matcher
6326  *   Flow matcher.
6327  * @param[in, out] key
6328  *   Flow matcher value.
6329  * @param[in] item
6330  *   Flow pattern to translate.
6331  * @param[in] inner
6332  *   Item is inner pattern.
6333  */
6334 static void
6335 flow_dv_translate_item_gre(void *matcher, void *key,
6336                            const struct rte_flow_item *item,
6337                            int inner)
6338 {
6339         const struct rte_flow_item_gre *gre_m = item->mask;
6340         const struct rte_flow_item_gre *gre_v = item->spec;
6341         void *headers_m;
6342         void *headers_v;
6343         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6344         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6345         struct {
6346                 union {
6347                         __extension__
6348                         struct {
6349                                 uint16_t version:3;
6350                                 uint16_t rsvd0:9;
6351                                 uint16_t s_present:1;
6352                                 uint16_t k_present:1;
6353                                 uint16_t rsvd_bit1:1;
6354                                 uint16_t c_present:1;
6355                         };
6356                         uint16_t value;
6357                 };
6358         } gre_crks_rsvd0_ver_m, gre_crks_rsvd0_ver_v;
6359
6360         if (inner) {
6361                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6362                                          inner_headers);
6363                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6364         } else {
6365                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6366                                          outer_headers);
6367                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6368         }
6369         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6370         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_GRE);
6371         if (!gre_v)
6372                 return;
6373         if (!gre_m)
6374                 gre_m = &rte_flow_item_gre_mask;
6375         MLX5_SET(fte_match_set_misc, misc_m, gre_protocol,
6376                  rte_be_to_cpu_16(gre_m->protocol));
6377         MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6378                  rte_be_to_cpu_16(gre_v->protocol & gre_m->protocol));
6379         gre_crks_rsvd0_ver_m.value = rte_be_to_cpu_16(gre_m->c_rsvd0_ver);
6380         gre_crks_rsvd0_ver_v.value = rte_be_to_cpu_16(gre_v->c_rsvd0_ver);
6381         MLX5_SET(fte_match_set_misc, misc_m, gre_c_present,
6382                  gre_crks_rsvd0_ver_m.c_present);
6383         MLX5_SET(fte_match_set_misc, misc_v, gre_c_present,
6384                  gre_crks_rsvd0_ver_v.c_present &
6385                  gre_crks_rsvd0_ver_m.c_present);
6386         MLX5_SET(fte_match_set_misc, misc_m, gre_k_present,
6387                  gre_crks_rsvd0_ver_m.k_present);
6388         MLX5_SET(fte_match_set_misc, misc_v, gre_k_present,
6389                  gre_crks_rsvd0_ver_v.k_present &
6390                  gre_crks_rsvd0_ver_m.k_present);
6391         MLX5_SET(fte_match_set_misc, misc_m, gre_s_present,
6392                  gre_crks_rsvd0_ver_m.s_present);
6393         MLX5_SET(fte_match_set_misc, misc_v, gre_s_present,
6394                  gre_crks_rsvd0_ver_v.s_present &
6395                  gre_crks_rsvd0_ver_m.s_present);
6396 }
6397
6398 /**
6399  * Add NVGRE item to matcher and to the value.
6400  *
6401  * @param[in, out] matcher
6402  *   Flow matcher.
6403  * @param[in, out] key
6404  *   Flow matcher value.
6405  * @param[in] item
6406  *   Flow pattern to translate.
6407  * @param[in] inner
6408  *   Item is inner pattern.
6409  */
6410 static void
6411 flow_dv_translate_item_nvgre(void *matcher, void *key,
6412                              const struct rte_flow_item *item,
6413                              int inner)
6414 {
6415         const struct rte_flow_item_nvgre *nvgre_m = item->mask;
6416         const struct rte_flow_item_nvgre *nvgre_v = item->spec;
6417         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6418         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6419         const char *tni_flow_id_m = (const char *)nvgre_m->tni;
6420         const char *tni_flow_id_v = (const char *)nvgre_v->tni;
6421         char *gre_key_m;
6422         char *gre_key_v;
6423         int size;
6424         int i;
6425
6426         /* For NVGRE, GRE header fields must be set with defined values. */
6427         const struct rte_flow_item_gre gre_spec = {
6428                 .c_rsvd0_ver = RTE_BE16(0x2000),
6429                 .protocol = RTE_BE16(RTE_ETHER_TYPE_TEB)
6430         };
6431         const struct rte_flow_item_gre gre_mask = {
6432                 .c_rsvd0_ver = RTE_BE16(0xB000),
6433                 .protocol = RTE_BE16(UINT16_MAX),
6434         };
6435         const struct rte_flow_item gre_item = {
6436                 .spec = &gre_spec,
6437                 .mask = &gre_mask,
6438                 .last = NULL,
6439         };
6440         flow_dv_translate_item_gre(matcher, key, &gre_item, inner);
6441         if (!nvgre_v)
6442                 return;
6443         if (!nvgre_m)
6444                 nvgre_m = &rte_flow_item_nvgre_mask;
6445         size = sizeof(nvgre_m->tni) + sizeof(nvgre_m->flow_id);
6446         gre_key_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, gre_key_h);
6447         gre_key_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, gre_key_h);
6448         memcpy(gre_key_m, tni_flow_id_m, size);
6449         for (i = 0; i < size; ++i)
6450                 gre_key_v[i] = gre_key_m[i] & tni_flow_id_v[i];
6451 }
6452
6453 /**
6454  * Add VXLAN item to matcher and to the value.
6455  *
6456  * @param[in, out] matcher
6457  *   Flow matcher.
6458  * @param[in, out] key
6459  *   Flow matcher value.
6460  * @param[in] item
6461  *   Flow pattern to translate.
6462  * @param[in] inner
6463  *   Item is inner pattern.
6464  */
6465 static void
6466 flow_dv_translate_item_vxlan(void *matcher, void *key,
6467                              const struct rte_flow_item *item,
6468                              int inner)
6469 {
6470         const struct rte_flow_item_vxlan *vxlan_m = item->mask;
6471         const struct rte_flow_item_vxlan *vxlan_v = item->spec;
6472         void *headers_m;
6473         void *headers_v;
6474         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6475         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6476         char *vni_m;
6477         char *vni_v;
6478         uint16_t dport;
6479         int size;
6480         int i;
6481
6482         if (inner) {
6483                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6484                                          inner_headers);
6485                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6486         } else {
6487                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6488                                          outer_headers);
6489                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6490         }
6491         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6492                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6493         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6494                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6495                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6496         }
6497         if (!vxlan_v)
6498                 return;
6499         if (!vxlan_m)
6500                 vxlan_m = &rte_flow_item_vxlan_mask;
6501         size = sizeof(vxlan_m->vni);
6502         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, vxlan_vni);
6503         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, vxlan_vni);
6504         memcpy(vni_m, vxlan_m->vni, size);
6505         for (i = 0; i < size; ++i)
6506                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6507 }
6508
6509 /**
6510  * Add VXLAN-GPE item to matcher and to the value.
6511  *
6512  * @param[in, out] matcher
6513  *   Flow matcher.
6514  * @param[in, out] key
6515  *   Flow matcher value.
6516  * @param[in] item
6517  *   Flow pattern to translate.
6518  * @param[in] inner
6519  *   Item is inner pattern.
6520  */
6521
6522 static void
6523 flow_dv_translate_item_vxlan_gpe(void *matcher, void *key,
6524                                  const struct rte_flow_item *item, int inner)
6525 {
6526         const struct rte_flow_item_vxlan_gpe *vxlan_m = item->mask;
6527         const struct rte_flow_item_vxlan_gpe *vxlan_v = item->spec;
6528         void *headers_m;
6529         void *headers_v;
6530         void *misc_m =
6531                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_3);
6532         void *misc_v =
6533                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
6534         char *vni_m;
6535         char *vni_v;
6536         uint16_t dport;
6537         int size;
6538         int i;
6539         uint8_t flags_m = 0xff;
6540         uint8_t flags_v = 0xc;
6541
6542         if (inner) {
6543                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6544                                          inner_headers);
6545                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6546         } else {
6547                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6548                                          outer_headers);
6549                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6550         }
6551         dport = item->type == RTE_FLOW_ITEM_TYPE_VXLAN ?
6552                 MLX5_UDP_PORT_VXLAN : MLX5_UDP_PORT_VXLAN_GPE;
6553         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6554                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6555                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6556         }
6557         if (!vxlan_v)
6558                 return;
6559         if (!vxlan_m)
6560                 vxlan_m = &rte_flow_item_vxlan_gpe_mask;
6561         size = sizeof(vxlan_m->vni);
6562         vni_m = MLX5_ADDR_OF(fte_match_set_misc3, misc_m, outer_vxlan_gpe_vni);
6563         vni_v = MLX5_ADDR_OF(fte_match_set_misc3, misc_v, outer_vxlan_gpe_vni);
6564         memcpy(vni_m, vxlan_m->vni, size);
6565         for (i = 0; i < size; ++i)
6566                 vni_v[i] = vni_m[i] & vxlan_v->vni[i];
6567         if (vxlan_m->flags) {
6568                 flags_m = vxlan_m->flags;
6569                 flags_v = vxlan_v->flags;
6570         }
6571         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_flags, flags_m);
6572         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_flags, flags_v);
6573         MLX5_SET(fte_match_set_misc3, misc_m, outer_vxlan_gpe_next_protocol,
6574                  vxlan_m->protocol);
6575         MLX5_SET(fte_match_set_misc3, misc_v, outer_vxlan_gpe_next_protocol,
6576                  vxlan_v->protocol);
6577 }
6578
6579 /**
6580  * Add Geneve item to matcher and to the value.
6581  *
6582  * @param[in, out] matcher
6583  *   Flow matcher.
6584  * @param[in, out] key
6585  *   Flow matcher value.
6586  * @param[in] item
6587  *   Flow pattern to translate.
6588  * @param[in] inner
6589  *   Item is inner pattern.
6590  */
6591
6592 static void
6593 flow_dv_translate_item_geneve(void *matcher, void *key,
6594                               const struct rte_flow_item *item, int inner)
6595 {
6596         const struct rte_flow_item_geneve *geneve_m = item->mask;
6597         const struct rte_flow_item_geneve *geneve_v = item->spec;
6598         void *headers_m;
6599         void *headers_v;
6600         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6601         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6602         uint16_t dport;
6603         uint16_t gbhdr_m;
6604         uint16_t gbhdr_v;
6605         char *vni_m;
6606         char *vni_v;
6607         size_t size, i;
6608
6609         if (inner) {
6610                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6611                                          inner_headers);
6612                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
6613         } else {
6614                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
6615                                          outer_headers);
6616                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6617         }
6618         dport = MLX5_UDP_PORT_GENEVE;
6619         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
6620                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
6621                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
6622         }
6623         if (!geneve_v)
6624                 return;
6625         if (!geneve_m)
6626                 geneve_m = &rte_flow_item_geneve_mask;
6627         size = sizeof(geneve_m->vni);
6628         vni_m = MLX5_ADDR_OF(fte_match_set_misc, misc_m, geneve_vni);
6629         vni_v = MLX5_ADDR_OF(fte_match_set_misc, misc_v, geneve_vni);
6630         memcpy(vni_m, geneve_m->vni, size);
6631         for (i = 0; i < size; ++i)
6632                 vni_v[i] = vni_m[i] & geneve_v->vni[i];
6633         MLX5_SET(fte_match_set_misc, misc_m, geneve_protocol_type,
6634                  rte_be_to_cpu_16(geneve_m->protocol));
6635         MLX5_SET(fte_match_set_misc, misc_v, geneve_protocol_type,
6636                  rte_be_to_cpu_16(geneve_v->protocol & geneve_m->protocol));
6637         gbhdr_m = rte_be_to_cpu_16(geneve_m->ver_opt_len_o_c_rsvd0);
6638         gbhdr_v = rte_be_to_cpu_16(geneve_v->ver_opt_len_o_c_rsvd0);
6639         MLX5_SET(fte_match_set_misc, misc_m, geneve_oam,
6640                  MLX5_GENEVE_OAMF_VAL(gbhdr_m));
6641         MLX5_SET(fte_match_set_misc, misc_v, geneve_oam,
6642                  MLX5_GENEVE_OAMF_VAL(gbhdr_v) & MLX5_GENEVE_OAMF_VAL(gbhdr_m));
6643         MLX5_SET(fte_match_set_misc, misc_m, geneve_opt_len,
6644                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
6645         MLX5_SET(fte_match_set_misc, misc_v, geneve_opt_len,
6646                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_v) &
6647                  MLX5_GENEVE_OPTLEN_VAL(gbhdr_m));
6648 }
6649
6650 /**
6651  * Add MPLS item to matcher and to the value.
6652  *
6653  * @param[in, out] matcher
6654  *   Flow matcher.
6655  * @param[in, out] key
6656  *   Flow matcher value.
6657  * @param[in] item
6658  *   Flow pattern to translate.
6659  * @param[in] prev_layer
6660  *   The protocol layer indicated in previous item.
6661  * @param[in] inner
6662  *   Item is inner pattern.
6663  */
6664 static void
6665 flow_dv_translate_item_mpls(void *matcher, void *key,
6666                             const struct rte_flow_item *item,
6667                             uint64_t prev_layer,
6668                             int inner)
6669 {
6670         const uint32_t *in_mpls_m = item->mask;
6671         const uint32_t *in_mpls_v = item->spec;
6672         uint32_t *out_mpls_m = 0;
6673         uint32_t *out_mpls_v = 0;
6674         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
6675         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
6676         void *misc2_m = MLX5_ADDR_OF(fte_match_param, matcher,
6677                                      misc_parameters_2);
6678         void *misc2_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
6679         void *headers_m = MLX5_ADDR_OF(fte_match_param, matcher, outer_headers);
6680         void *headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
6681
6682         switch (prev_layer) {
6683         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
6684                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xffff);
6685                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
6686                          MLX5_UDP_PORT_MPLS);
6687                 break;
6688         case MLX5_FLOW_LAYER_GRE:
6689                 MLX5_SET(fte_match_set_misc, misc_m, gre_protocol, 0xffff);
6690                 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol,
6691                          RTE_ETHER_TYPE_MPLS);
6692                 break;
6693         default:
6694                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
6695                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
6696                          IPPROTO_MPLS);
6697                 break;
6698         }
6699         if (!in_mpls_v)
6700                 return;
6701         if (!in_mpls_m)
6702                 in_mpls_m = (const uint32_t *)&rte_flow_item_mpls_mask;
6703         switch (prev_layer) {
6704         case MLX5_FLOW_LAYER_OUTER_L4_UDP:
6705                 out_mpls_m =
6706                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
6707                                                  outer_first_mpls_over_udp);
6708                 out_mpls_v =
6709                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
6710                                                  outer_first_mpls_over_udp);
6711                 break;
6712         case MLX5_FLOW_LAYER_GRE:
6713                 out_mpls_m =
6714                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_m,
6715                                                  outer_first_mpls_over_gre);
6716                 out_mpls_v =
6717                         (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2, misc2_v,
6718                                                  outer_first_mpls_over_gre);
6719                 break;
6720         default:
6721                 /* Inner MPLS not over GRE is not supported. */
6722                 if (!inner) {
6723                         out_mpls_m =
6724                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
6725                                                          misc2_m,
6726                                                          outer_first_mpls);
6727                         out_mpls_v =
6728                                 (uint32_t *)MLX5_ADDR_OF(fte_match_set_misc2,
6729                                                          misc2_v,
6730                                                          outer_first_mpls);
6731                 }
6732                 break;
6733         }
6734         if (out_mpls_m && out_mpls_v) {
6735                 *out_mpls_m = *in_mpls_m;
6736                 *out_mpls_v = *in_mpls_v & *in_mpls_m;
6737         }
6738 }
6739
6740 /**
6741  * Add metadata register item to matcher
6742  *
6743  * @param[in, out] matcher
6744  *   Flow matcher.
6745  * @param[in, out] key
6746  *   Flow matcher value.
6747  * @param[in] reg_type
6748  *   Type of device metadata register
6749  * @param[in] value
6750  *   Register value
6751  * @param[in] mask
6752  *   Register mask
6753  */
6754 static void
6755 flow_dv_match_meta_reg(void *matcher, void *key,
6756                        enum modify_reg reg_type,
6757                        uint32_t data, uint32_t mask)
6758 {
6759         void *misc2_m =
6760                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
6761         void *misc2_v =
6762                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
6763         uint32_t temp;
6764
6765         data &= mask;
6766         switch (reg_type) {
6767         case REG_A:
6768                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_a, mask);
6769                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_a, data);
6770                 break;
6771         case REG_B:
6772                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_b, mask);
6773                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
6774                 break;
6775         case REG_C_0:
6776                 /*
6777                  * The metadata register C0 field might be divided into
6778                  * source vport index and META item value, we should set
6779                  * this field according to specified mask, not as whole one.
6780                  */
6781                 temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
6782                 temp |= mask;
6783                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
6784                 temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
6785                 temp &= ~mask;
6786                 temp |= data;
6787                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
6788                 break;
6789         case REG_C_1:
6790                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
6791                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_1, data);
6792                 break;
6793         case REG_C_2:
6794                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_2, mask);
6795                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_2, data);
6796                 break;
6797         case REG_C_3:
6798                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_3, mask);
6799                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_3, data);
6800                 break;
6801         case REG_C_4:
6802                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_4, mask);
6803                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_4, data);
6804                 break;
6805         case REG_C_5:
6806                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_5, mask);
6807                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_5, data);
6808                 break;
6809         case REG_C_6:
6810                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_6, mask);
6811                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_6, data);
6812                 break;
6813         case REG_C_7:
6814                 MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_7, mask);
6815                 MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_7, data);
6816                 break;
6817         default:
6818                 MLX5_ASSERT(false);
6819                 break;
6820         }
6821 }
6822
6823 /**
6824  * Add MARK item to matcher
6825  *
6826  * @param[in] dev
6827  *   The device to configure through.
6828  * @param[in, out] matcher
6829  *   Flow matcher.
6830  * @param[in, out] key
6831  *   Flow matcher value.
6832  * @param[in] item
6833  *   Flow pattern to translate.
6834  */
6835 static void
6836 flow_dv_translate_item_mark(struct rte_eth_dev *dev,
6837                             void *matcher, void *key,
6838                             const struct rte_flow_item *item)
6839 {
6840         struct mlx5_priv *priv = dev->data->dev_private;
6841         const struct rte_flow_item_mark *mark;
6842         uint32_t value;
6843         uint32_t mask;
6844
6845         mark = item->mask ? (const void *)item->mask :
6846                             &rte_flow_item_mark_mask;
6847         mask = mark->id & priv->sh->dv_mark_mask;
6848         mark = (const void *)item->spec;
6849         MLX5_ASSERT(mark);
6850         value = mark->id & priv->sh->dv_mark_mask & mask;
6851         if (mask) {
6852                 enum modify_reg reg;
6853
6854                 /* Get the metadata register index for the mark. */
6855                 reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
6856                 MLX5_ASSERT(reg > 0);
6857                 if (reg == REG_C_0) {
6858                         struct mlx5_priv *priv = dev->data->dev_private;
6859                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6860                         uint32_t shl_c0 = rte_bsf32(msk_c0);
6861
6862                         mask &= msk_c0;
6863                         mask <<= shl_c0;
6864                         value <<= shl_c0;
6865                 }
6866                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
6867         }
6868 }
6869
6870 /**
6871  * Add META item to matcher
6872  *
6873  * @param[in] dev
6874  *   The devich to configure through.
6875  * @param[in, out] matcher
6876  *   Flow matcher.
6877  * @param[in, out] key
6878  *   Flow matcher value.
6879  * @param[in] attr
6880  *   Attributes of flow that includes this item.
6881  * @param[in] item
6882  *   Flow pattern to translate.
6883  */
6884 static void
6885 flow_dv_translate_item_meta(struct rte_eth_dev *dev,
6886                             void *matcher, void *key,
6887                             const struct rte_flow_attr *attr,
6888                             const struct rte_flow_item *item)
6889 {
6890         const struct rte_flow_item_meta *meta_m;
6891         const struct rte_flow_item_meta *meta_v;
6892
6893         meta_m = (const void *)item->mask;
6894         if (!meta_m)
6895                 meta_m = &rte_flow_item_meta_mask;
6896         meta_v = (const void *)item->spec;
6897         if (meta_v) {
6898                 int reg;
6899                 uint32_t value = meta_v->data;
6900                 uint32_t mask = meta_m->data;
6901
6902                 reg = flow_dv_get_metadata_reg(dev, attr, NULL);
6903                 if (reg < 0)
6904                         return;
6905                 /*
6906                  * In datapath code there is no endianness
6907                  * coversions for perfromance reasons, all
6908                  * pattern conversions are done in rte_flow.
6909                  */
6910                 value = rte_cpu_to_be_32(value);
6911                 mask = rte_cpu_to_be_32(mask);
6912                 if (reg == REG_C_0) {
6913                         struct mlx5_priv *priv = dev->data->dev_private;
6914                         uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6915                         uint32_t shl_c0 = rte_bsf32(msk_c0);
6916 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6917                         uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
6918
6919                         value >>= shr_c0;
6920                         mask >>= shr_c0;
6921 #endif
6922                         value <<= shl_c0;
6923                         mask <<= shl_c0;
6924                         MLX5_ASSERT(msk_c0);
6925                         MLX5_ASSERT(!(~msk_c0 & mask));
6926                 }
6927                 flow_dv_match_meta_reg(matcher, key, reg, value, mask);
6928         }
6929 }
6930
6931 /**
6932  * Add vport metadata Reg C0 item to matcher
6933  *
6934  * @param[in, out] matcher
6935  *   Flow matcher.
6936  * @param[in, out] key
6937  *   Flow matcher value.
6938  * @param[in] reg
6939  *   Flow pattern to translate.
6940  */
6941 static void
6942 flow_dv_translate_item_meta_vport(void *matcher, void *key,
6943                                   uint32_t value, uint32_t mask)
6944 {
6945         flow_dv_match_meta_reg(matcher, key, REG_C_0, value, mask);
6946 }
6947
6948 /**
6949  * Add tag item to matcher
6950  *
6951  * @param[in] dev
6952  *   The devich to configure through.
6953  * @param[in, out] matcher
6954  *   Flow matcher.
6955  * @param[in, out] key
6956  *   Flow matcher value.
6957  * @param[in] item
6958  *   Flow pattern to translate.
6959  */
6960 static void
6961 flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
6962                                 void *matcher, void *key,
6963                                 const struct rte_flow_item *item)
6964 {
6965         const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
6966         const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
6967         uint32_t mask, value;
6968
6969         MLX5_ASSERT(tag_v);
6970         value = tag_v->data;
6971         mask = tag_m ? tag_m->data : UINT32_MAX;
6972         if (tag_v->id == REG_C_0) {
6973                 struct mlx5_priv *priv = dev->data->dev_private;
6974                 uint32_t msk_c0 = priv->sh->dv_regc0_mask;
6975                 uint32_t shl_c0 = rte_bsf32(msk_c0);
6976
6977                 mask &= msk_c0;
6978                 mask <<= shl_c0;
6979                 value <<= shl_c0;
6980         }
6981         flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
6982 }
6983
6984 /**
6985  * Add TAG item to matcher
6986  *
6987  * @param[in] dev
6988  *   The devich to configure through.
6989  * @param[in, out] matcher
6990  *   Flow matcher.
6991  * @param[in, out] key
6992  *   Flow matcher value.
6993  * @param[in] item
6994  *   Flow pattern to translate.
6995  */
6996 static void
6997 flow_dv_translate_item_tag(struct rte_eth_dev *dev,
6998                            void *matcher, void *key,
6999                            const struct rte_flow_item *item)
7000 {
7001         const struct rte_flow_item_tag *tag_v = item->spec;
7002         const struct rte_flow_item_tag *tag_m = item->mask;
7003         enum modify_reg reg;
7004
7005         MLX5_ASSERT(tag_v);
7006         tag_m = tag_m ? tag_m : &rte_flow_item_tag_mask;
7007         /* Get the metadata register index for the tag. */
7008         reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, tag_v->index, NULL);
7009         MLX5_ASSERT(reg > 0);
7010         flow_dv_match_meta_reg(matcher, key, reg, tag_v->data, tag_m->data);
7011 }
7012
7013 /**
7014  * Add source vport match to the specified matcher.
7015  *
7016  * @param[in, out] matcher
7017  *   Flow matcher.
7018  * @param[in, out] key
7019  *   Flow matcher value.
7020  * @param[in] port
7021  *   Source vport value to match
7022  * @param[in] mask
7023  *   Mask
7024  */
7025 static void
7026 flow_dv_translate_item_source_vport(void *matcher, void *key,
7027                                     int16_t port, uint16_t mask)
7028 {
7029         void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7030         void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7031
7032         MLX5_SET(fte_match_set_misc, misc_m, source_port, mask);
7033         MLX5_SET(fte_match_set_misc, misc_v, source_port, port);
7034 }
7035
7036 /**
7037  * Translate port-id item to eswitch match on  port-id.
7038  *
7039  * @param[in] dev
7040  *   The devich to configure through.
7041  * @param[in, out] matcher
7042  *   Flow matcher.
7043  * @param[in, out] key
7044  *   Flow matcher value.
7045  * @param[in] item
7046  *   Flow pattern to translate.
7047  *
7048  * @return
7049  *   0 on success, a negative errno value otherwise.
7050  */
7051 static int
7052 flow_dv_translate_item_port_id(struct rte_eth_dev *dev, void *matcher,
7053                                void *key, const struct rte_flow_item *item)
7054 {
7055         const struct rte_flow_item_port_id *pid_m = item ? item->mask : NULL;
7056         const struct rte_flow_item_port_id *pid_v = item ? item->spec : NULL;
7057         struct mlx5_priv *priv;
7058         uint16_t mask, id;
7059
7060         mask = pid_m ? pid_m->id : 0xffff;
7061         id = pid_v ? pid_v->id : dev->data->port_id;
7062         priv = mlx5_port_to_eswitch_info(id, item == NULL);
7063         if (!priv)
7064                 return -rte_errno;
7065         /* Translate to vport field or to metadata, depending on mode. */
7066         if (priv->vport_meta_mask)
7067                 flow_dv_translate_item_meta_vport(matcher, key,
7068                                                   priv->vport_meta_tag,
7069                                                   priv->vport_meta_mask);
7070         else
7071                 flow_dv_translate_item_source_vport(matcher, key,
7072                                                     priv->vport_id, mask);
7073         return 0;
7074 }
7075
7076 /**
7077  * Add ICMP6 item to matcher and to the value.
7078  *
7079  * @param[in, out] matcher
7080  *   Flow matcher.
7081  * @param[in, out] key
7082  *   Flow matcher value.
7083  * @param[in] item
7084  *   Flow pattern to translate.
7085  * @param[in] inner
7086  *   Item is inner pattern.
7087  */
7088 static void
7089 flow_dv_translate_item_icmp6(void *matcher, void *key,
7090                               const struct rte_flow_item *item,
7091                               int inner)
7092 {
7093         const struct rte_flow_item_icmp6 *icmp6_m = item->mask;
7094         const struct rte_flow_item_icmp6 *icmp6_v = item->spec;
7095         void *headers_m;
7096         void *headers_v;
7097         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7098                                      misc_parameters_3);
7099         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7100         if (inner) {
7101                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7102                                          inner_headers);
7103                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7104         } else {
7105                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7106                                          outer_headers);
7107                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7108         }
7109         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7110         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMPV6);
7111         if (!icmp6_v)
7112                 return;
7113         if (!icmp6_m)
7114                 icmp6_m = &rte_flow_item_icmp6_mask;
7115         /*
7116          * Force flow only to match the non-fragmented IPv6 ICMPv6 packets.
7117          * If only the protocol is specified, no need to match the frag.
7118          */
7119         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
7120         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
7121         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_type, icmp6_m->type);
7122         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_type,
7123                  icmp6_v->type & icmp6_m->type);
7124         MLX5_SET(fte_match_set_misc3, misc3_m, icmpv6_code, icmp6_m->code);
7125         MLX5_SET(fte_match_set_misc3, misc3_v, icmpv6_code,
7126                  icmp6_v->code & icmp6_m->code);
7127 }
7128
7129 /**
7130  * Add ICMP item to matcher and to the value.
7131  *
7132  * @param[in, out] matcher
7133  *   Flow matcher.
7134  * @param[in, out] key
7135  *   Flow matcher value.
7136  * @param[in] item
7137  *   Flow pattern to translate.
7138  * @param[in] inner
7139  *   Item is inner pattern.
7140  */
7141 static void
7142 flow_dv_translate_item_icmp(void *matcher, void *key,
7143                             const struct rte_flow_item *item,
7144                             int inner)
7145 {
7146         const struct rte_flow_item_icmp *icmp_m = item->mask;
7147         const struct rte_flow_item_icmp *icmp_v = item->spec;
7148         void *headers_m;
7149         void *headers_v;
7150         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7151                                      misc_parameters_3);
7152         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7153         if (inner) {
7154                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7155                                          inner_headers);
7156                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7157         } else {
7158                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7159                                          outer_headers);
7160                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7161         }
7162         MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xFF);
7163         MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ICMP);
7164         if (!icmp_v)
7165                 return;
7166         if (!icmp_m)
7167                 icmp_m = &rte_flow_item_icmp_mask;
7168         /*
7169          * Force flow only to match the non-fragmented IPv4 ICMP packets.
7170          * If only the protocol is specified, no need to match the frag.
7171          */
7172         MLX5_SET(fte_match_set_lyr_2_4, headers_m, frag, 1);
7173         MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
7174         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_type,
7175                  icmp_m->hdr.icmp_type);
7176         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_type,
7177                  icmp_v->hdr.icmp_type & icmp_m->hdr.icmp_type);
7178         MLX5_SET(fte_match_set_misc3, misc3_m, icmp_code,
7179                  icmp_m->hdr.icmp_code);
7180         MLX5_SET(fte_match_set_misc3, misc3_v, icmp_code,
7181                  icmp_v->hdr.icmp_code & icmp_m->hdr.icmp_code);
7182 }
7183
7184 /**
7185  * Add GTP item to matcher and to the value.
7186  *
7187  * @param[in, out] matcher
7188  *   Flow matcher.
7189  * @param[in, out] key
7190  *   Flow matcher value.
7191  * @param[in] item
7192  *   Flow pattern to translate.
7193  * @param[in] inner
7194  *   Item is inner pattern.
7195  */
7196 static void
7197 flow_dv_translate_item_gtp(void *matcher, void *key,
7198                            const struct rte_flow_item *item, int inner)
7199 {
7200         const struct rte_flow_item_gtp *gtp_m = item->mask;
7201         const struct rte_flow_item_gtp *gtp_v = item->spec;
7202         void *headers_m;
7203         void *headers_v;
7204         void *misc3_m = MLX5_ADDR_OF(fte_match_param, matcher,
7205                                      misc_parameters_3);
7206         void *misc3_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_3);
7207         uint16_t dport = RTE_GTPU_UDP_PORT;
7208
7209         if (inner) {
7210                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7211                                          inner_headers);
7212                 headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
7213         } else {
7214                 headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
7215                                          outer_headers);
7216                 headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
7217         }
7218         if (!MLX5_GET16(fte_match_set_lyr_2_4, headers_v, udp_dport)) {
7219                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, udp_dport, 0xFFFF);
7220                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, dport);
7221         }
7222         if (!gtp_v)
7223                 return;
7224         if (!gtp_m)
7225                 gtp_m = &rte_flow_item_gtp_mask;
7226         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_flags,
7227                  gtp_m->v_pt_rsv_flags);
7228         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_flags,
7229                  gtp_v->v_pt_rsv_flags & gtp_m->v_pt_rsv_flags);
7230         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_msg_type, gtp_m->msg_type);
7231         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_msg_type,
7232                  gtp_v->msg_type & gtp_m->msg_type);
7233         MLX5_SET(fte_match_set_misc3, misc3_m, gtpu_teid,
7234                  rte_be_to_cpu_32(gtp_m->teid));
7235         MLX5_SET(fte_match_set_misc3, misc3_v, gtpu_teid,
7236                  rte_be_to_cpu_32(gtp_v->teid & gtp_m->teid));
7237 }
7238
7239 static uint32_t matcher_zero[MLX5_ST_SZ_DW(fte_match_param)] = { 0 };
7240
7241 #define HEADER_IS_ZERO(match_criteria, headers)                              \
7242         !(memcmp(MLX5_ADDR_OF(fte_match_param, match_criteria, headers),     \
7243                  matcher_zero, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
7244
7245 /**
7246  * Calculate flow matcher enable bitmap.
7247  *
7248  * @param match_criteria
7249  *   Pointer to flow matcher criteria.
7250  *
7251  * @return
7252  *   Bitmap of enabled fields.
7253  */
7254 static uint8_t
7255 flow_dv_matcher_enable(uint32_t *match_criteria)
7256 {
7257         uint8_t match_criteria_enable;
7258
7259         match_criteria_enable =
7260                 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
7261                 MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT;
7262         match_criteria_enable |=
7263                 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
7264                 MLX5_MATCH_CRITERIA_ENABLE_MISC_BIT;
7265         match_criteria_enable |=
7266                 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
7267                 MLX5_MATCH_CRITERIA_ENABLE_INNER_BIT;
7268         match_criteria_enable |=
7269                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
7270                 MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
7271         match_criteria_enable |=
7272                 (!HEADER_IS_ZERO(match_criteria, misc_parameters_3)) <<
7273                 MLX5_MATCH_CRITERIA_ENABLE_MISC3_BIT;
7274         return match_criteria_enable;
7275 }
7276
7277
7278 /**
7279  * Get a flow table.
7280  *
7281  * @param[in, out] dev
7282  *   Pointer to rte_eth_dev structure.
7283  * @param[in] table_id
7284  *   Table id to use.
7285  * @param[in] egress
7286  *   Direction of the table.
7287  * @param[in] transfer
7288  *   E-Switch or NIC flow.
7289  * @param[out] error
7290  *   pointer to error structure.
7291  *
7292  * @return
7293  *   Returns tables resource based on the index, NULL in case of failed.
7294  */
7295 static struct mlx5_flow_tbl_resource *
7296 flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
7297                          uint32_t table_id, uint8_t egress,
7298                          uint8_t transfer,
7299                          struct rte_flow_error *error)
7300 {
7301         struct mlx5_priv *priv = dev->data->dev_private;
7302         struct mlx5_dev_ctx_shared *sh = priv->sh;
7303         struct mlx5_flow_tbl_resource *tbl;
7304         union mlx5_flow_tbl_key table_key = {
7305                 {
7306                         .table_id = table_id,
7307                         .reserved = 0,
7308                         .domain = !!transfer,
7309                         .direction = !!egress,
7310                 }
7311         };
7312         struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
7313                                                          table_key.v64);
7314         struct mlx5_flow_tbl_data_entry *tbl_data;
7315         uint32_t idx = 0;
7316         int ret;
7317         void *domain;
7318
7319         if (pos) {
7320                 tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
7321                                         entry);
7322                 tbl = &tbl_data->tbl;
7323                 rte_atomic32_inc(&tbl->refcnt);
7324                 return tbl;
7325         }
7326         tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
7327         if (!tbl_data) {
7328                 rte_flow_error_set(error, ENOMEM,
7329                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7330                                    NULL,
7331                                    "cannot allocate flow table data entry");
7332                 return NULL;
7333         }
7334         tbl_data->idx = idx;
7335         tbl = &tbl_data->tbl;
7336         pos = &tbl_data->entry;
7337         if (transfer)
7338                 domain = sh->fdb_domain;
7339         else if (egress)
7340                 domain = sh->tx_domain;
7341         else
7342                 domain = sh->rx_domain;
7343         tbl->obj = mlx5_glue->dr_create_flow_tbl(domain, table_id);
7344         if (!tbl->obj) {
7345                 rte_flow_error_set(error, ENOMEM,
7346                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7347                                    NULL, "cannot create flow table object");
7348                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7349                 return NULL;
7350         }
7351         /*
7352          * No multi-threads now, but still better to initialize the reference
7353          * count before insert it into the hash list.
7354          */
7355         rte_atomic32_init(&tbl->refcnt);
7356         /* Jump action reference count is initialized here. */
7357         rte_atomic32_init(&tbl_data->jump.refcnt);
7358         pos->key = table_key.v64;
7359         ret = mlx5_hlist_insert(sh->flow_tbls, pos);
7360         if (ret < 0) {
7361                 rte_flow_error_set(error, -ret,
7362                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7363                                    "cannot insert flow table data entry");
7364                 mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
7365                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
7366         }
7367         rte_atomic32_inc(&tbl->refcnt);
7368         return tbl;
7369 }
7370
7371 /**
7372  * Release a flow table.
7373  *
7374  * @param[in] dev
7375  *   Pointer to rte_eth_dev structure.
7376  * @param[in] tbl
7377  *   Table resource to be released.
7378  *
7379  * @return
7380  *   Returns 0 if table was released, else return 1;
7381  */
7382 static int
7383 flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
7384                              struct mlx5_flow_tbl_resource *tbl)
7385 {
7386         struct mlx5_priv *priv = dev->data->dev_private;
7387         struct mlx5_dev_ctx_shared *sh = priv->sh;
7388         struct mlx5_flow_tbl_data_entry *tbl_data =
7389                 container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7390
7391         if (!tbl)
7392                 return 0;
7393         if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
7394                 struct mlx5_hlist_entry *pos = &tbl_data->entry;
7395
7396                 mlx5_glue->dr_destroy_flow_tbl(tbl->obj);
7397                 tbl->obj = NULL;
7398                 /* remove the entry from the hash list and free memory. */
7399                 mlx5_hlist_remove(sh->flow_tbls, pos);
7400                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
7401                                 tbl_data->idx);
7402                 return 0;
7403         }
7404         return 1;
7405 }
7406
7407 /**
7408  * Register the flow matcher.
7409  *
7410  * @param[in, out] dev
7411  *   Pointer to rte_eth_dev structure.
7412  * @param[in, out] matcher
7413  *   Pointer to flow matcher.
7414  * @param[in, out] key
7415  *   Pointer to flow table key.
7416  * @parm[in, out] dev_flow
7417  *   Pointer to the dev_flow.
7418  * @param[out] error
7419  *   pointer to error structure.
7420  *
7421  * @return
7422  *   0 on success otherwise -errno and errno is set.
7423  */
7424 static int
7425 flow_dv_matcher_register(struct rte_eth_dev *dev,
7426                          struct mlx5_flow_dv_matcher *matcher,
7427                          union mlx5_flow_tbl_key *key,
7428                          struct mlx5_flow *dev_flow,
7429                          struct rte_flow_error *error)
7430 {
7431         struct mlx5_priv *priv = dev->data->dev_private;
7432         struct mlx5_dev_ctx_shared *sh = priv->sh;
7433         struct mlx5_flow_dv_matcher *cache_matcher;
7434         struct mlx5dv_flow_matcher_attr dv_attr = {
7435                 .type = IBV_FLOW_ATTR_NORMAL,
7436                 .match_mask = (void *)&matcher->mask,
7437         };
7438         struct mlx5_flow_tbl_resource *tbl;
7439         struct mlx5_flow_tbl_data_entry *tbl_data;
7440
7441         tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
7442                                        key->domain, error);
7443         if (!tbl)
7444                 return -rte_errno;      /* No need to refill the error info */
7445         tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
7446         /* Lookup from cache. */
7447         LIST_FOREACH(cache_matcher, &tbl_data->matchers, next) {
7448                 if (matcher->crc == cache_matcher->crc &&
7449                     matcher->priority == cache_matcher->priority &&
7450                     !memcmp((const void *)matcher->mask.buf,
7451                             (const void *)cache_matcher->mask.buf,
7452                             cache_matcher->mask.size)) {
7453                         DRV_LOG(DEBUG,
7454                                 "%s group %u priority %hd use %s "
7455                                 "matcher %p: refcnt %d++",
7456                                 key->domain ? "FDB" : "NIC", key->table_id,
7457                                 cache_matcher->priority,
7458                                 key->direction ? "tx" : "rx",
7459                                 (void *)cache_matcher,
7460                                 rte_atomic32_read(&cache_matcher->refcnt));
7461                         rte_atomic32_inc(&cache_matcher->refcnt);
7462                         dev_flow->handle->dvh.matcher = cache_matcher;
7463                         /* old matcher should not make the table ref++. */
7464                         flow_dv_tbl_resource_release(dev, tbl);
7465                         return 0;
7466                 }
7467         }
7468         /* Register new matcher. */
7469         cache_matcher = rte_calloc(__func__, 1, sizeof(*cache_matcher), 0);
7470         if (!cache_matcher) {
7471                 flow_dv_tbl_resource_release(dev, tbl);
7472                 return rte_flow_error_set(error, ENOMEM,
7473                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7474                                           "cannot allocate matcher memory");
7475         }
7476         *cache_matcher = *matcher;
7477         dv_attr.match_criteria_enable =
7478                 flow_dv_matcher_enable(cache_matcher->mask.buf);
7479         dv_attr.priority = matcher->priority;
7480         if (key->direction)
7481                 dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
7482         cache_matcher->matcher_object =
7483                 mlx5_glue->dv_create_flow_matcher(sh->ctx, &dv_attr, tbl->obj);
7484         if (!cache_matcher->matcher_object) {
7485                 rte_free(cache_matcher);
7486 #ifdef HAVE_MLX5DV_DR
7487                 flow_dv_tbl_resource_release(dev, tbl);
7488 #endif
7489                 return rte_flow_error_set(error, ENOMEM,
7490                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7491                                           NULL, "cannot create matcher");
7492         }
7493         /* Save the table information */
7494         cache_matcher->tbl = tbl;
7495         rte_atomic32_init(&cache_matcher->refcnt);
7496         /* only matcher ref++, table ref++ already done above in get API. */
7497         rte_atomic32_inc(&cache_matcher->refcnt);
7498         LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
7499         dev_flow->handle->dvh.matcher = cache_matcher;
7500         DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
7501                 key->domain ? "FDB" : "NIC", key->table_id,
7502                 cache_matcher->priority,
7503                 key->direction ? "tx" : "rx", (void *)cache_matcher,
7504                 rte_atomic32_read(&cache_matcher->refcnt));
7505         return 0;
7506 }
7507
7508 /**
7509  * Find existing tag resource or create and register a new one.
7510  *
7511  * @param dev[in, out]
7512  *   Pointer to rte_eth_dev structure.
7513  * @param[in, out] tag_be24
7514  *   Tag value in big endian then R-shift 8.
7515  * @parm[in, out] dev_flow
7516  *   Pointer to the dev_flow.
7517  * @param[out] error
7518  *   pointer to error structure.
7519  *
7520  * @return
7521  *   0 on success otherwise -errno and errno is set.
7522  */
7523 static int
7524 flow_dv_tag_resource_register
7525                         (struct rte_eth_dev *dev,
7526                          uint32_t tag_be24,
7527                          struct mlx5_flow *dev_flow,
7528                          struct rte_flow_error *error)
7529 {
7530         struct mlx5_priv *priv = dev->data->dev_private;
7531         struct mlx5_dev_ctx_shared *sh = priv->sh;
7532         struct mlx5_flow_dv_tag_resource *cache_resource;
7533         struct mlx5_hlist_entry *entry;
7534
7535         /* Lookup a matching resource from cache. */
7536         entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
7537         if (entry) {
7538                 cache_resource = container_of
7539                         (entry, struct mlx5_flow_dv_tag_resource, entry);
7540                 rte_atomic32_inc(&cache_resource->refcnt);
7541                 dev_flow->handle->dvh.rix_tag = cache_resource->idx;
7542                 dev_flow->dv.tag_resource = cache_resource;
7543                 DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
7544                         (void *)cache_resource,
7545                         rte_atomic32_read(&cache_resource->refcnt));
7546                 return 0;
7547         }
7548         /* Register new resource. */
7549         cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG],
7550                                        &dev_flow->handle->dvh.rix_tag);
7551         if (!cache_resource)
7552                 return rte_flow_error_set(error, ENOMEM,
7553                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
7554                                           "cannot allocate resource memory");
7555         cache_resource->entry.key = (uint64_t)tag_be24;
7556         cache_resource->action = mlx5_glue->dv_create_flow_action_tag(tag_be24);
7557         if (!cache_resource->action) {
7558                 rte_free(cache_resource);
7559                 return rte_flow_error_set(error, ENOMEM,
7560                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7561                                           NULL, "cannot create action");
7562         }
7563         rte_atomic32_init(&cache_resource->refcnt);
7564         rte_atomic32_inc(&cache_resource->refcnt);
7565         if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
7566                 mlx5_glue->destroy_flow_action(cache_resource->action);
7567                 rte_free(cache_resource);
7568                 return rte_flow_error_set(error, EEXIST,
7569                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
7570                                           NULL, "cannot insert tag");
7571         }
7572         dev_flow->dv.tag_resource = cache_resource;
7573         DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
7574                 (void *)cache_resource,
7575                 rte_atomic32_read(&cache_resource->refcnt));
7576         return 0;
7577 }
7578
7579 /**
7580  * Release the tag.
7581  *
7582  * @param dev
7583  *   Pointer to Ethernet device.
7584  * @param tag_idx
7585  *   Tag index.
7586  *
7587  * @return
7588  *   1 while a reference on it exists, 0 when freed.
7589  */
7590 static int
7591 flow_dv_tag_release(struct rte_eth_dev *dev,
7592                     uint32_t tag_idx)
7593 {
7594         struct mlx5_priv *priv = dev->data->dev_private;
7595         struct mlx5_dev_ctx_shared *sh = priv->sh;
7596         struct mlx5_flow_dv_tag_resource *tag;
7597
7598         tag = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
7599         if (!tag)
7600                 return 0;
7601         DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
7602                 dev->data->port_id, (void *)tag,
7603                 rte_atomic32_read(&tag->refcnt));
7604         if (rte_atomic32_dec_and_test(&tag->refcnt)) {
7605                 claim_zero(mlx5_glue->destroy_flow_action(tag->action));
7606                 mlx5_hlist_remove(sh->tag_table, &tag->entry);
7607                 DRV_LOG(DEBUG, "port %u tag %p: removed",
7608                         dev->data->port_id, (void *)tag);
7609                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TAG], tag_idx);
7610                 return 0;
7611         }
7612         return 1;
7613 }
7614
7615 /**
7616  * Translate port ID action to vport.
7617  *
7618  * @param[in] dev
7619  *   Pointer to rte_eth_dev structure.
7620  * @param[in] action
7621  *   Pointer to the port ID action.
7622  * @param[out] dst_port_id
7623  *   The target port ID.
7624  * @param[out] error
7625  *   Pointer to the error structure.
7626  *
7627  * @return
7628  *   0 on success, a negative errno value otherwise and rte_errno is set.
7629  */
7630 static int
7631 flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
7632                                  const struct rte_flow_action *action,
7633                                  uint32_t *dst_port_id,
7634                                  struct rte_flow_error *error)
7635 {
7636         uint32_t port;
7637         struct mlx5_priv *priv;
7638         const struct rte_flow_action_port_id *conf =
7639                         (const struct rte_flow_action_port_id *)action->conf;
7640
7641         port = conf->original ? dev->data->port_id : conf->id;
7642         priv = mlx5_port_to_eswitch_info(port, false);
7643         if (!priv)
7644                 return rte_flow_error_set(error, -rte_errno,
7645                                           RTE_FLOW_ERROR_TYPE_ACTION,
7646                                           NULL,
7647                                           "No eswitch info was found for port");
7648 #ifdef HAVE_MLX5DV_DR_DEVX_PORT
7649         /*
7650          * This parameter is transferred to
7651          * mlx5dv_dr_action_create_dest_ib_port().
7652          */
7653         *dst_port_id = priv->dev_port;
7654 #else
7655         /*
7656          * Legacy mode, no LAG configurations is supported.
7657          * This parameter is transferred to
7658          * mlx5dv_dr_action_create_dest_vport().
7659          */
7660         *dst_port_id = priv->vport_id;
7661 #endif
7662         return 0;
7663 }
7664
7665 /**
7666  * Create a counter with aging configuration.
7667  *
7668  * @param[in] dev
7669  *   Pointer to rte_eth_dev structure.
7670  * @param[out] count
7671  *   Pointer to the counter action configuration.
7672  * @param[in] age
7673  *   Pointer to the aging action configuration.
7674  *
7675  * @return
7676  *   Index to flow counter on success, 0 otherwise.
7677  */
7678 static uint32_t
7679 flow_dv_translate_create_counter(struct rte_eth_dev *dev,
7680                                 struct mlx5_flow *dev_flow,
7681                                 const struct rte_flow_action_count *count,
7682                                 const struct rte_flow_action_age *age)
7683 {
7684         uint32_t counter;
7685         struct mlx5_age_param *age_param;
7686
7687         counter = flow_dv_counter_alloc(dev,
7688                                 count ? count->shared : 0,
7689                                 count ? count->id : 0,
7690                                 dev_flow->dv.group, !!age);
7691         if (!counter || age == NULL)
7692                 return counter;
7693         age_param  = flow_dv_counter_idx_get_age(dev, counter);
7694         /*
7695          * The counter age accuracy may have a bit delay. Have 3/4
7696          * second bias on the timeount in order to let it age in time.
7697          */
7698         age_param->context = age->context ? age->context :
7699                 (void *)(uintptr_t)(dev_flow->flow_idx);
7700         /*
7701          * The counter age accuracy may have a bit delay. Have 3/4
7702          * second bias on the timeount in order to let it age in time.
7703          */
7704         age_param->timeout = age->timeout * 10 - MLX5_AGING_TIME_DELAY;
7705         /* Set expire time in unit of 0.1 sec. */
7706         age_param->port_id = dev->data->port_id;
7707         age_param->expire = age_param->timeout +
7708                         rte_rdtsc() / (rte_get_tsc_hz() / 10);
7709         rte_atomic16_set(&age_param->state, AGE_CANDIDATE);
7710         return counter;
7711 }
7712 /**
7713  * Add Tx queue matcher
7714  *
7715  * @param[in] dev
7716  *   Pointer to the dev struct.
7717  * @param[in, out] matcher
7718  *   Flow matcher.
7719  * @param[in, out] key
7720  *   Flow matcher value.
7721  * @param[in] item
7722  *   Flow pattern to translate.
7723  * @param[in] inner
7724  *   Item is inner pattern.
7725  */
7726 static void
7727 flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
7728                                 void *matcher, void *key,
7729                                 const struct rte_flow_item *item)
7730 {
7731         const struct mlx5_rte_flow_item_tx_queue *queue_m;
7732         const struct mlx5_rte_flow_item_tx_queue *queue_v;
7733         void *misc_m =
7734                 MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
7735         void *misc_v =
7736                 MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
7737         struct mlx5_txq_ctrl *txq;
7738         uint32_t queue;
7739
7740
7741         queue_m = (const void *)item->mask;
7742         if (!queue_m)
7743                 return;
7744         queue_v = (const void *)item->spec;
7745         if (!queue_v)
7746                 return;
7747         txq = mlx5_txq_get(dev, queue_v->queue);
7748         if (!txq)
7749                 return;
7750         queue = txq->obj->sq->id;
7751         MLX5_SET(fte_match_set_misc, misc_m, source_sqn, queue_m->queue);
7752         MLX5_SET(fte_match_set_misc, misc_v, source_sqn,
7753                  queue & queue_m->queue);
7754         mlx5_txq_release(dev, queue_v->queue);
7755 }
7756
7757 /**
7758  * Set the hash fields according to the @p flow information.
7759  *
7760  * @param[in] dev_flow
7761  *   Pointer to the mlx5_flow.
7762  * @param[in] rss_desc
7763  *   Pointer to the mlx5_flow_rss_desc.
7764  */
7765 static void
7766 flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
7767                        struct mlx5_flow_rss_desc *rss_desc)
7768 {
7769         uint64_t items = dev_flow->handle->layers;
7770         int rss_inner = 0;
7771         uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
7772
7773         dev_flow->hash_fields = 0;
7774 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
7775         if (rss_desc->level >= 2) {
7776                 dev_flow->hash_fields |= IBV_RX_HASH_INNER;
7777                 rss_inner = 1;
7778         }
7779 #endif
7780         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV4)) ||
7781             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV4))) {
7782                 if (rss_types & MLX5_IPV4_LAYER_TYPES) {
7783                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
7784                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV4;
7785                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
7786                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV4;
7787                         else
7788                                 dev_flow->hash_fields |= MLX5_IPV4_IBV_RX_HASH;
7789                 }
7790         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L3_IPV6)) ||
7791                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L3_IPV6))) {
7792                 if (rss_types & MLX5_IPV6_LAYER_TYPES) {
7793                         if (rss_types & ETH_RSS_L3_SRC_ONLY)
7794                                 dev_flow->hash_fields |= IBV_RX_HASH_SRC_IPV6;
7795                         else if (rss_types & ETH_RSS_L3_DST_ONLY)
7796                                 dev_flow->hash_fields |= IBV_RX_HASH_DST_IPV6;
7797                         else
7798                                 dev_flow->hash_fields |= MLX5_IPV6_IBV_RX_HASH;
7799                 }
7800         }
7801         if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
7802             (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP))) {
7803                 if (rss_types & ETH_RSS_UDP) {
7804                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
7805                                 dev_flow->hash_fields |=
7806                                                 IBV_RX_HASH_SRC_PORT_UDP;
7807                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
7808                                 dev_flow->hash_fields |=
7809                                                 IBV_RX_HASH_DST_PORT_UDP;
7810                         else
7811                                 dev_flow->hash_fields |= MLX5_UDP_IBV_RX_HASH;
7812                 }
7813         } else if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_TCP)) ||
7814                    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_TCP))) {
7815                 if (rss_types & ETH_RSS_TCP) {
7816                         if (rss_types & ETH_RSS_L4_SRC_ONLY)
7817                                 dev_flow->hash_fields |=
7818                                                 IBV_RX_HASH_SRC_PORT_TCP;
7819                         else if (rss_types & ETH_RSS_L4_DST_ONLY)
7820                                 dev_flow->hash_fields |=
7821                                                 IBV_RX_HASH_DST_PORT_TCP;
7822                         else
7823                                 dev_flow->hash_fields |= MLX5_TCP_IBV_RX_HASH;
7824                 }
7825         }
7826 }
7827
7828 /**
7829  * Fill the flow with DV spec, lock free
7830  * (mutex should be acquired by caller).
7831  *
7832  * @param[in] dev
7833  *   Pointer to rte_eth_dev structure.
7834  * @param[in, out] dev_flow
7835  *   Pointer to the sub flow.
7836  * @param[in] attr
7837  *   Pointer to the flow attributes.
7838  * @param[in] items
7839  *   Pointer to the list of items.
7840  * @param[in] actions
7841  *   Pointer to the list of actions.
7842  * @param[out] error
7843  *   Pointer to the error structure.
7844  *
7845  * @return
7846  *   0 on success, a negative errno value otherwise and rte_errno is set.
7847  */
7848 static int
7849 __flow_dv_translate(struct rte_eth_dev *dev,
7850                     struct mlx5_flow *dev_flow,
7851                     const struct rte_flow_attr *attr,
7852                     const struct rte_flow_item items[],
7853                     const struct rte_flow_action actions[],
7854                     struct rte_flow_error *error)
7855 {
7856         struct mlx5_priv *priv = dev->data->dev_private;
7857         struct mlx5_dev_config *dev_conf = &priv->config;
7858         struct rte_flow *flow = dev_flow->flow;
7859         struct mlx5_flow_handle *handle = dev_flow->handle;
7860         struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
7861                                               priv->rss_desc)
7862                                               [!!priv->flow_nested_idx];
7863         uint64_t item_flags = 0;
7864         uint64_t last_item = 0;
7865         uint64_t action_flags = 0;
7866         uint64_t priority = attr->priority;
7867         struct mlx5_flow_dv_matcher matcher = {
7868                 .mask = {
7869                         .size = sizeof(matcher.mask.buf),
7870                 },
7871         };
7872         int actions_n = 0;
7873         bool actions_end = false;
7874         union {
7875                 struct mlx5_flow_dv_modify_hdr_resource res;
7876                 uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
7877                             sizeof(struct mlx5_modification_cmd) *
7878                             (MLX5_MAX_MODIFY_NUM + 1)];
7879         } mhdr_dummy;
7880         struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
7881         const struct rte_flow_action_count *count = NULL;
7882         const struct rte_flow_action_age *age = NULL;
7883         union flow_dv_attr flow_attr = { .attr = 0 };
7884         uint32_t tag_be;
7885         union mlx5_flow_tbl_key tbl_key;
7886         uint32_t modify_action_position = UINT32_MAX;
7887         void *match_mask = matcher.mask.buf;
7888         void *match_value = dev_flow->dv.value.buf;
7889         uint8_t next_protocol = 0xff;
7890         struct rte_vlan_hdr vlan = { 0 };
7891         uint32_t table;
7892         int ret = 0;
7893
7894         mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
7895                                            MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
7896         ret = mlx5_flow_group_to_table(attr, dev_flow->external, attr->group,
7897                                        !!priv->fdb_def_rule, &table, error);
7898         if (ret)
7899                 return ret;
7900         dev_flow->dv.group = table;
7901         if (attr->transfer)
7902                 mhdr_res->ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB;
7903         if (priority == MLX5_FLOW_PRIO_RSVD)
7904                 priority = dev_conf->flow_prio - 1;
7905         /* number of actions must be set to 0 in case of dirty stack. */
7906         mhdr_res->actions_num = 0;
7907         for (; !actions_end ; actions++) {
7908                 const struct rte_flow_action_queue *queue;
7909                 const struct rte_flow_action_rss *rss;
7910                 const struct rte_flow_action *action = actions;
7911                 const uint8_t *rss_key;
7912                 const struct rte_flow_action_jump *jump_data;
7913                 const struct rte_flow_action_meter *mtr;
7914                 struct mlx5_flow_tbl_resource *tbl;
7915                 uint32_t port_id = 0;
7916                 struct mlx5_flow_dv_port_id_action_resource port_id_resource;
7917                 int action_type = actions->type;
7918                 const struct rte_flow_action *found_action = NULL;
7919                 struct mlx5_flow_meter *fm = NULL;
7920
7921                 if (!mlx5_flow_os_action_supported(action_type))
7922                         return rte_flow_error_set(error, ENOTSUP,
7923                                                   RTE_FLOW_ERROR_TYPE_ACTION,
7924                                                   actions,
7925                                                   "action not supported");
7926                 switch (action_type) {
7927                 case RTE_FLOW_ACTION_TYPE_VOID:
7928                         break;
7929                 case RTE_FLOW_ACTION_TYPE_PORT_ID:
7930                         if (flow_dv_translate_action_port_id(dev, action,
7931                                                              &port_id, error))
7932                                 return -rte_errno;
7933                         port_id_resource.port_id = port_id;
7934                         MLX5_ASSERT(!handle->rix_port_id_action);
7935                         if (flow_dv_port_id_action_resource_register
7936                             (dev, &port_id_resource, dev_flow, error))
7937                                 return -rte_errno;
7938                         dev_flow->dv.actions[actions_n++] =
7939                                         dev_flow->dv.port_id_action->action;
7940                         action_flags |= MLX5_FLOW_ACTION_PORT_ID;
7941                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_PORT_ID;
7942                         break;
7943                 case RTE_FLOW_ACTION_TYPE_FLAG:
7944                         action_flags |= MLX5_FLOW_ACTION_FLAG;
7945                         dev_flow->handle->mark = 1;
7946                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7947                                 struct rte_flow_action_mark mark = {
7948                                         .id = MLX5_FLOW_MARK_DEFAULT,
7949                                 };
7950
7951                                 if (flow_dv_convert_action_mark(dev, &mark,
7952                                                                 mhdr_res,
7953                                                                 error))
7954                                         return -rte_errno;
7955                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
7956                                 break;
7957                         }
7958                         tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
7959                         /*
7960                          * Only one FLAG or MARK is supported per device flow
7961                          * right now. So the pointer to the tag resource must be
7962                          * zero before the register process.
7963                          */
7964                         MLX5_ASSERT(!handle->dvh.rix_tag);
7965                         if (flow_dv_tag_resource_register(dev, tag_be,
7966                                                           dev_flow, error))
7967                                 return -rte_errno;
7968                         MLX5_ASSERT(dev_flow->dv.tag_resource);
7969                         dev_flow->dv.actions[actions_n++] =
7970                                         dev_flow->dv.tag_resource->action;
7971                         break;
7972                 case RTE_FLOW_ACTION_TYPE_MARK:
7973                         action_flags |= MLX5_FLOW_ACTION_MARK;
7974                         dev_flow->handle->mark = 1;
7975                         if (dev_conf->dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
7976                                 const struct rte_flow_action_mark *mark =
7977                                         (const struct rte_flow_action_mark *)
7978                                                 actions->conf;
7979
7980                                 if (flow_dv_convert_action_mark(dev, mark,
7981                                                                 mhdr_res,
7982                                                                 error))
7983                                         return -rte_errno;
7984                                 action_flags |= MLX5_FLOW_ACTION_MARK_EXT;
7985                                 break;
7986                         }
7987                         /* Fall-through */
7988                 case MLX5_RTE_FLOW_ACTION_TYPE_MARK:
7989                         /* Legacy (non-extensive) MARK action. */
7990                         tag_be = mlx5_flow_mark_set
7991                               (((const struct rte_flow_action_mark *)
7992                                (actions->conf))->id);
7993                         MLX5_ASSERT(!handle->dvh.rix_tag);
7994                         if (flow_dv_tag_resource_register(dev, tag_be,
7995                                                           dev_flow, error))
7996                                 return -rte_errno;
7997                         MLX5_ASSERT(dev_flow->dv.tag_resource);
7998                         dev_flow->dv.actions[actions_n++] =
7999                                         dev_flow->dv.tag_resource->action;
8000                         break;
8001                 case RTE_FLOW_ACTION_TYPE_SET_META:
8002                         if (flow_dv_convert_action_set_meta
8003                                 (dev, mhdr_res, attr,
8004                                  (const struct rte_flow_action_set_meta *)
8005                                   actions->conf, error))
8006                                 return -rte_errno;
8007                         action_flags |= MLX5_FLOW_ACTION_SET_META;
8008                         break;
8009                 case RTE_FLOW_ACTION_TYPE_SET_TAG:
8010                         if (flow_dv_convert_action_set_tag
8011                                 (dev, mhdr_res,
8012                                  (const struct rte_flow_action_set_tag *)
8013                                   actions->conf, error))
8014                                 return -rte_errno;
8015                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8016                         break;
8017                 case RTE_FLOW_ACTION_TYPE_DROP:
8018                         action_flags |= MLX5_FLOW_ACTION_DROP;
8019                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
8020                         break;
8021                 case RTE_FLOW_ACTION_TYPE_QUEUE:
8022                         queue = actions->conf;
8023                         rss_desc->queue_num = 1;
8024                         rss_desc->queue[0] = queue->index;
8025                         action_flags |= MLX5_FLOW_ACTION_QUEUE;
8026                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
8027                         break;
8028                 case RTE_FLOW_ACTION_TYPE_RSS:
8029                         rss = actions->conf;
8030                         memcpy(rss_desc->queue, rss->queue,
8031                                rss->queue_num * sizeof(uint16_t));
8032                         rss_desc->queue_num = rss->queue_num;
8033                         /* NULL RSS key indicates default RSS key. */
8034                         rss_key = !rss->key ? rss_hash_default_key : rss->key;
8035                         memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
8036                         /*
8037                          * rss->level and rss.types should be set in advance
8038                          * when expanding items for RSS.
8039                          */
8040                         action_flags |= MLX5_FLOW_ACTION_RSS;
8041                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
8042                         break;
8043                 case RTE_FLOW_ACTION_TYPE_AGE:
8044                 case RTE_FLOW_ACTION_TYPE_COUNT:
8045                         if (!dev_conf->devx) {
8046                                 return rte_flow_error_set
8047                                               (error, ENOTSUP,
8048                                                RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8049                                                NULL,
8050                                                "count action not supported");
8051                         }
8052                         /* Save information first, will apply later. */
8053                         if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT)
8054                                 count = action->conf;
8055                         else
8056                                 age = action->conf;
8057                         action_flags |= MLX5_FLOW_ACTION_COUNT;
8058                         break;
8059                 case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
8060                         dev_flow->dv.actions[actions_n++] =
8061                                                 priv->sh->pop_vlan_action;
8062                         action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
8063                         break;
8064                 case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
8065                         if (!(action_flags &
8066                               MLX5_FLOW_ACTION_OF_SET_VLAN_VID))
8067                                 flow_dev_get_vlan_info_from_items(items, &vlan);
8068                         vlan.eth_proto = rte_be_to_cpu_16
8069                              ((((const struct rte_flow_action_of_push_vlan *)
8070                                                    actions->conf)->ethertype));
8071                         found_action = mlx5_flow_find_action
8072                                         (actions + 1,
8073                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID);
8074                         if (found_action)
8075                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
8076                         found_action = mlx5_flow_find_action
8077                                         (actions + 1,
8078                                          RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP);
8079                         if (found_action)
8080                                 mlx5_update_vlan_vid_pcp(found_action, &vlan);
8081                         if (flow_dv_create_action_push_vlan
8082                                             (dev, attr, &vlan, dev_flow, error))
8083                                 return -rte_errno;
8084                         dev_flow->dv.actions[actions_n++] =
8085                                         dev_flow->dv.push_vlan_res->action;
8086                         action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
8087                         break;
8088                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
8089                         /* of_vlan_push action handled this action */
8090                         MLX5_ASSERT(action_flags &
8091                                     MLX5_FLOW_ACTION_OF_PUSH_VLAN);
8092                         break;
8093                 case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
8094                         if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
8095                                 break;
8096                         flow_dev_get_vlan_info_from_items(items, &vlan);
8097                         mlx5_update_vlan_vid_pcp(actions, &vlan);
8098                         /* If no VLAN push - this is a modify header action */
8099                         if (flow_dv_convert_action_modify_vlan_vid
8100                                                 (mhdr_res, actions, error))
8101                                 return -rte_errno;
8102                         action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
8103                         break;
8104                 case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
8105                 case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
8106                         if (flow_dv_create_action_l2_encap(dev, actions,
8107                                                            dev_flow,
8108                                                            attr->transfer,
8109                                                            error))
8110                                 return -rte_errno;
8111                         dev_flow->dv.actions[actions_n++] =
8112                                         dev_flow->dv.encap_decap->action;
8113                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
8114                         break;
8115                 case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
8116                 case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
8117                         if (flow_dv_create_action_l2_decap(dev, dev_flow,
8118                                                            attr->transfer,
8119                                                            error))
8120                                 return -rte_errno;
8121                         dev_flow->dv.actions[actions_n++] =
8122                                         dev_flow->dv.encap_decap->action;
8123                         action_flags |= MLX5_FLOW_ACTION_DECAP;
8124                         break;
8125                 case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
8126                         /* Handle encap with preceding decap. */
8127                         if (action_flags & MLX5_FLOW_ACTION_DECAP) {
8128                                 if (flow_dv_create_action_raw_encap
8129                                         (dev, actions, dev_flow, attr, error))
8130                                         return -rte_errno;
8131                                 dev_flow->dv.actions[actions_n++] =
8132                                         dev_flow->dv.encap_decap->action;
8133                         } else {
8134                                 /* Handle encap without preceding decap. */
8135                                 if (flow_dv_create_action_l2_encap
8136                                     (dev, actions, dev_flow, attr->transfer,
8137                                      error))
8138                                         return -rte_errno;
8139                                 dev_flow->dv.actions[actions_n++] =
8140                                         dev_flow->dv.encap_decap->action;
8141                         }
8142                         action_flags |= MLX5_FLOW_ACTION_ENCAP;
8143                         break;
8144                 case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
8145                         while ((++action)->type == RTE_FLOW_ACTION_TYPE_VOID)
8146                                 ;
8147                         if (action->type != RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
8148                                 if (flow_dv_create_action_l2_decap
8149                                     (dev, dev_flow, attr->transfer, error))
8150                                         return -rte_errno;
8151                                 dev_flow->dv.actions[actions_n++] =
8152                                         dev_flow->dv.encap_decap->action;
8153                         }
8154                         /* If decap is followed by encap, handle it at encap. */
8155                         action_flags |= MLX5_FLOW_ACTION_DECAP;
8156                         break;
8157                 case RTE_FLOW_ACTION_TYPE_JUMP:
8158                         jump_data = action->conf;
8159                         ret = mlx5_flow_group_to_table(attr, dev_flow->external,
8160                                                        jump_data->group,
8161                                                        !!priv->fdb_def_rule,
8162                                                        &table, error);
8163                         if (ret)
8164                                 return ret;
8165                         tbl = flow_dv_tbl_resource_get(dev, table,
8166                                                        attr->egress,
8167                                                        attr->transfer, error);
8168                         if (!tbl)
8169                                 return rte_flow_error_set
8170                                                 (error, errno,
8171                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8172                                                  NULL,
8173                                                  "cannot create jump action.");
8174                         if (flow_dv_jump_tbl_resource_register
8175                             (dev, tbl, dev_flow, error)) {
8176                                 flow_dv_tbl_resource_release(dev, tbl);
8177                                 return rte_flow_error_set
8178                                                 (error, errno,
8179                                                  RTE_FLOW_ERROR_TYPE_ACTION,
8180                                                  NULL,
8181                                                  "cannot create jump action.");
8182                         }
8183                         dev_flow->dv.actions[actions_n++] =
8184                                         dev_flow->dv.jump->action;
8185                         action_flags |= MLX5_FLOW_ACTION_JUMP;
8186                         dev_flow->handle->fate_action = MLX5_FLOW_FATE_JUMP;
8187                         break;
8188                 case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
8189                 case RTE_FLOW_ACTION_TYPE_SET_MAC_DST:
8190                         if (flow_dv_convert_action_modify_mac
8191                                         (mhdr_res, actions, error))
8192                                 return -rte_errno;
8193                         action_flags |= actions->type ==
8194                                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC ?
8195                                         MLX5_FLOW_ACTION_SET_MAC_SRC :
8196                                         MLX5_FLOW_ACTION_SET_MAC_DST;
8197                         break;
8198                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC:
8199                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DST:
8200                         if (flow_dv_convert_action_modify_ipv4
8201                                         (mhdr_res, actions, error))
8202                                 return -rte_errno;
8203                         action_flags |= actions->type ==
8204                                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC ?
8205                                         MLX5_FLOW_ACTION_SET_IPV4_SRC :
8206                                         MLX5_FLOW_ACTION_SET_IPV4_DST;
8207                         break;
8208                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC:
8209                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DST:
8210                         if (flow_dv_convert_action_modify_ipv6
8211                                         (mhdr_res, actions, error))
8212                                 return -rte_errno;
8213                         action_flags |= actions->type ==
8214                                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC ?
8215                                         MLX5_FLOW_ACTION_SET_IPV6_SRC :
8216                                         MLX5_FLOW_ACTION_SET_IPV6_DST;
8217                         break;
8218                 case RTE_FLOW_ACTION_TYPE_SET_TP_SRC:
8219                 case RTE_FLOW_ACTION_TYPE_SET_TP_DST:
8220                         if (flow_dv_convert_action_modify_tp
8221                                         (mhdr_res, actions, items,
8222                                          &flow_attr, dev_flow, !!(action_flags &
8223                                          MLX5_FLOW_ACTION_DECAP), error))
8224                                 return -rte_errno;
8225                         action_flags |= actions->type ==
8226                                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC ?
8227                                         MLX5_FLOW_ACTION_SET_TP_SRC :
8228                                         MLX5_FLOW_ACTION_SET_TP_DST;
8229                         break;
8230                 case RTE_FLOW_ACTION_TYPE_DEC_TTL:
8231                         if (flow_dv_convert_action_modify_dec_ttl
8232                                         (mhdr_res, items, &flow_attr, dev_flow,
8233                                          !!(action_flags &
8234                                          MLX5_FLOW_ACTION_DECAP), error))
8235                                 return -rte_errno;
8236                         action_flags |= MLX5_FLOW_ACTION_DEC_TTL;
8237                         break;
8238                 case RTE_FLOW_ACTION_TYPE_SET_TTL:
8239                         if (flow_dv_convert_action_modify_ttl
8240                                         (mhdr_res, actions, items, &flow_attr,
8241                                          dev_flow, !!(action_flags &
8242                                          MLX5_FLOW_ACTION_DECAP), error))
8243                                 return -rte_errno;
8244                         action_flags |= MLX5_FLOW_ACTION_SET_TTL;
8245                         break;
8246                 case RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ:
8247                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ:
8248                         if (flow_dv_convert_action_modify_tcp_seq
8249                                         (mhdr_res, actions, error))
8250                                 return -rte_errno;
8251                         action_flags |= actions->type ==
8252                                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ ?
8253                                         MLX5_FLOW_ACTION_INC_TCP_SEQ :
8254                                         MLX5_FLOW_ACTION_DEC_TCP_SEQ;
8255                         break;
8256
8257                 case RTE_FLOW_ACTION_TYPE_INC_TCP_ACK:
8258                 case RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK:
8259                         if (flow_dv_convert_action_modify_tcp_ack
8260                                         (mhdr_res, actions, error))
8261                                 return -rte_errno;
8262                         action_flags |= actions->type ==
8263                                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK ?
8264                                         MLX5_FLOW_ACTION_INC_TCP_ACK :
8265                                         MLX5_FLOW_ACTION_DEC_TCP_ACK;
8266                         break;
8267                 case MLX5_RTE_FLOW_ACTION_TYPE_TAG:
8268                         if (flow_dv_convert_action_set_reg
8269                                         (mhdr_res, actions, error))
8270                                 return -rte_errno;
8271                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8272                         break;
8273                 case MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG:
8274                         if (flow_dv_convert_action_copy_mreg
8275                                         (dev, mhdr_res, actions, error))
8276                                 return -rte_errno;
8277                         action_flags |= MLX5_FLOW_ACTION_SET_TAG;
8278                         break;
8279                 case MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS:
8280                         action_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
8281                         dev_flow->handle->fate_action =
8282                                         MLX5_FLOW_FATE_DEFAULT_MISS;
8283                         break;
8284                 case RTE_FLOW_ACTION_TYPE_METER:
8285                         mtr = actions->conf;
8286                         if (!flow->meter) {
8287                                 fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
8288                                                             attr, error);
8289                                 if (!fm)
8290                                         return rte_flow_error_set(error,
8291                                                 rte_errno,
8292                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8293                                                 NULL,
8294                                                 "meter not found "
8295                                                 "or invalid parameters");
8296                                 flow->meter = fm->idx;
8297                         }
8298                         /* Set the meter action. */
8299                         if (!fm) {
8300                                 fm = mlx5_ipool_get(priv->sh->ipool
8301                                                 [MLX5_IPOOL_MTR], flow->meter);
8302                                 if (!fm)
8303                                         return rte_flow_error_set(error,
8304                                                 rte_errno,
8305                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8306                                                 NULL,
8307                                                 "meter not found "
8308                                                 "or invalid parameters");
8309                         }
8310                         dev_flow->dv.actions[actions_n++] =
8311                                 fm->mfts->meter_action;
8312                         action_flags |= MLX5_FLOW_ACTION_METER;
8313                         break;
8314                 case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
8315                         if (flow_dv_convert_action_modify_ipv4_dscp(mhdr_res,
8316                                                               actions, error))
8317                                 return -rte_errno;
8318                         action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
8319                         break;
8320                 case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
8321                         if (flow_dv_convert_action_modify_ipv6_dscp(mhdr_res,
8322                                                               actions, error))
8323                                 return -rte_errno;
8324                         action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
8325                         break;
8326                 case RTE_FLOW_ACTION_TYPE_END:
8327                         actions_end = true;
8328                         if (mhdr_res->actions_num) {
8329                                 /* create modify action if needed. */
8330                                 if (flow_dv_modify_hdr_resource_register
8331                                         (dev, mhdr_res, dev_flow, error))
8332                                         return -rte_errno;
8333                                 dev_flow->dv.actions[modify_action_position] =
8334                                         handle->dvh.modify_hdr->action;
8335                         }
8336                         if (action_flags & MLX5_FLOW_ACTION_COUNT) {
8337                                 flow->counter =
8338                                         flow_dv_translate_create_counter(dev,
8339                                                 dev_flow, count, age);
8340
8341                                 if (!flow->counter)
8342                                         return rte_flow_error_set
8343                                                 (error, rte_errno,
8344                                                 RTE_FLOW_ERROR_TYPE_ACTION,
8345                                                 NULL,
8346                                                 "cannot create counter"
8347                                                 " object.");
8348                                 dev_flow->dv.actions[actions_n++] =
8349                                           (flow_dv_counter_get_by_idx(dev,
8350                                           flow->counter, NULL))->action;
8351                         }
8352                         break;
8353                 default:
8354                         break;
8355                 }
8356                 if (mhdr_res->actions_num &&
8357                     modify_action_position == UINT32_MAX)
8358                         modify_action_position = actions_n++;
8359         }
8360         dev_flow->dv.actions_n = actions_n;
8361         dev_flow->act_flags = action_flags;
8362         for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
8363                 int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
8364                 int item_type = items->type;
8365
8366                 if (!mlx5_flow_os_item_supported(item_type))
8367                         return rte_flow_error_set(error, ENOTSUP,
8368                                                   RTE_FLOW_ERROR_TYPE_ITEM,
8369                                                   NULL, "item not supported");
8370                 switch (item_type) {
8371                 case RTE_FLOW_ITEM_TYPE_PORT_ID:
8372                         flow_dv_translate_item_port_id(dev, match_mask,
8373                                                        match_value, items);
8374                         last_item = MLX5_FLOW_ITEM_PORT_ID;
8375                         break;
8376                 case RTE_FLOW_ITEM_TYPE_ETH:
8377                         flow_dv_translate_item_eth(match_mask, match_value,
8378                                                    items, tunnel,
8379                                                    dev_flow->dv.group);
8380                         matcher.priority = action_flags &
8381                                         MLX5_FLOW_ACTION_DEFAULT_MISS &&
8382                                         !dev_flow->external ?
8383                                         MLX5_PRIORITY_MAP_L3 :
8384                                         MLX5_PRIORITY_MAP_L2;
8385                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 :
8386                                              MLX5_FLOW_LAYER_OUTER_L2;
8387                         break;
8388                 case RTE_FLOW_ITEM_TYPE_VLAN:
8389                         flow_dv_translate_item_vlan(dev_flow,
8390                                                     match_mask, match_value,
8391                                                     items, tunnel,
8392                                                     dev_flow->dv.group);
8393                         matcher.priority = MLX5_PRIORITY_MAP_L2;
8394                         last_item = tunnel ? (MLX5_FLOW_LAYER_INNER_L2 |
8395                                               MLX5_FLOW_LAYER_INNER_VLAN) :
8396                                              (MLX5_FLOW_LAYER_OUTER_L2 |
8397                                               MLX5_FLOW_LAYER_OUTER_VLAN);
8398                         break;
8399                 case RTE_FLOW_ITEM_TYPE_IPV4:
8400                         mlx5_flow_tunnel_ip_check(items, next_protocol,
8401                                                   &item_flags, &tunnel);
8402                         flow_dv_translate_item_ipv4(match_mask, match_value,
8403                                                     items, item_flags, tunnel,
8404                                                     dev_flow->dv.group);
8405                         matcher.priority = MLX5_PRIORITY_MAP_L3;
8406                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
8407                                              MLX5_FLOW_LAYER_OUTER_L3_IPV4;
8408                         if (items->mask != NULL &&
8409                             ((const struct rte_flow_item_ipv4 *)
8410                              items->mask)->hdr.next_proto_id) {
8411                                 next_protocol =
8412                                         ((const struct rte_flow_item_ipv4 *)
8413                                          (items->spec))->hdr.next_proto_id;
8414                                 next_protocol &=
8415                                         ((const struct rte_flow_item_ipv4 *)
8416                                          (items->mask))->hdr.next_proto_id;
8417                         } else {
8418                                 /* Reset for inner layer. */
8419                                 next_protocol = 0xff;
8420                         }
8421                         break;
8422                 case RTE_FLOW_ITEM_TYPE_IPV6:
8423                         mlx5_flow_tunnel_ip_check(items, next_protocol,
8424                                                   &item_flags, &tunnel);
8425                         flow_dv_translate_item_ipv6(match_mask, match_value,
8426                                                     items, item_flags, tunnel,
8427                                                     dev_flow->dv.group);
8428                         matcher.priority = MLX5_PRIORITY_MAP_L3;
8429                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
8430                                              MLX5_FLOW_LAYER_OUTER_L3_IPV6;
8431                         if (items->mask != NULL &&
8432                             ((const struct rte_flow_item_ipv6 *)
8433                              items->mask)->hdr.proto) {
8434                                 next_protocol =
8435                                         ((const struct rte_flow_item_ipv6 *)
8436                                          items->spec)->hdr.proto;
8437                                 next_protocol &=
8438                                         ((const struct rte_flow_item_ipv6 *)
8439                                          items->mask)->hdr.proto;
8440                         } else {
8441                                 /* Reset for inner layer. */
8442                                 next_protocol = 0xff;
8443                         }
8444                         break;
8445                 case RTE_FLOW_ITEM_TYPE_TCP:
8446                         flow_dv_translate_item_tcp(match_mask, match_value,
8447                                                    items, tunnel);
8448                         matcher.priority = MLX5_PRIORITY_MAP_L4;
8449                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
8450                                              MLX5_FLOW_LAYER_OUTER_L4_TCP;
8451                         break;
8452                 case RTE_FLOW_ITEM_TYPE_UDP:
8453                         flow_dv_translate_item_udp(match_mask, match_value,
8454                                                    items, tunnel);
8455                         matcher.priority = MLX5_PRIORITY_MAP_L4;
8456                         last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
8457                                              MLX5_FLOW_LAYER_OUTER_L4_UDP;
8458                         break;
8459                 case RTE_FLOW_ITEM_TYPE_GRE:
8460                         flow_dv_translate_item_gre(match_mask, match_value,
8461                                                    items, tunnel);
8462                         matcher.priority = rss_desc->level >= 2 ?
8463                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8464                         last_item = MLX5_FLOW_LAYER_GRE;
8465                         break;
8466                 case RTE_FLOW_ITEM_TYPE_GRE_KEY:
8467                         flow_dv_translate_item_gre_key(match_mask,
8468                                                        match_value, items);
8469                         last_item = MLX5_FLOW_LAYER_GRE_KEY;
8470                         break;
8471                 case RTE_FLOW_ITEM_TYPE_NVGRE:
8472                         flow_dv_translate_item_nvgre(match_mask, match_value,
8473                                                      items, tunnel);
8474                         matcher.priority = rss_desc->level >= 2 ?
8475                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8476                         last_item = MLX5_FLOW_LAYER_GRE;
8477                         break;
8478                 case RTE_FLOW_ITEM_TYPE_VXLAN:
8479                         flow_dv_translate_item_vxlan(match_mask, match_value,
8480                                                      items, tunnel);
8481                         matcher.priority = rss_desc->level >= 2 ?
8482                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8483                         last_item = MLX5_FLOW_LAYER_VXLAN;
8484                         break;
8485                 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
8486                         flow_dv_translate_item_vxlan_gpe(match_mask,
8487                                                          match_value, items,
8488                                                          tunnel);
8489                         matcher.priority = rss_desc->level >= 2 ?
8490                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8491                         last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
8492                         break;
8493                 case RTE_FLOW_ITEM_TYPE_GENEVE:
8494                         flow_dv_translate_item_geneve(match_mask, match_value,
8495                                                       items, tunnel);
8496                         matcher.priority = rss_desc->level >= 2 ?
8497                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8498                         last_item = MLX5_FLOW_LAYER_GENEVE;
8499                         break;
8500                 case RTE_FLOW_ITEM_TYPE_MPLS:
8501                         flow_dv_translate_item_mpls(match_mask, match_value,
8502                                                     items, last_item, tunnel);
8503                         matcher.priority = rss_desc->level >= 2 ?
8504                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8505                         last_item = MLX5_FLOW_LAYER_MPLS;
8506                         break;
8507                 case RTE_FLOW_ITEM_TYPE_MARK:
8508                         flow_dv_translate_item_mark(dev, match_mask,
8509                                                     match_value, items);
8510                         last_item = MLX5_FLOW_ITEM_MARK;
8511                         break;
8512                 case RTE_FLOW_ITEM_TYPE_META:
8513                         flow_dv_translate_item_meta(dev, match_mask,
8514                                                     match_value, attr, items);
8515                         last_item = MLX5_FLOW_ITEM_METADATA;
8516                         break;
8517                 case RTE_FLOW_ITEM_TYPE_ICMP:
8518                         flow_dv_translate_item_icmp(match_mask, match_value,
8519                                                     items, tunnel);
8520                         last_item = MLX5_FLOW_LAYER_ICMP;
8521                         break;
8522                 case RTE_FLOW_ITEM_TYPE_ICMP6:
8523                         flow_dv_translate_item_icmp6(match_mask, match_value,
8524                                                       items, tunnel);
8525                         last_item = MLX5_FLOW_LAYER_ICMP6;
8526                         break;
8527                 case RTE_FLOW_ITEM_TYPE_TAG:
8528                         flow_dv_translate_item_tag(dev, match_mask,
8529                                                    match_value, items);
8530                         last_item = MLX5_FLOW_ITEM_TAG;
8531                         break;
8532                 case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
8533                         flow_dv_translate_mlx5_item_tag(dev, match_mask,
8534                                                         match_value, items);
8535                         last_item = MLX5_FLOW_ITEM_TAG;
8536                         break;
8537                 case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
8538                         flow_dv_translate_item_tx_queue(dev, match_mask,
8539                                                         match_value,
8540                                                         items);
8541                         last_item = MLX5_FLOW_ITEM_TX_QUEUE;
8542                         break;
8543                 case RTE_FLOW_ITEM_TYPE_GTP:
8544                         flow_dv_translate_item_gtp(match_mask, match_value,
8545                                                    items, tunnel);
8546                         matcher.priority = rss_desc->level >= 2 ?
8547                                     MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
8548                         last_item = MLX5_FLOW_LAYER_GTP;
8549                         break;
8550                 default:
8551                         break;
8552                 }
8553                 item_flags |= last_item;
8554         }
8555         /*
8556          * When E-Switch mode is enabled, we have two cases where we need to
8557          * set the source port manually.
8558          * The first one, is in case of Nic steering rule, and the second is
8559          * E-Switch rule where no port_id item was found. In both cases
8560          * the source port is set according the current port in use.
8561          */
8562         if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
8563             (priv->representor || priv->master)) {
8564                 if (flow_dv_translate_item_port_id(dev, match_mask,
8565                                                    match_value, NULL))
8566                         return -rte_errno;
8567         }
8568 #ifdef RTE_LIBRTE_MLX5_DEBUG
8569         MLX5_ASSERT(!flow_dv_check_valid_spec(matcher.mask.buf,
8570                                               dev_flow->dv.value.buf));
8571 #endif
8572         /*
8573          * Layers may be already initialized from prefix flow if this dev_flow
8574          * is the suffix flow.
8575          */
8576         handle->layers |= item_flags;
8577         if (action_flags & MLX5_FLOW_ACTION_RSS)
8578                 flow_dv_hashfields_set(dev_flow, rss_desc);
8579         /* Register matcher. */
8580         matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
8581                                     matcher.mask.size);
8582         matcher.priority = mlx5_flow_adjust_priority(dev, priority,
8583                                                      matcher.priority);
8584         /* reserved field no needs to be set to 0 here. */
8585         tbl_key.domain = attr->transfer;
8586         tbl_key.direction = attr->egress;
8587         tbl_key.table_id = dev_flow->dv.group;
8588         if (flow_dv_matcher_register(dev, &matcher, &tbl_key, dev_flow, error))
8589                 return -rte_errno;
8590         return 0;
8591 }
8592
8593 /**
8594  * Apply the flow to the NIC, lock free,
8595  * (mutex should be acquired by caller).
8596  *
8597  * @param[in] dev
8598  *   Pointer to the Ethernet device structure.
8599  * @param[in, out] flow
8600  *   Pointer to flow structure.
8601  * @param[out] error
8602  *   Pointer to error structure.
8603  *
8604  * @return
8605  *   0 on success, a negative errno value otherwise and rte_errno is set.
8606  */
8607 static int
8608 __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
8609                 struct rte_flow_error *error)
8610 {
8611         struct mlx5_flow_dv_workspace *dv;
8612         struct mlx5_flow_handle *dh;
8613         struct mlx5_flow_handle_dv *dv_h;
8614         struct mlx5_flow *dev_flow;
8615         struct mlx5_priv *priv = dev->data->dev_private;
8616         uint32_t handle_idx;
8617         int n;
8618         int err;
8619         int idx;
8620
8621         for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
8622                 dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
8623                 dv = &dev_flow->dv;
8624                 dh = dev_flow->handle;
8625                 dv_h = &dh->dvh;
8626                 n = dv->actions_n;
8627                 if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
8628                         if (dv->transfer) {
8629                                 dv->actions[n++] = priv->sh->esw_drop_action;
8630                         } else {
8631                                 struct mlx5_hrxq *drop_hrxq;
8632                                 drop_hrxq = mlx5_hrxq_drop_new(dev);
8633                                 if (!drop_hrxq) {
8634                                         rte_flow_error_set
8635                                                 (error, errno,
8636                                                  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8637                                                  NULL,
8638                                                  "cannot get drop hash queue");
8639                                         goto error;
8640                                 }
8641                                 /*
8642                                  * Drop queues will be released by the specify
8643                                  * mlx5_hrxq_drop_release() function. Assign
8644                                  * the special index to hrxq to mark the queue
8645                                  * has been allocated.
8646                                  */
8647                                 dh->rix_hrxq = UINT32_MAX;
8648                                 dv->actions[n++] = drop_hrxq->action;
8649                         }
8650                 } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
8651                         struct mlx5_hrxq *hrxq;
8652                         uint32_t hrxq_idx;
8653                         struct mlx5_flow_rss_desc *rss_desc =
8654                                 &((struct mlx5_flow_rss_desc *)priv->rss_desc)
8655                                 [!!priv->flow_nested_idx];
8656
8657                         MLX5_ASSERT(rss_desc->queue_num);
8658                         hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
8659                                                  MLX5_RSS_HASH_KEY_LEN,
8660                                                  dev_flow->hash_fields,
8661                                                  rss_desc->queue,
8662                                                  rss_desc->queue_num);
8663                         if (!hrxq_idx) {
8664                                 hrxq_idx = mlx5_hrxq_new
8665                                                 (dev, rss_desc->key,
8666                                                 MLX5_RSS_HASH_KEY_LEN,
8667                                                 dev_flow->hash_fields,
8668                                                 rss_desc->queue,
8669                                                 rss_desc->queue_num,
8670                                                 !!(dh->layers &
8671                                                 MLX5_FLOW_LAYER_TUNNEL));
8672                         }
8673                         hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
8674                                               hrxq_idx);
8675                         if (!hrxq) {
8676                                 rte_flow_error_set
8677                                         (error, rte_errno,
8678                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8679                                          "cannot get hash queue");
8680                                 goto error;
8681                         }
8682                         dh->rix_hrxq = hrxq_idx;
8683                         dv->actions[n++] = hrxq->action;
8684                 } else if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS) {
8685                         if (flow_dv_default_miss_resource_register
8686                                         (dev, error)) {
8687                                 rte_flow_error_set
8688                                         (error, rte_errno,
8689                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
8690                                          "cannot create default miss resource");
8691                                 goto error_default_miss;
8692                         }
8693                         dh->rix_default_fate =  MLX5_FLOW_FATE_DEFAULT_MISS;
8694                         dv->actions[n++] = priv->sh->default_miss.action;
8695                 }
8696                 dh->drv_flow =
8697                         mlx5_glue->dv_create_flow(dv_h->matcher->matcher_object,
8698                                                   (void *)&dv->value, n,
8699                                                   dv->actions);
8700                 if (!dh->drv_flow) {
8701                         rte_flow_error_set(error, errno,
8702                                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
8703                                            NULL,
8704                                            "hardware refuses to create flow");
8705                         goto error;
8706                 }
8707                 if (priv->vmwa_context &&
8708                     dh->vf_vlan.tag && !dh->vf_vlan.created) {
8709                         /*
8710                          * The rule contains the VLAN pattern.
8711                          * For VF we are going to create VLAN
8712                          * interface to make hypervisor set correct
8713                          * e-Switch vport context.
8714                          */
8715                         mlx5_vlan_vmwa_acquire(dev, &dh->vf_vlan);
8716                 }
8717         }
8718         return 0;
8719 error:
8720         if (dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
8721                 flow_dv_default_miss_resource_release(dev);
8722 error_default_miss:
8723         err = rte_errno; /* Save rte_errno before cleanup. */
8724         SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
8725                        handle_idx, dh, next) {
8726                 /* hrxq is union, don't clear it if the flag is not set. */
8727                 if (dh->rix_hrxq) {
8728                         if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
8729                                 mlx5_hrxq_drop_release(dev);
8730                                 dh->rix_hrxq = 0;
8731                         } else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
8732                                 mlx5_hrxq_release(dev, dh->rix_hrxq);
8733                                 dh->rix_hrxq = 0;
8734                         }
8735                 }
8736                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
8737                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
8738         }
8739         rte_errno = err; /* Restore rte_errno. */
8740         return -rte_errno;
8741 }
8742
8743 /**
8744  * Release the flow matcher.
8745  *
8746  * @param dev
8747  *   Pointer to Ethernet device.
8748  * @param handle
8749  *   Pointer to mlx5_flow_handle.
8750  *
8751  * @return
8752  *   1 while a reference on it exists, 0 when freed.
8753  */
8754 static int
8755 flow_dv_matcher_release(struct rte_eth_dev *dev,
8756                         struct mlx5_flow_handle *handle)
8757 {
8758         struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
8759
8760         MLX5_ASSERT(matcher->matcher_object);
8761         DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
8762                 dev->data->port_id, (void *)matcher,
8763                 rte_atomic32_read(&matcher->refcnt));
8764         if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
8765                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
8766                            (matcher->matcher_object));
8767                 LIST_REMOVE(matcher, next);
8768                 /* table ref-- in release interface. */
8769                 flow_dv_tbl_resource_release(dev, matcher->tbl);
8770                 rte_free(matcher);
8771                 DRV_LOG(DEBUG, "port %u matcher %p: removed",
8772                         dev->data->port_id, (void *)matcher);
8773                 return 0;
8774         }
8775         return 1;
8776 }
8777
8778 /**
8779  * Release an encap/decap resource.
8780  *
8781  * @param dev
8782  *   Pointer to Ethernet device.
8783  * @param handle
8784  *   Pointer to mlx5_flow_handle.
8785  *
8786  * @return
8787  *   1 while a reference on it exists, 0 when freed.
8788  */
8789 static int
8790 flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
8791                                      struct mlx5_flow_handle *handle)
8792 {
8793         struct mlx5_priv *priv = dev->data->dev_private;
8794         uint32_t idx = handle->dvh.rix_encap_decap;
8795         struct mlx5_flow_dv_encap_decap_resource *cache_resource;
8796
8797         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
8798                          idx);
8799         if (!cache_resource)
8800                 return 0;
8801         MLX5_ASSERT(cache_resource->action);
8802         DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
8803                 (void *)cache_resource,
8804                 rte_atomic32_read(&cache_resource->refcnt));
8805         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8806                 claim_zero(mlx5_glue->destroy_flow_action
8807                                 (cache_resource->action));
8808                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
8809                              &priv->sh->encaps_decaps, idx,
8810                              cache_resource, next);
8811                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
8812                 DRV_LOG(DEBUG, "encap/decap resource %p: removed",
8813                         (void *)cache_resource);
8814                 return 0;
8815         }
8816         return 1;
8817 }
8818
8819 /**
8820  * Release an jump to table action resource.
8821  *
8822  * @param dev
8823  *   Pointer to Ethernet device.
8824  * @param handle
8825  *   Pointer to mlx5_flow_handle.
8826  *
8827  * @return
8828  *   1 while a reference on it exists, 0 when freed.
8829  */
8830 static int
8831 flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
8832                                   struct mlx5_flow_handle *handle)
8833 {
8834         struct mlx5_priv *priv = dev->data->dev_private;
8835         struct mlx5_flow_dv_jump_tbl_resource *cache_resource;
8836         struct mlx5_flow_tbl_data_entry *tbl_data;
8837
8838         tbl_data = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_JUMP],
8839                              handle->rix_jump);
8840         if (!tbl_data)
8841                 return 0;
8842         cache_resource = &tbl_data->jump;
8843         MLX5_ASSERT(cache_resource->action);
8844         DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
8845                 (void *)cache_resource,
8846                 rte_atomic32_read(&cache_resource->refcnt));
8847         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8848                 claim_zero(mlx5_glue->destroy_flow_action
8849                                 (cache_resource->action));
8850                 /* jump action memory free is inside the table release. */
8851                 flow_dv_tbl_resource_release(dev, &tbl_data->tbl);
8852                 DRV_LOG(DEBUG, "jump table resource %p: removed",
8853                         (void *)cache_resource);
8854                 return 0;
8855         }
8856         return 1;
8857 }
8858
8859 /**
8860  * Release a default miss resource.
8861  *
8862  * @param dev
8863  *   Pointer to Ethernet device.
8864  * @return
8865  *   1 while a reference on it exists, 0 when freed.
8866  */
8867 static int
8868 flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
8869 {
8870         struct mlx5_priv *priv = dev->data->dev_private;
8871         struct mlx5_dev_ctx_shared *sh = priv->sh;
8872         struct mlx5_flow_default_miss_resource *cache_resource =
8873                         &sh->default_miss;
8874
8875         MLX5_ASSERT(cache_resource->action);
8876         DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--",
8877                         (void *)cache_resource->action,
8878                         rte_atomic32_read(&cache_resource->refcnt));
8879         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8880                 claim_zero(mlx5_glue->destroy_flow_action
8881                                 (cache_resource->action));
8882                 DRV_LOG(DEBUG, "default miss resource %p: removed",
8883                                 (void *)cache_resource->action);
8884                 return 0;
8885         }
8886         return 1;
8887 }
8888
8889 /**
8890  * Release a modify-header resource.
8891  *
8892  * @param handle
8893  *   Pointer to mlx5_flow_handle.
8894  *
8895  * @return
8896  *   1 while a reference on it exists, 0 when freed.
8897  */
8898 static int
8899 flow_dv_modify_hdr_resource_release(struct mlx5_flow_handle *handle)
8900 {
8901         struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
8902                                                         handle->dvh.modify_hdr;
8903
8904         MLX5_ASSERT(cache_resource->action);
8905         DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
8906                 (void *)cache_resource,
8907                 rte_atomic32_read(&cache_resource->refcnt));
8908         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8909                 claim_zero(mlx5_glue->destroy_flow_action
8910                                 (cache_resource->action));
8911                 LIST_REMOVE(cache_resource, next);
8912                 rte_free(cache_resource);
8913                 DRV_LOG(DEBUG, "modify-header resource %p: removed",
8914                         (void *)cache_resource);
8915                 return 0;
8916         }
8917         return 1;
8918 }
8919
8920 /**
8921  * Release port ID action resource.
8922  *
8923  * @param dev
8924  *   Pointer to Ethernet device.
8925  * @param handle
8926  *   Pointer to mlx5_flow_handle.
8927  *
8928  * @return
8929  *   1 while a reference on it exists, 0 when freed.
8930  */
8931 static int
8932 flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
8933                                         struct mlx5_flow_handle *handle)
8934 {
8935         struct mlx5_priv *priv = dev->data->dev_private;
8936         struct mlx5_flow_dv_port_id_action_resource *cache_resource;
8937         uint32_t idx = handle->rix_port_id_action;
8938
8939         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
8940                                         idx);
8941         if (!cache_resource)
8942                 return 0;
8943         MLX5_ASSERT(cache_resource->action);
8944         DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
8945                 (void *)cache_resource,
8946                 rte_atomic32_read(&cache_resource->refcnt));
8947         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8948                 claim_zero(mlx5_glue->destroy_flow_action
8949                                 (cache_resource->action));
8950                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
8951                              &priv->sh->port_id_action_list, idx,
8952                              cache_resource, next);
8953                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PORT_ID], idx);
8954                 DRV_LOG(DEBUG, "port id action resource %p: removed",
8955                         (void *)cache_resource);
8956                 return 0;
8957         }
8958         return 1;
8959 }
8960
8961 /**
8962  * Release push vlan action resource.
8963  *
8964  * @param dev
8965  *   Pointer to Ethernet device.
8966  * @param handle
8967  *   Pointer to mlx5_flow_handle.
8968  *
8969  * @return
8970  *   1 while a reference on it exists, 0 when freed.
8971  */
8972 static int
8973 flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
8974                                           struct mlx5_flow_handle *handle)
8975 {
8976         struct mlx5_priv *priv = dev->data->dev_private;
8977         uint32_t idx = handle->dvh.rix_push_vlan;
8978         struct mlx5_flow_dv_push_vlan_action_resource *cache_resource;
8979
8980         cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
8981                                         idx);
8982         if (!cache_resource)
8983                 return 0;
8984         MLX5_ASSERT(cache_resource->action);
8985         DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
8986                 (void *)cache_resource,
8987                 rte_atomic32_read(&cache_resource->refcnt));
8988         if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
8989                 claim_zero(mlx5_glue->destroy_flow_action
8990                                 (cache_resource->action));
8991                 ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
8992                              &priv->sh->push_vlan_action_list, idx,
8993                              cache_resource, next);
8994                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
8995                 DRV_LOG(DEBUG, "push vlan action resource %p: removed",
8996                         (void *)cache_resource);
8997                 return 0;
8998         }
8999         return 1;
9000 }
9001
9002 /**
9003  * Release the fate resource.
9004  *
9005  * @param dev
9006  *   Pointer to Ethernet device.
9007  * @param handle
9008  *   Pointer to mlx5_flow_handle.
9009  */
9010 static void
9011 flow_dv_fate_resource_release(struct rte_eth_dev *dev,
9012                                struct mlx5_flow_handle *handle)
9013 {
9014         if (!handle->rix_fate)
9015                 return;
9016         switch (handle->fate_action) {
9017         case MLX5_FLOW_FATE_DROP:
9018                 mlx5_hrxq_drop_release(dev);
9019                 break;
9020         case MLX5_FLOW_FATE_QUEUE:
9021                 mlx5_hrxq_release(dev, handle->rix_hrxq);
9022                 break;
9023         case MLX5_FLOW_FATE_JUMP:
9024                 flow_dv_jump_tbl_resource_release(dev, handle);
9025                 break;
9026         case MLX5_FLOW_FATE_PORT_ID:
9027                 flow_dv_port_id_action_resource_release(dev, handle);
9028                 break;
9029         case MLX5_FLOW_FATE_DEFAULT_MISS:
9030                 flow_dv_default_miss_resource_release(dev);
9031                 break;
9032         default:
9033                 DRV_LOG(DEBUG, "Incorrect fate action:%d", handle->fate_action);
9034                 break;
9035         }
9036         handle->rix_fate = 0;
9037 }
9038
9039 /**
9040  * Remove the flow from the NIC but keeps it in memory.
9041  * Lock free, (mutex should be acquired by caller).
9042  *
9043  * @param[in] dev
9044  *   Pointer to Ethernet device.
9045  * @param[in, out] flow
9046  *   Pointer to flow structure.
9047  */
9048 static void
9049 __flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
9050 {
9051         struct mlx5_flow_handle *dh;
9052         uint32_t handle_idx;
9053         struct mlx5_priv *priv = dev->data->dev_private;
9054
9055         if (!flow)
9056                 return;
9057         handle_idx = flow->dev_handles;
9058         while (handle_idx) {
9059                 dh = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9060                                     handle_idx);
9061                 if (!dh)
9062                         return;
9063                 if (dh->drv_flow) {
9064                         claim_zero(mlx5_glue->dv_destroy_flow(dh->drv_flow));
9065                         dh->drv_flow = NULL;
9066                 }
9067                 if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
9068                     dh->fate_action == MLX5_FLOW_FATE_QUEUE ||
9069                     dh->fate_action == MLX5_FLOW_FATE_DEFAULT_MISS)
9070                         flow_dv_fate_resource_release(dev, dh);
9071                 if (dh->vf_vlan.tag && dh->vf_vlan.created)
9072                         mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
9073                 handle_idx = dh->next.next;
9074         }
9075 }
9076
9077 /**
9078  * Remove the flow from the NIC and the memory.
9079  * Lock free, (mutex should be acquired by caller).
9080  *
9081  * @param[in] dev
9082  *   Pointer to the Ethernet device structure.
9083  * @param[in, out] flow
9084  *   Pointer to flow structure.
9085  */
9086 static void
9087 __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
9088 {
9089         struct mlx5_flow_handle *dev_handle;
9090         struct mlx5_priv *priv = dev->data->dev_private;
9091
9092         if (!flow)
9093                 return;
9094         __flow_dv_remove(dev, flow);
9095         if (flow->counter) {
9096                 flow_dv_counter_release(dev, flow->counter);
9097                 flow->counter = 0;
9098         }
9099         if (flow->meter) {
9100                 struct mlx5_flow_meter *fm;
9101
9102                 fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
9103                                     flow->meter);
9104                 if (fm)
9105                         mlx5_flow_meter_detach(fm);
9106                 flow->meter = 0;
9107         }
9108         while (flow->dev_handles) {
9109                 uint32_t tmp_idx = flow->dev_handles;
9110
9111                 dev_handle = mlx5_ipool_get(priv->sh->ipool
9112                                             [MLX5_IPOOL_MLX5_FLOW], tmp_idx);
9113                 if (!dev_handle)
9114                         return;
9115                 flow->dev_handles = dev_handle->next.next;
9116                 if (dev_handle->dvh.matcher)
9117                         flow_dv_matcher_release(dev, dev_handle);
9118                 if (dev_handle->dvh.rix_encap_decap)
9119                         flow_dv_encap_decap_resource_release(dev, dev_handle);
9120                 if (dev_handle->dvh.modify_hdr)
9121                         flow_dv_modify_hdr_resource_release(dev_handle);
9122                 if (dev_handle->dvh.rix_push_vlan)
9123                         flow_dv_push_vlan_action_resource_release(dev,
9124                                                                   dev_handle);
9125                 if (dev_handle->dvh.rix_tag)
9126                         flow_dv_tag_release(dev,
9127                                             dev_handle->dvh.rix_tag);
9128                 flow_dv_fate_resource_release(dev, dev_handle);
9129                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
9130                            tmp_idx);
9131         }
9132 }
9133
9134 /**
9135  * Query a dv flow  rule for its statistics via devx.
9136  *
9137  * @param[in] dev
9138  *   Pointer to Ethernet device.
9139  * @param[in] flow
9140  *   Pointer to the sub flow.
9141  * @param[out] data
9142  *   data retrieved by the query.
9143  * @param[out] error
9144  *   Perform verbose error reporting if not NULL.
9145  *
9146  * @return
9147  *   0 on success, a negative errno value otherwise and rte_errno is set.
9148  */
9149 static int
9150 flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
9151                     void *data, struct rte_flow_error *error)
9152 {
9153         struct mlx5_priv *priv = dev->data->dev_private;
9154         struct rte_flow_query_count *qc = data;
9155
9156         if (!priv->config.devx)
9157                 return rte_flow_error_set(error, ENOTSUP,
9158                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9159                                           NULL,
9160                                           "counters are not supported");
9161         if (flow->counter) {
9162                 uint64_t pkts, bytes;
9163                 struct mlx5_flow_counter *cnt;
9164
9165                 cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
9166                                                  NULL);
9167                 int err = _flow_dv_query_count(dev, flow->counter, &pkts,
9168                                                &bytes);
9169
9170                 if (err)
9171                         return rte_flow_error_set(error, -err,
9172                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9173                                         NULL, "cannot read counters");
9174                 qc->hits_set = 1;
9175                 qc->bytes_set = 1;
9176                 qc->hits = pkts - cnt->hits;
9177                 qc->bytes = bytes - cnt->bytes;
9178                 if (qc->reset) {
9179                         cnt->hits = pkts;
9180                         cnt->bytes = bytes;
9181                 }
9182                 return 0;
9183         }
9184         return rte_flow_error_set(error, EINVAL,
9185                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9186                                   NULL,
9187                                   "counters are not available");
9188 }
9189
9190 /**
9191  * Query a flow.
9192  *
9193  * @see rte_flow_query()
9194  * @see rte_flow_ops
9195  */
9196 static int
9197 flow_dv_query(struct rte_eth_dev *dev,
9198               struct rte_flow *flow __rte_unused,
9199               const struct rte_flow_action *actions __rte_unused,
9200               void *data __rte_unused,
9201               struct rte_flow_error *error __rte_unused)
9202 {
9203         int ret = -EINVAL;
9204
9205         for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
9206                 switch (actions->type) {
9207                 case RTE_FLOW_ACTION_TYPE_VOID:
9208                         break;
9209                 case RTE_FLOW_ACTION_TYPE_COUNT:
9210                         ret = flow_dv_query_count(dev, flow, data, error);
9211                         break;
9212                 default:
9213                         return rte_flow_error_set(error, ENOTSUP,
9214                                                   RTE_FLOW_ERROR_TYPE_ACTION,
9215                                                   actions,
9216                                                   "action not supported");
9217                 }
9218         }
9219         return ret;
9220 }
9221
9222 /**
9223  * Destroy the meter table set.
9224  * Lock free, (mutex should be acquired by caller).
9225  *
9226  * @param[in] dev
9227  *   Pointer to Ethernet device.
9228  * @param[in] tbl
9229  *   Pointer to the meter table set.
9230  *
9231  * @return
9232  *   Always 0.
9233  */
9234 static int
9235 flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
9236                         struct mlx5_meter_domains_infos *tbl)
9237 {
9238         struct mlx5_priv *priv = dev->data->dev_private;
9239         struct mlx5_meter_domains_infos *mtd =
9240                                 (struct mlx5_meter_domains_infos *)tbl;
9241
9242         if (!mtd || !priv->config.dv_flow_en)
9243                 return 0;
9244         if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
9245                 claim_zero(mlx5_glue->dv_destroy_flow
9246                           (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
9247         if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
9248                 claim_zero(mlx5_glue->dv_destroy_flow
9249                           (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
9250         if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
9251                 claim_zero(mlx5_glue->dv_destroy_flow
9252                           (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
9253         if (mtd->egress.color_matcher)
9254                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9255                           (mtd->egress.color_matcher));
9256         if (mtd->egress.any_matcher)
9257                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9258                           (mtd->egress.any_matcher));
9259         if (mtd->egress.tbl)
9260                 flow_dv_tbl_resource_release(dev, mtd->egress.tbl);
9261         if (mtd->egress.sfx_tbl)
9262                 flow_dv_tbl_resource_release(dev, mtd->egress.sfx_tbl);
9263         if (mtd->ingress.color_matcher)
9264                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9265                           (mtd->ingress.color_matcher));
9266         if (mtd->ingress.any_matcher)
9267                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9268                           (mtd->ingress.any_matcher));
9269         if (mtd->ingress.tbl)
9270                 flow_dv_tbl_resource_release(dev, mtd->ingress.tbl);
9271         if (mtd->ingress.sfx_tbl)
9272                 flow_dv_tbl_resource_release(dev, mtd->ingress.sfx_tbl);
9273         if (mtd->transfer.color_matcher)
9274                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9275                           (mtd->transfer.color_matcher));
9276         if (mtd->transfer.any_matcher)
9277                 claim_zero(mlx5_glue->dv_destroy_flow_matcher
9278                           (mtd->transfer.any_matcher));
9279         if (mtd->transfer.tbl)
9280                 flow_dv_tbl_resource_release(dev, mtd->transfer.tbl);
9281         if (mtd->transfer.sfx_tbl)
9282                 flow_dv_tbl_resource_release(dev, mtd->transfer.sfx_tbl);
9283         if (mtd->drop_actn)
9284                 claim_zero(mlx5_glue->destroy_flow_action(mtd->drop_actn));
9285         rte_free(mtd);
9286         return 0;
9287 }
9288
9289 /* Number of meter flow actions, count and jump or count and drop. */
9290 #define METER_ACTIONS 2
9291
9292 /**
9293  * Create specify domain meter table and suffix table.
9294  *
9295  * @param[in] dev
9296  *   Pointer to Ethernet device.
9297  * @param[in,out] mtb
9298  *   Pointer to DV meter table set.
9299  * @param[in] egress
9300  *   Table attribute.
9301  * @param[in] transfer
9302  *   Table attribute.
9303  * @param[in] color_reg_c_idx
9304  *   Reg C index for color match.
9305  *
9306  * @return
9307  *   0 on success, -1 otherwise and rte_errno is set.
9308  */
9309 static int
9310 flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
9311                            struct mlx5_meter_domains_infos *mtb,
9312                            uint8_t egress, uint8_t transfer,
9313                            uint32_t color_reg_c_idx)
9314 {
9315         struct mlx5_priv *priv = dev->data->dev_private;
9316         struct mlx5_dev_ctx_shared *sh = priv->sh;
9317         struct mlx5_flow_dv_match_params mask = {
9318                 .size = sizeof(mask.buf),
9319         };
9320         struct mlx5_flow_dv_match_params value = {
9321                 .size = sizeof(value.buf),
9322         };
9323         struct mlx5dv_flow_matcher_attr dv_attr = {
9324                 .type = IBV_FLOW_ATTR_NORMAL,
9325                 .priority = 0,
9326                 .match_criteria_enable = 0,
9327                 .match_mask = (void *)&mask,
9328         };
9329         void *actions[METER_ACTIONS];
9330         struct mlx5_meter_domain_info *dtb;
9331         struct rte_flow_error error;
9332         int i = 0;
9333
9334         if (transfer)
9335                 dtb = &mtb->transfer;
9336         else if (egress)
9337                 dtb = &mtb->egress;
9338         else
9339                 dtb = &mtb->ingress;
9340         /* Create the meter table with METER level. */
9341         dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
9342                                             egress, transfer, &error);
9343         if (!dtb->tbl) {
9344                 DRV_LOG(ERR, "Failed to create meter policer table.");
9345                 return -1;
9346         }
9347         /* Create the meter suffix table with SUFFIX level. */
9348         dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
9349                                             MLX5_FLOW_TABLE_LEVEL_SUFFIX,
9350                                             egress, transfer, &error);
9351         if (!dtb->sfx_tbl) {
9352                 DRV_LOG(ERR, "Failed to create meter suffix table.");
9353                 return -1;
9354         }
9355         /* Create matchers, Any and Color. */
9356         dv_attr.priority = 3;
9357         dv_attr.match_criteria_enable = 0;
9358         dtb->any_matcher = mlx5_glue->dv_create_flow_matcher(sh->ctx,
9359                                                              &dv_attr,
9360                                                              dtb->tbl->obj);
9361         if (!dtb->any_matcher) {
9362                 DRV_LOG(ERR, "Failed to create meter"
9363                              " policer default matcher.");
9364                 goto error_exit;
9365         }
9366         dv_attr.priority = 0;
9367         dv_attr.match_criteria_enable =
9368                                 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
9369         flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
9370                                rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
9371         dtb->color_matcher = mlx5_glue->dv_create_flow_matcher(sh->ctx,
9372                                                                &dv_attr,
9373                                                                dtb->tbl->obj);
9374         if (!dtb->color_matcher) {
9375                 DRV_LOG(ERR, "Failed to create meter policer color matcher.");
9376                 goto error_exit;
9377         }
9378         if (mtb->count_actns[RTE_MTR_DROPPED])
9379                 actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
9380         actions[i++] = mtb->drop_actn;
9381         /* Default rule: lowest priority, match any, actions: drop. */
9382         dtb->policer_rules[RTE_MTR_DROPPED] =
9383                         mlx5_glue->dv_create_flow(dtb->any_matcher,
9384                                                  (void *)&value, i, actions);
9385         if (!dtb->policer_rules[RTE_MTR_DROPPED]) {
9386                 DRV_LOG(ERR, "Failed to create meter policer drop rule.");
9387                 goto error_exit;
9388         }
9389         return 0;
9390 error_exit:
9391         return -1;
9392 }
9393
9394 /**
9395  * Create the needed meter and suffix tables.
9396  * Lock free, (mutex should be acquired by caller).
9397  *
9398  * @param[in] dev
9399  *   Pointer to Ethernet device.
9400  * @param[in] fm
9401  *   Pointer to the flow meter.
9402  *
9403  * @return
9404  *   Pointer to table set on success, NULL otherwise and rte_errno is set.
9405  */
9406 static struct mlx5_meter_domains_infos *
9407 flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
9408                        const struct mlx5_flow_meter *fm)
9409 {
9410         struct mlx5_priv *priv = dev->data->dev_private;
9411         struct mlx5_meter_domains_infos *mtb;
9412         int ret;
9413         int i;
9414
9415         if (!priv->mtr_en) {
9416                 rte_errno = ENOTSUP;
9417                 return NULL;
9418         }
9419         mtb = rte_calloc(__func__, 1, sizeof(*mtb), 0);
9420         if (!mtb) {
9421                 DRV_LOG(ERR, "Failed to allocate memory for meter.");
9422                 return NULL;
9423         }
9424         /* Create meter count actions */
9425         for (i = 0; i <= RTE_MTR_DROPPED; i++) {
9426                 struct mlx5_flow_counter *cnt;
9427                 if (!fm->policer_stats.cnt[i])
9428                         continue;
9429                 cnt = flow_dv_counter_get_by_idx(dev,
9430                       fm->policer_stats.cnt[i], NULL);
9431                 mtb->count_actns[i] = cnt->action;
9432         }
9433         /* Create drop action. */
9434         mtb->drop_actn = mlx5_glue->dr_create_flow_action_drop();
9435         if (!mtb->drop_actn) {
9436                 DRV_LOG(ERR, "Failed to create drop action.");
9437                 goto error_exit;
9438         }
9439         /* Egress meter table. */
9440         ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
9441         if (ret) {
9442                 DRV_LOG(ERR, "Failed to prepare egress meter table.");
9443                 goto error_exit;
9444         }
9445         /* Ingress meter table. */
9446         ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
9447         if (ret) {
9448                 DRV_LOG(ERR, "Failed to prepare ingress meter table.");
9449                 goto error_exit;
9450         }
9451         /* FDB meter table. */
9452         if (priv->config.dv_esw_en) {
9453                 ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
9454                                                  priv->mtr_color_reg);
9455                 if (ret) {
9456                         DRV_LOG(ERR, "Failed to prepare fdb meter table.");
9457                         goto error_exit;
9458                 }
9459         }
9460         return mtb;
9461 error_exit:
9462         flow_dv_destroy_mtr_tbl(dev, mtb);
9463         return NULL;
9464 }
9465
9466 /**
9467  * Destroy domain policer rule.
9468  *
9469  * @param[in] dt
9470  *   Pointer to domain table.
9471  */
9472 static void
9473 flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
9474 {
9475         int i;
9476
9477         for (i = 0; i < RTE_MTR_DROPPED; i++) {
9478                 if (dt->policer_rules[i]) {
9479                         claim_zero(mlx5_glue->dv_destroy_flow
9480                                   (dt->policer_rules[i]));
9481                         dt->policer_rules[i] = NULL;
9482                 }
9483         }
9484         if (dt->jump_actn) {
9485                 claim_zero(mlx5_glue->destroy_flow_action(dt->jump_actn));
9486                 dt->jump_actn = NULL;
9487         }
9488 }
9489
9490 /**
9491  * Destroy policer rules.
9492  *
9493  * @param[in] dev
9494  *   Pointer to Ethernet device.
9495  * @param[in] fm
9496  *   Pointer to flow meter structure.
9497  * @param[in] attr
9498  *   Pointer to flow attributes.
9499  *
9500  * @return
9501  *   Always 0.
9502  */
9503 static int
9504 flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
9505                               const struct mlx5_flow_meter *fm,
9506                               const struct rte_flow_attr *attr)
9507 {
9508         struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL;
9509
9510         if (!mtb)
9511                 return 0;
9512         if (attr->egress)
9513                 flow_dv_destroy_domain_policer_rule(&mtb->egress);
9514         if (attr->ingress)
9515                 flow_dv_destroy_domain_policer_rule(&mtb->ingress);
9516         if (attr->transfer)
9517                 flow_dv_destroy_domain_policer_rule(&mtb->transfer);
9518         return 0;
9519 }
9520
9521 /**
9522  * Create specify domain meter policer rule.
9523  *
9524  * @param[in] fm
9525  *   Pointer to flow meter structure.
9526  * @param[in] mtb
9527  *   Pointer to DV meter table set.
9528  * @param[in] mtr_reg_c
9529  *   Color match REG_C.
9530  *
9531  * @return
9532  *   0 on success, -1 otherwise.
9533  */
9534 static int
9535 flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
9536                                     struct mlx5_meter_domain_info *dtb,
9537                                     uint8_t mtr_reg_c)
9538 {
9539         struct mlx5_flow_dv_match_params matcher = {
9540                 .size = sizeof(matcher.buf),
9541         };
9542         struct mlx5_flow_dv_match_params value = {
9543                 .size = sizeof(value.buf),
9544         };
9545         struct mlx5_meter_domains_infos *mtb = fm->mfts;
9546         void *actions[METER_ACTIONS];
9547         int i;
9548
9549         /* Create jump action. */
9550         if (!dtb->jump_actn)
9551                 dtb->jump_actn =
9552                         mlx5_glue->dr_create_flow_action_dest_flow_tbl
9553                                                         (dtb->sfx_tbl->obj);
9554         if (!dtb->jump_actn) {
9555                 DRV_LOG(ERR, "Failed to create policer jump action.");
9556                 goto error;
9557         }
9558         for (i = 0; i < RTE_MTR_DROPPED; i++) {
9559                 int j = 0;
9560
9561                 flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
9562                                        rte_col_2_mlx5_col(i), UINT8_MAX);
9563                 if (mtb->count_actns[i])
9564                         actions[j++] = mtb->count_actns[i];
9565                 if (fm->action[i] == MTR_POLICER_ACTION_DROP)
9566                         actions[j++] = mtb->drop_actn;
9567                 else
9568                         actions[j++] = dtb->jump_actn;
9569                 dtb->policer_rules[i] =
9570                         mlx5_glue->dv_create_flow(dtb->color_matcher,
9571                                                  (void *)&value,
9572                                                   j, actions);
9573                 if (!dtb->policer_rules[i]) {
9574                         DRV_LOG(ERR, "Failed to create policer rule.");
9575                         goto error;
9576                 }
9577         }
9578         return 0;
9579 error:
9580         rte_errno = errno;
9581         return -1;
9582 }
9583
9584 /**
9585  * Create policer rules.
9586  *
9587  * @param[in] dev
9588  *   Pointer to Ethernet device.
9589  * @param[in] fm
9590  *   Pointer to flow meter structure.
9591  * @param[in] attr
9592  *   Pointer to flow attributes.
9593  *
9594  * @return
9595  *   0 on success, -1 otherwise.
9596  */
9597 static int
9598 flow_dv_create_policer_rules(struct rte_eth_dev *dev,
9599                              struct mlx5_flow_meter *fm,
9600                              const struct rte_flow_attr *attr)
9601 {
9602         struct mlx5_priv *priv = dev->data->dev_private;
9603         struct mlx5_meter_domains_infos *mtb = fm->mfts;
9604         int ret;
9605
9606         if (attr->egress) {
9607                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
9608                                                 priv->mtr_color_reg);
9609                 if (ret) {
9610                         DRV_LOG(ERR, "Failed to create egress policer.");
9611                         goto error;
9612                 }
9613         }
9614         if (attr->ingress) {
9615                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
9616                                                 priv->mtr_color_reg);
9617                 if (ret) {
9618                         DRV_LOG(ERR, "Failed to create ingress policer.");
9619                         goto error;
9620                 }
9621         }
9622         if (attr->transfer) {
9623                 ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
9624                                                 priv->mtr_color_reg);
9625                 if (ret) {
9626                         DRV_LOG(ERR, "Failed to create transfer policer.");
9627                         goto error;
9628                 }
9629         }
9630         return 0;
9631 error:
9632         flow_dv_destroy_policer_rules(dev, fm, attr);
9633         return -1;
9634 }
9635
9636 /**
9637  * Query a devx counter.
9638  *
9639  * @param[in] dev
9640  *   Pointer to the Ethernet device structure.
9641  * @param[in] cnt
9642  *   Index to the flow counter.
9643  * @param[in] clear
9644  *   Set to clear the counter statistics.
9645  * @param[out] pkts
9646  *   The statistics value of packets.
9647  * @param[out] bytes
9648  *   The statistics value of bytes.
9649  *
9650  * @return
9651  *   0 on success, otherwise return -1.
9652  */
9653 static int
9654 flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
9655                       uint64_t *pkts, uint64_t *bytes)
9656 {
9657         struct mlx5_priv *priv = dev->data->dev_private;
9658         struct mlx5_flow_counter *cnt;
9659         uint64_t inn_pkts, inn_bytes;
9660         int ret;
9661
9662         if (!priv->config.devx)
9663                 return -1;
9664
9665         ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
9666         if (ret)
9667                 return -1;
9668         cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
9669         *pkts = inn_pkts - cnt->hits;
9670         *bytes = inn_bytes - cnt->bytes;
9671         if (clear) {
9672                 cnt->hits = inn_pkts;
9673                 cnt->bytes = inn_bytes;
9674         }
9675         return 0;
9676 }
9677
9678 /**
9679  * Get aged-out flows.
9680  *
9681  * @param[in] dev
9682  *   Pointer to the Ethernet device structure.
9683  * @param[in] context
9684  *   The address of an array of pointers to the aged-out flows contexts.
9685  * @param[in] nb_contexts
9686  *   The length of context array pointers.
9687  * @param[out] error
9688  *   Perform verbose error reporting if not NULL. Initialized in case of
9689  *   error only.
9690  *
9691  * @return
9692  *   how many contexts get in success, otherwise negative errno value.
9693  *   if nb_contexts is 0, return the amount of all aged contexts.
9694  *   if nb_contexts is not 0 , return the amount of aged flows reported
9695  *   in the context array.
9696  * @note: only stub for now
9697  */
9698 static int
9699 flow_get_aged_flows(struct rte_eth_dev *dev,
9700                     void **context,
9701                     uint32_t nb_contexts,
9702                     struct rte_flow_error *error)
9703 {
9704         struct mlx5_priv *priv = dev->data->dev_private;
9705         struct mlx5_age_info *age_info;
9706         struct mlx5_age_param *age_param;
9707         struct mlx5_flow_counter *counter;
9708         int nb_flows = 0;
9709
9710         if (nb_contexts && !context)
9711                 return rte_flow_error_set(error, EINVAL,
9712                                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
9713                                           NULL,
9714                                           "Should assign at least one flow or"
9715                                           " context to get if nb_contexts != 0");
9716         age_info = GET_PORT_AGE_INFO(priv);
9717         rte_spinlock_lock(&age_info->aged_sl);
9718         TAILQ_FOREACH(counter, &age_info->aged_counters, next) {
9719                 nb_flows++;
9720                 if (nb_contexts) {
9721                         age_param = MLX5_CNT_TO_AGE(counter);
9722                         context[nb_flows - 1] = age_param->context;
9723                         if (!(--nb_contexts))
9724                                 break;
9725                 }
9726         }
9727         rte_spinlock_unlock(&age_info->aged_sl);
9728         MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
9729         return nb_flows;
9730 }
9731
9732 /*
9733  * Mutex-protected thunk to lock-free  __flow_dv_translate().
9734  */
9735 static int
9736 flow_dv_translate(struct rte_eth_dev *dev,
9737                   struct mlx5_flow *dev_flow,
9738                   const struct rte_flow_attr *attr,
9739                   const struct rte_flow_item items[],
9740                   const struct rte_flow_action actions[],
9741                   struct rte_flow_error *error)
9742 {
9743         int ret;
9744
9745         flow_dv_shared_lock(dev);
9746         ret = __flow_dv_translate(dev, dev_flow, attr, items, actions, error);
9747         flow_dv_shared_unlock(dev);
9748         return ret;
9749 }
9750
9751 /*
9752  * Mutex-protected thunk to lock-free  __flow_dv_apply().
9753  */
9754 static int
9755 flow_dv_apply(struct rte_eth_dev *dev,
9756               struct rte_flow *flow,
9757               struct rte_flow_error *error)
9758 {
9759         int ret;
9760
9761         flow_dv_shared_lock(dev);
9762         ret = __flow_dv_apply(dev, flow, error);
9763         flow_dv_shared_unlock(dev);
9764         return ret;
9765 }
9766
9767 /*
9768  * Mutex-protected thunk to lock-free __flow_dv_remove().
9769  */
9770 static void
9771 flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
9772 {
9773         flow_dv_shared_lock(dev);
9774         __flow_dv_remove(dev, flow);
9775         flow_dv_shared_unlock(dev);
9776 }
9777
9778 /*
9779  * Mutex-protected thunk to lock-free __flow_dv_destroy().
9780  */
9781 static void
9782 flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
9783 {
9784         flow_dv_shared_lock(dev);
9785         __flow_dv_destroy(dev, flow);
9786         flow_dv_shared_unlock(dev);
9787 }
9788
9789 /*
9790  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
9791  */
9792 static uint32_t
9793 flow_dv_counter_allocate(struct rte_eth_dev *dev)
9794 {
9795         uint32_t cnt;
9796
9797         flow_dv_shared_lock(dev);
9798         cnt = flow_dv_counter_alloc(dev, 0, 0, 1, 0);
9799         flow_dv_shared_unlock(dev);
9800         return cnt;
9801 }
9802
9803 /*
9804  * Mutex-protected thunk to lock-free flow_dv_counter_release().
9805  */
9806 static void
9807 flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
9808 {
9809         flow_dv_shared_lock(dev);
9810         flow_dv_counter_release(dev, cnt);
9811         flow_dv_shared_unlock(dev);
9812 }
9813
9814 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
9815         .validate = flow_dv_validate,
9816         .prepare = flow_dv_prepare,
9817         .translate = flow_dv_translate,
9818         .apply = flow_dv_apply,
9819         .remove = flow_dv_remove,
9820         .destroy = flow_dv_destroy,
9821         .query = flow_dv_query,
9822         .create_mtr_tbls = flow_dv_create_mtr_tbl,
9823         .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
9824         .create_policer_rules = flow_dv_create_policer_rules,
9825         .destroy_policer_rules = flow_dv_destroy_policer_rules,
9826         .counter_alloc = flow_dv_counter_allocate,
9827         .counter_free = flow_dv_counter_free,
9828         .counter_query = flow_dv_counter_query,
9829         .get_aged_flows = flow_get_aged_flows,
9830 };
9831
9832 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */